gen-pipeline.sh (6263B)
1 #!/bin/sh 2 # mk/gen-pipeline.sh - emit .conductor.yml from the dependency graph. 3 # 4 # ./mk/gen-pipeline.sh print the pipeline 5 # ./mk/gen-pipeline.sh --write write it to .conductor.yml 6 # ./mk/gen-pipeline.sh --check exit 1 if .conductor.yml is out of date 7 # 8 # WHY GENERATED 9 # ------------- 10 # The conductor does not run repository code while building its task graph -- 11 # the pipeline file is data, read at the pushed commit. So the per-package 12 # dependency edges have to be written out literally. Maintaining ~30 tasks and 13 # their `needs` by hand against packages/*/template guarantees they drift, and 14 # the failure mode is a package building before the library it links against. 15 # 16 # Generating instead means makedepends stays the single source of truth, and 17 # tests/unit/test_deps.sh fails if the committed file no longer matches. 18 set -eu 19 20 HERE=$(cd "$(dirname "$0")" && pwd) 21 ROOT=$(cd "${HERE}/.." && pwd) 22 OUT="${ROOT}/.conductor.yml" 23 24 MODE=print 25 case "${1:-}" in 26 --write) MODE=write ;; 27 --check) MODE=check ;; 28 -h|--help) sed -n '2,8p' "$0"; exit 0 ;; 29 "") ;; 30 *) echo "gen-pipeline.sh: unknown argument: $1" >&2; exit 1 ;; 31 esac 32 33 # Task names allow letters, digits, underscore, dot and hyphen, so `libstdc++` 34 # cannot be used directly. `+` becomes `x` (libstdc++ -> libstdcxx, the 35 # conventional spelling) rather than a hyphen, which would render it 36 # `libstdc--` and read like a typo. Anything else unexpected becomes a hyphen. 37 task_name() { 38 printf 'pkg-%s' "$(printf '%s' "$1" | sed 's/+/x/g' | tr -c 'A-Za-z0-9_.-' '-')" 39 } 40 41 emit() { 42 cat <<'HEADER' 43 # GENERATED by mk/gen-pipeline.sh -- do not edit by hand. 44 # 45 # Regenerate with: ./mk/gen-pipeline.sh --write 46 # Verified by: tests/unit/test_deps.sh 47 # 48 # One task per package, with `needs` taken from each template's makedepends, 49 # so a package never builds before something it links against. The list is 50 # written out rather than discovered because the conductor treats this file as 51 # data and will not execute repository code to build its graph. 52 # 53 # No mounts are provided and artifacts are not shared between tasks. Each build 54 # therefore seeds its sysroot from the published S3 repo (mk/sysroot.sh) and 55 # publishes its own result back (mk/publish.sh). `needs` is what guarantees a 56 # dependency is already published by the time a dependent starts. 57 # 58 # Credentials arrive as environment: BUCKET_ACCESS_KEY, BUCKET_SECRET_KEY, 59 # BUCKET_NAME, BUCKET_ENDPOINT (URL), BUCKET_REGION (region name), 60 # BUCKET_PUBLIC_URL (public https endpoint, e.g. https://repo.unos.finwo.net/). 61 # 62 # Signing comes from the unos-sign-key worker feature, which mounts the key and 63 # exports UNOS_KEY_APK_RSA_PRI. Each worker holds a distinct key, so the signer 64 # varies with scheduling and packages/unos-keys must trust all of them. 65 66 version: 1 67 68 defaults: 69 image: debian:bookworm-slim 70 timeout: 2h 71 env: 72 DEBIAN_FRONTEND: noninteractive 73 74 tasks: 75 check: 76 timeout: 30m 77 arch: [x86_64] 78 requires: [unos-sign-key] 79 script: 80 - ./mk/ci-setup.sh 81 # Before deps.sh check, not after: muon is a declared hostmakedepends of 82 # apk-tools but is built here rather than installed from a distribution, 83 # so the check reports it missing until bootstrap has produced it. 84 - ./mk/bootstrap-host.sh 85 - ./mk/deps.sh check 86 - ./mk/gen-pipeline.sh --check 87 - ./tests/run.sh --unit 88 - ./mk/bootstrap-repo.sh 89 90 HEADER 91 92 # One task per package, in dependency order so the file reads top-down. 93 for p in $(cd "${ROOT}" && ./mk/deps.sh order); do 94 deps=$(cd "${ROOT}" && ./mk/deps.sh vars "$p" | sed -n 's/^makedepends: *//p') 95 printf ' %s:\n' "$(task_name "$p")" 96 printf ' needs:\n' 97 printf ' - check\n' 98 for d in ${deps}; do 99 printf ' - %s\n' "$(task_name "$d")" 100 done 101 # arch is declared so $ARCH exists in the script; it is only injected for 102 # tasks that declare it. One arch today, but adding aarch64 is then a 103 # single edit here rather than a rewrite. 104 printf ' arch: [x86_64]\n' 105 # The signing key lives on the worker and is never seen by the conductor; 106 # the task asks for it by capability name. Project-scoped on purpose: the 107 # feature namespace is global to the conductor, which hosts more than UNOS. 108 # The feature provides UNOS_KEY_APK_RSA_PRI, which mk/sign-key.inc reads. 109 printf ' requires: [unos-sign-key]\n' 110 printf ' script:\n' 111 printf ' - ./mk/check-revision.sh %s --repo "${BUCKET_PUBLIC_URL%%/}/$ARCH" || exit 0\n' "$p" 112 printf ' - ./mk/ci-setup.sh\n' 113 printf ' - ./mk/bootstrap-host.sh\n' 114 printf ' - ./mk/sysroot.sh --arch "$ARCH" --repo "$BUCKET_PUBLIC_URL" %s\n' "$p" 115 printf ' - ./mk/build.sh %s\n' "$p" 116 printf ' - ./mk/publish.sh --arch "$ARCH" %s\n' "$p" 117 printf '\n' 118 done 119 120 # The index is rewritten once, after every package has been published. 121 # Doing it per-package would race: two packages finishing together would 122 # each rebuild the index from their own partial view of the repo. 123 cat <<'FOOTER' 124 index: 125 needs: 126 FOOTER 127 for p in $(cd "${ROOT}" && ./mk/deps.sh order); do 128 printf ' - %s\n' "$(task_name "$p")" 129 done 130 cat <<'FOOTER' 131 arch: [x86_64] 132 requires: [unos-sign-key] 133 script: 134 - ./mk/ci-setup.sh 135 - ./mk/bootstrap-host.sh 136 # Pull every published package down so the index describes the whole 137 # repo, not just whatever this container happened to build. 138 - ./mk/sysroot.sh --arch "$ARCH" --repo "$BUCKET_PUBLIC_URL" 139 - ./mk/repo-index.sh 140 - ./mk/publish.sh --arch "$ARCH" --index-only 141 FOOTER 142 } 143 144 case "${MODE}" in 145 print) emit ;; 146 write) emit > "${OUT}"; echo "wrote ${OUT}" ;; 147 check) 148 tmp=$(mktemp) 149 trap 'rm -f "${tmp}"' EXIT 150 emit > "${tmp}" 151 if [ ! -f "${OUT}" ]; then 152 echo "gen-pipeline.sh: .conductor.yml is missing; run ./mk/gen-pipeline.sh --write" >&2 153 exit 1 154 fi 155 if ! diff -u "${OUT}" "${tmp}" >/dev/null 2>&1; then 156 echo "gen-pipeline.sh: .conductor.yml is out of date" >&2 157 diff -u "${OUT}" "${tmp}" | head -40 >&2 158 echo "gen-pipeline.sh: run ./mk/gen-pipeline.sh --write" >&2 159 exit 1 160 fi 161 echo "gen-pipeline.sh: .conductor.yml is current" 162 ;; 163 esac