gen-pipeline.sh (6475B)
1 #!/bin/sh 2 # mk/gen-pipeline.sh - emit .conductor.yml from the dependency graph. 3 # 4 # ./mk/gen-pipeline.sh print the pipeline 5 # ./mk/gen-pipeline.sh --write write it to .conductor.yml 6 # ./mk/gen-pipeline.sh --check exit 1 if .conductor.yml is out of date 7 # 8 # The conductor treats this file as data and will not run repository code to 9 # build its graph, so every edge has to be written out literally. Generating 10 # keeps makedepends the single source of truth; test_deps.sh catches drift. 11 set -eu 12 13 HERE=$(cd "$(dirname "$0")" && pwd) 14 ROOT=$(cd "${HERE}/.." && pwd) 15 OUT="${ROOT}/.conductor.yml" 16 17 # shellcheck disable=SC1091 18 . "${HERE}/channels.inc" 19 20 # Unconditional: an existing tree may be stale, and the pkgvers come from it. 21 for c in $(chan_targets); do 22 "${HERE}/channel-assemble.sh" "${c}" >/dev/null 23 done 24 25 BUILDER_IMAGE="${UNOS_BUILDER_IMAGE:-finwo/unos-builder}" 26 [ -f "${ROOT}/builder/VERSION" ] || { echo "gen-pipeline.sh: no builder/VERSION" >&2; exit 1; } 27 read -r BUILDER_VERSION < "${ROOT}/builder/VERSION" 28 [ -n "${BUILDER_VERSION}" ] || { echo "gen-pipeline.sh: builder/VERSION is empty" >&2; exit 1; } 29 BUILDER_TAG="${BUILDER_IMAGE}:${BUILDER_VERSION}" 30 31 MODE=print 32 case "${1:-}" in 33 --write) MODE=write ;; 34 --check) MODE=check ;; 35 -h|--help) sed -n '2,8p' "$0"; exit 0 ;; 36 "") ;; 37 *) echo "gen-pipeline.sh: unknown argument: $1" >&2; exit 1 ;; 38 esac 39 40 # Task names allow letters, digits, underscore, dot and hyphen, so `libstdc++` 41 # cannot be used directly. `+` becomes `x` (libstdc++ -> libstdcxx, the 42 # conventional spelling) rather than a hyphen, which would render it 43 # `libstdc--` and read like a typo. Anything else unexpected becomes a hyphen. 44 task_name() { 45 printf 'pkg-%s-%s' "$1" "$(printf '%s' "$2" | sed 's/+/x/g' | tr -c 'A-Za-z0-9_.-' '-')" 46 } 47 48 # Resolved here rather than in CI: the pipeline is generated, and sourcing a 49 # template needs UNOS_MKDIR (kernel.inc) which a bare CI checkout lacks. 50 pkgver() { 51 ( 52 version=; revision=0 53 SYSROOT=; DESTDIR=; WRKSRC=; FILESDIR=; WORK=; SRCDEST= 54 UNOS_MKDIR="${HERE}" 55 die() { :; } ; msg() { :; } ; vinstall() { :; } ; vmkdir() { :; } 56 . "$(chan_pkgdir "$1")/$2/template" >/dev/null 2>&1 || true 57 [ -n "${version}" ] || { echo "gen-pipeline.sh: no version for $2 in $1" >&2; exit 1; } 58 printf '%s-r%s' "${version}" "${revision}" 59 ) 60 } 61 62 emit() { 63 cat <<HEADER 64 # GENERATED by mk/gen-pipeline.sh -- do not edit by hand. 65 # 66 # Regenerate with: ./mk/gen-pipeline.sh --write 67 # Verified by: tests/unit/test_deps.sh 68 69 version: 1 70 71 defaults: 72 image: ${BUILDER_TAG} 73 timeout: 2h 74 env: 75 DEBIAN_FRONTEND: noninteractive 76 77 tasks: 78 builder: 79 timeout: 30m 80 arch: [x86_64] 81 image: docker:28-cli 82 requires: [docker] 83 script: 84 - ./mk/publish-builder.sh 85 86 check: 87 needs: 88 - builder 89 timeout: 30m 90 arch: [x86_64] 91 requires: [unos-sign-key] 92 script: 93 # --check, not an install: the toolchain is baked into the builder image, 94 # and this is what fails when a template outgrows it without a bump. 95 - ./mk/ci-setup.sh --check 96 # Before deps.sh check, not after: muon is a declared hostmakedepends of 97 # apk-tools but is built here rather than installed from a distribution, 98 # so the check reports it missing until bootstrap has produced it. 99 - ./mk/bootstrap-host.sh 100 - ./mk/channel-lint.sh 101 - ./mk/gen-pipeline.sh --check 102 - ./tests/run.sh --unit 103 HEADER 104 for c in $(chan_targets); do 105 printf ' - ./mk/channel-assemble.sh %s\n' "${c}" 106 printf ' - ./mk/deps.sh --channel %s check\n' "${c}" 107 printf ' - ./mk/bootstrap-repo.sh --channel %s\n' "${c}" 108 done 109 printf '\n' 110 111 for c in $(chan_targets); do 112 for p in $(cd "${ROOT}" && ./mk/deps.sh --channel "${c}" order); do 113 # graph, not vars: makedepends may name a subpackage, and the task that 114 # builds it is the owning template's. 115 deps=$(cd "${ROOT}" && ./mk/deps.sh --channel "${c}" graph | awk -v p="$p" '$1 == p { print $2 }') 116 printf ' %s:\n' "$(task_name "${c}" "$p")" 117 printf ' needs:\n' 118 printf ' - check\n' 119 for d in ${deps}; do 120 printf ' - %s\n' "$(task_name "${c}" "$d")" 121 done 122 printf ' arch: [x86_64]\n' 123 printf ' requires: [unos-sign-key]\n' 124 printf ' script:\n' 125 printf ' - ./mk/check-revision.sh --channel %s %s --pkgver %s --repo "$BUCKET_PUBLIC_URL" || { rc=$?; [ "$rc" = 1 ] && exit 0; exit "$rc"; }\n' "${c}" "$p" "$(pkgver "${c}" "$p")" 126 printf ' - ./mk/channel-assemble.sh %s\n' "${c}" 127 printf ' - ./mk/bootstrap-host.sh\n' 128 printf ' - ./mk/sysroot.sh --channel %s --arch "$ARCH" --repo "$BUCKET_PUBLIC_URL" %s\n' "${c}" "$p" 129 printf ' - ./mk/build.sh --channel %s %s\n' "${c}" "$p" 130 printf ' - ./mk/publish.sh --channel %s --arch "$ARCH" %s\n' "${c}" "$p" 131 printf '\n' 132 done 133 done 134 135 # One index per channel, rewritten after every package in it has published. 136 # Per-package would race: two finishing together would each rebuild the index 137 # from their own partial view of the repo. 138 for c in $(chan_targets); do 139 printf ' index-%s:\n' "${c}" 140 printf ' after:\n' 141 for p in $(cd "${ROOT}" && ./mk/deps.sh --channel "${c}" order); do 142 printf ' - %s\n' "$(task_name "${c}" "$p")" 143 done 144 printf ' arch: [x86_64]\n' 145 printf ' requires: [unos-sign-key]\n' 146 printf ' script:\n' 147 printf ' - ./mk/channel-assemble.sh %s\n' "${c}" 148 printf ' - ./mk/bootstrap-host.sh\n' 149 printf ' - ./mk/repo-pull.sh --channel %s --arch "$ARCH" --repo "$BUCKET_PUBLIC_URL"\n' "${c}" 150 printf ' - ./mk/repo-index.sh --channel %s\n' "${c}" 151 printf ' - ./mk/publish.sh --channel %s --arch "$ARCH" --index-only\n' "${c}" 152 printf '\n' 153 done 154 } 155 156 case "${MODE}" in 157 print) emit ;; 158 write) emit > "${OUT}"; echo "wrote ${OUT}" ;; 159 check) 160 tmp=$(mktemp) 161 trap 'rm -f "${tmp}"' EXIT 162 emit > "${tmp}" 163 if [ ! -f "${OUT}" ]; then 164 echo "gen-pipeline.sh: .conductor.yml is missing; run ./mk/gen-pipeline.sh --write" >&2 165 exit 1 166 fi 167 if ! diff -u "${OUT}" "${tmp}" >/dev/null 2>&1; then 168 echo "gen-pipeline.sh: .conductor.yml is out of date" >&2 169 diff -u "${OUT}" "${tmp}" | head -40 >&2 170 echo "gen-pipeline.sh: run ./mk/gen-pipeline.sh --write" >&2 171 exit 1 172 fi 173 echo "gen-pipeline.sh: .conductor.yml is current" 174 ;; 175 esac