publish.sh (11703B)
1 #!/bin/sh 2 # deploy/publish.sh - build and push the images to Docker Hub 3 # 4 # Run by the publish job in .conductor.yml, which restricts it to main and 5 # to release tags. Safe to run by hand for a dry run. 6 # 7 # What gets pushed, from CONDUCTOR_REF: 8 # 9 # refs/heads/main finwo/conductor:main and :latest 10 # refs/heads/topic finwo/conductor:topic 11 # refs/tags/v1.2.0 finwo/conductor:1.2.0 and :1.2 12 # 13 # A branch publishes its own name. main carries :latest on top of that, 14 # and nothing else ever moves it, so :latest is always what main is. 15 # 16 # Needs REGISTRY_USERNAME and REGISTRY_TOKEN as project variables. They are 17 # masked in the log by the conductor, but this still keeps them off the 18 # command line, where they would show up in a process list. 19 # 20 # DRY_RUN=1 build for the host architecture only, and push nothing 21 22 set -eu 23 24 : "${CONDUCTOR_REF:?CONDUCTOR_REF is not set}" 25 : "${CONDUCTOR_SHA:?CONDUCTOR_SHA is not set}" 26 27 DRY_RUN="${DRY_RUN:-0}" 28 CONDUCTOR_IMAGE="${CONDUCTOR_IMAGE:-finwo/conductor}" 29 WORKER_IMAGE="${WORKER_IMAGE:-finwo/conductor-worker}" 30 31 # riscv64 has no node image and no static docker CLI, which is why both 32 # Dockerfiles are built on Alpine. Everything here is available there. 33 PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64,linux/riscv64}" 34 35 ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) 36 SHORT=$(printf '%s' "${CONDUCTOR_SHA}" | cut -c1-12) 37 38 log() { printf '\n== %s\n' "$*"; } 39 fail() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } 40 41 # A docker tag is [A-Za-z0-9_][A-Za-z0-9._-]*, so a branch such as 42 # feature/x cannot be used as it stands. 43 docker_tag() { 44 tag=$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-' | cut -c1-128) 45 case "${tag}" in 46 [!A-Za-z0-9_]*) tag="x${tag}" ;; 47 esac 48 printf '%s\n' "${tag}" 49 } 50 51 # What a branch publishes on top of its own name. Give another branch an 52 # alias, a pre-release series for instance, by adding a line here. 53 branch_aliases() { 54 case "$1" in 55 main) printf 'latest\n' ;; 56 esac 57 } 58 59 # Turns a ref into the list of tags it should publish. 60 # 61 # A version tag also publishes the major.minor series, so v1.2.1 moves 62 # :1.2 as well. Deliberately no bare major: :1 moving across minor 63 # releases tends to surprise people more than it helps them. 64 tags_for_ref() { 65 ref=$1 66 case "${ref}" in 67 refs/heads/*) 68 branch=${ref#refs/heads/} 69 docker_tag "${branch}" 70 branch_aliases "${branch}" 71 ;; 72 refs/tags/v*) 73 version=${ref#refs/tags/v} 74 case "${version}" in 75 *[!0-9.]*) 76 fail "tag ${ref} is not a plain version, refusing to guess what to publish" 77 ;; 78 esac 79 printf '%s\n' "${version}" 80 # v1.2.3 -> 1.2, but v1.2 stays as it is. 81 series=$(printf '%s' "${version}" | cut -d. -f1-2) 82 [ "${series}" = "${version}" ] || printf '%s\n' "${series}" 83 ;; 84 *) 85 fail "ref ${ref} does not publish anything; the only: rule in .conductor.yml should have prevented this" 86 ;; 87 esac 88 } 89 90 TAGS=$(tags_for_ref "${CONDUCTOR_REF}") 91 92 log "publishing from ${CONDUCTOR_REF} at ${SHORT}" 93 printf 'tags:\n' 94 printf ' %s\n' ${TAGS} 95 printf 'platforms: %s\n' "${PLATFORMS}" 96 97 # Collects --tag arguments for one image. 98 tag_args() { 99 image=$1 100 for tag in ${TAGS}; do 101 printf -- '--tag\n%s:%s\n' "${image}" "${tag}" 102 done 103 } 104 105 if [ "${DRY_RUN}" = 1 ]; then 106 log "dry run: building for this architecture only, pushing nothing" 107 for spec in "deploy/Dockerfile ${CONDUCTOR_IMAGE}" "deploy/Dockerfile.worker ${WORKER_IMAGE}"; do 108 # shellcheck disable=SC2086 109 set -- ${spec} 110 dockerfile=$1 111 image=$2 112 printf '\n-- %s\n' "${image}" 113 docker build -f "${ROOT}/${dockerfile}" -t "${image}:dry-run" "${ROOT}" >/dev/null 114 printf 'would push:\n' 115 for tag in ${TAGS}; do printf ' %s:%s\n' "${image}" "${tag}"; done 116 done 117 printf '\nPASSED (dry run)\n' 118 exit 0 119 fi 120 121 # In CI the credentials come from project variables. Run by hand on a 122 # machine where somebody has already signed in, there is nothing to 123 # supply, and demanding a token again would be busywork. 124 if [ -n "${REGISTRY_TOKEN:-}" ]; then 125 : "${REGISTRY_USERNAME:?REGISTRY_USERNAME is set without REGISTRY_TOKEN}" 126 SIGN_IN=1 127 elif docker system info 2>/dev/null | grep -q '^ *Username:'; then 128 SIGN_IN=0 129 log "using the existing docker login for $(docker system info 2>/dev/null | sed -n 's/^ *Username: *//p')" 130 else 131 fail "not signed in to a registry, and REGISTRY_TOKEN is not set. 132 Either export REGISTRY_USERNAME and REGISTRY_TOKEN, or run: 133 docker login -u <user>" 134 fi 135 136 log "registering emulators" 137 # arm64 and riscv64 are emulated unless the worker is that architecture. 138 # Without this, buildx cannot run the foreign binaries a build needs. 139 # 140 # This reports success even when it achieved nothing: if the host has no 141 # binfmt_misc mounted, the registrations go into the container's own mount 142 # namespace and vanish with it. The check further down is what actually 143 # decides whether emulation works, rather than this command's exit code. 144 docker run --privileged --rm tonistiigi/binfmt:qemu-v9.2.2 --install all >/dev/null 145 146 # Whether the builder can really produce every platform we are asking for. 147 builder_covers() { 148 available=$(docker buildx inspect "$1" 2>/dev/null | sed -n 's/^Platforms:[[:space:]]*//p') 149 for want in $(printf '%s' "${PLATFORMS}" | tr ',' ' '); do 150 # Commas around the list so linux/arm does not match linux/arm64. 151 printf ',%s,' "${available}" | tr -d ' ' | grep -q ",${want}," || return 1 152 done 153 return 0 154 } 155 156 # Platform coverage is necessary but not sufficient: with the emulators 157 # registered even the default builder claims every architecture, and then 158 # refuses the build because the docker driver cannot produce more than one 159 # platform at a time. The driver is the part that decides. 160 builder_usable() { 161 docker buildx inspect "$1" 2>/dev/null | grep -q '^Driver:[[:space:]]*docker-container' \ 162 && builder_covers "$1" 163 } 164 165 log "preparing the builder" 166 # A builder bootstrapped before the emulators were registered does not pick 167 # them up afterwards, so one that cannot do the job is replaced rather than 168 # reused. 169 if docker buildx inspect conductor-publish >/dev/null 2>&1 && ! builder_usable conductor-publish; then 170 log "the existing builder cannot do this, replacing it" 171 docker buildx rm conductor-publish >/dev/null 2>&1 || true 172 fi 173 174 docker buildx inspect conductor-publish >/dev/null 2>&1 \ 175 || docker buildx create --name conductor-publish --driver docker-container --bootstrap >/dev/null 176 docker buildx use conductor-publish 177 178 builder_usable conductor-publish || fail "the builder cannot produce ${PLATFORMS}. 179 It reports: $(docker buildx inspect conductor-publish 2>/dev/null | sed -n 's/^Platforms:[[:space:]]*//p') 180 Emulation is registered through binfmt_misc, which has to be mounted on 181 the host running the docker daemon: 182 sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 183 Without it the installer above succeeds and registers nothing, and the 184 build fails later with 'exec format error'." 185 186 if [ "${SIGN_IN}" = 1 ]; then 187 log "signing in" 188 printf '%s' "${REGISTRY_TOKEN}" | docker login --username "${REGISTRY_USERNAME}" --password-stdin >/dev/null 189 190 # Whatever happens next, do not leave the credentials behind on a 191 # worker that is shared with other people's jobs. Only when this 192 # script created the session: logging out of one somebody else 193 # established would be rude, and on a workstation, baffling. 194 cleanup() { docker logout >/dev/null 2>&1 || true; } 195 trap cleanup EXIT 196 fi 197 198 log "building and pushing ${CONDUCTOR_IMAGE}" 199 # shellcheck disable=SC2046 200 docker buildx build \ 201 --platform "${PLATFORMS}" \ 202 --file "${ROOT}/deploy/Dockerfile" \ 203 $(tag_args "${CONDUCTOR_IMAGE}" | tr '\n' ' ') \ 204 --push \ 205 "${ROOT}" 206 207 log "building and pushing ${WORKER_IMAGE}" 208 # shellcheck disable=SC2046 209 docker buildx build \ 210 --platform "${PLATFORMS}" \ 211 --file "${ROOT}/deploy/Dockerfile.worker" \ 212 $(tag_args "${WORKER_IMAGE}" | tr '\n' ' ') \ 213 --push \ 214 "${ROOT}" 215 216 # Pushing an image does not touch the repository page, so the readme is 217 # sent separately or Docker Hub shows nothing at all. Kept next to the 218 # Dockerfiles so it is reviewed with them rather than edited in a web 219 # form and forgotten. 220 # 221 # Needs an API token rather than the registry session, so this is skipped 222 # when the script is run against an existing docker login. The release 223 # itself is done by then, and a stale readme is not worth failing it for. 224 sync_readme() { 225 image=$1 226 file=$2 227 summary=$3 228 repo=${image#*/} 229 namespace=${image%%/*} 230 231 [ -f "${file}" ] || { log "no readme at ${file}, skipping"; return 0; } 232 233 if [ -z "${REGISTRY_TOKEN:-}" ]; then 234 log "no REGISTRY_TOKEN, leaving the ${image} readme alone" 235 return 0 236 fi 237 238 if ! command -v curl >/dev/null 2>&1; then 239 log "curl is missing, leaving the ${image} readme alone" 240 return 0 241 fi 242 243 api=$(curl -fsS -X POST https://hub.docker.com/v2/auth/token \ 244 -H 'Content-Type: application/json' \ 245 -d "{\"identifier\":\"${REGISTRY_USERNAME}\",\"secret\":\"${REGISTRY_TOKEN}\"}" \ 246 | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p') 247 248 [ -n "${api}" ] || { log "could not get a hub api token, leaving the readme alone"; return 0; } 249 250 # The body is json, so the markdown has to be escaped rather than 251 # pasted: it is full of quotes, backslashes and newlines. The summary 252 # comes through the environment so awk does the escaping, not the shell. 253 payload=$(SUMMARY="${summary}" awk ' 254 function esc(s) { 255 gsub(/\\/, "\\\\", s) 256 gsub(/"/, "\\\"", s) 257 gsub(/\t/, "\\t", s) 258 return s 259 } 260 BEGIN { 261 printf "{" 262 # Sending an empty one would clear what is already there. 263 summary = esc(ENVIRON["SUMMARY"]) 264 if (summary != "") printf "\"description\":\"%s\",", summary 265 printf "\"full_description\":\"" 266 } 267 { printf "%s\\n", esc($0) } 268 END { printf "\"}" } 269 ' "${file}") 270 271 if printf '%s' "${payload}" | curl -fsS -X PATCH \ 272 "https://hub.docker.com/v2/repositories/${namespace}/${repo}/" \ 273 -H 'Content-Type: application/json' \ 274 -H "Authorization: Bearer ${api}" \ 275 --data-binary @- >/dev/null 276 then 277 log "updated the ${image} readme" 278 else 279 log "could not update the ${image} readme" 280 fi 281 } 282 283 log "updating the repository pages" 284 285 # The short description under the image name. Taken from package.json so it 286 # is not maintained in a web form as well as here. Left alone if it cannot 287 # be read, rather than failing a release that has already been pushed. 288 PKG_SUMMARY=$(sed -n 's/.*"description"[[:space:]]*:[[:space:]]*"\(.*\)".*/\1/p' "${ROOT}/package.json" | head -n 1) 289 [ -n "${PKG_SUMMARY}" ] || log "no description in package.json, leaving the short descriptions alone" 290 291 sync_readme "${CONDUCTOR_IMAGE}" "${ROOT}/deploy/hub/conductor.md" "${PKG_SUMMARY}" 292 sync_readme "${WORKER_IMAGE}" "${ROOT}/deploy/hub/conductor-worker.md" "Worker for conductor: pulls tasks and runs each in its own container" 293 294 log "verifying the manifests" 295 for image in "${CONDUCTOR_IMAGE}" "${WORKER_IMAGE}"; do 296 first=$(printf '%s' "${TAGS}" | head -1) 297 docker buildx imagetools inspect "${image}:${first}" \ 298 | grep -i platform \ 299 || fail "no platforms reported for ${image}:${first}" 300 done 301 302 printf '\nPASSED\n'