publish.sh (5583B)
1 #!/bin/sh 2 # mk/publish.sh - upload built packages and the index to the S3 repo. 3 # 4 # ./mk/publish.sh publish every .apk plus APKINDEX for $ARCH 5 # ./mk/publish.sh zlib popt publish only those packages (plus index) 6 # ./mk/publish.sh --index-only re-upload APKINDEX.tar.gz and nothing else 7 # ./mk/publish.sh --dry-run print what would be uploaded 8 # 9 # Environment (supplied by the CI runner; no mounts are provided): 10 # 11 # BUCKET_ACCESS_KEY S3 access key id 12 # BUCKET_SECRET_KEY S3 secret access key 13 # BUCKET_NAME bucket to write into 14 # BUCKET_ENDPOINT base URL of the S3 service, e.g. https://s3.finwo.net 15 # BUCKET_REGION region name, e.g. eu-west1 16 # 17 # Uses `curl --aws-sigv4` rather than awscli or rclone: curl is already needed 18 # to fetch sources, signs SigV4 natively since 7.75, and pulling in a Python or 19 # Go client for four PUTs is not worth the image size or the supply chain. 20 set -eu 21 22 HERE=$(cd "$(dirname "$0")" && pwd) 23 ROOT=$(cd "${HERE}/.." && pwd) 24 25 # shellcheck disable=SC1091 26 . "${HERE}/channels.inc" 27 28 ARCH="${ARCH:-x86_64}" 29 DRY=0 30 INDEX_ONLY=0 31 PKGS="" 32 CHANNEL="${UNOS_CHANNEL:-lts}" 33 34 while [ $# -gt 0 ]; do 35 case "$1" in 36 --arch) ARCH="$2"; shift 2 ;; 37 --arch=*) ARCH="${1#--arch=}"; shift ;; 38 --channel) CHANNEL="$2"; shift 2 ;; 39 --channel=*) CHANNEL="${1#--channel=}"; shift ;; 40 --dry-run) DRY=1; shift ;; 41 --index-only) INDEX_ONLY=1; shift ;; 42 -h|--help) sed -n '2,20p' "$0"; exit 0 ;; 43 -*) echo "publish.sh: unknown option: $1" >&2; exit 1 ;; 44 *) PKGS="${PKGS} $1"; shift ;; 45 esac 46 done 47 48 msg() { printf '==> %s\n' "$*"; } 49 die() { printf 'publish.sh: error: %s\n' "$*" >&2; exit 1; } 50 51 ARCH=$(chan_arch "${ARCH}") 52 export UNOS_CHANNEL="${CHANNEL}" 53 54 chan_exists "${CHANNEL}" || die "no such channel: ${CHANNEL}" 55 [ "$(chan_conf "${CHANNEL}" publish)" = "no" ] && die "channel '${CHANNEL}' is publish=no" 56 57 REPODIR=$(chan_repodir "${CHANNEL}" "${ARCH}") 58 [ -d "${REPODIR}" ] || die "no such repo dir: ${REPODIR}" 59 PREFIX=$(chan_prefix "${CHANNEL}" "${ARCH}") 60 61 if [ "${DRY}" != "1" ]; then 62 for v in BUCKET_ACCESS_KEY BUCKET_SECRET_KEY BUCKET_NAME BUCKET_ENDPOINT BUCKET_REGION; do 63 eval "val=\${${v}:-}" 64 [ -n "${val}" ] || die "${v} is not set" 65 done 66 67 # These two are easy to transpose, and transposing them produces a signature 68 # mismatch rather than anything that names the real problem. Check the shape 69 # instead of trusting the order they were exported in. 70 case "${BUCKET_ENDPOINT}" in 71 http://*|https://*) ;; 72 *) die "BUCKET_ENDPOINT must be a URL (e.g. https://s3.finwo.net), got '${BUCKET_ENDPOINT}'. 73 Note BUCKET_REGION is the region name (e.g. eu-west1); the two look swapped." ;; 74 esac 75 case "${BUCKET_REGION}" in 76 http://*|https://*) die "BUCKET_REGION must be a region name (e.g. eu-west1), not a URL. 77 Note BUCKET_ENDPOINT is the URL; the two look swapped." ;; 78 esac 79 80 command -v curl >/dev/null 2>&1 || die "curl is required" 81 fi 82 83 # put <localfile> <key> 84 put() { 85 src="$1" 86 key="$2" 87 # Built after the dry-run check on purpose: --dry-run must work with no 88 # credentials in the environment at all, and set -u would abort here. 89 if [ "${DRY}" = "1" ]; then 90 printf ' would PUT %s -> %s\n' "${src}" "${key}" 91 return 0 92 fi 93 # `+` must be percent-encoded or SigV4 signs it while S3 reads it as a space. 94 enc=$(printf '%s' "${key}" | sed 's/+/%2B/g') 95 url="${BUCKET_ENDPOINT%/}/${BUCKET_NAME}/${enc}" 96 sha=$(sha256sum "${src}" | cut -d' ' -f1) 97 code=$(curl -sS -o /dev/null -w '%{http_code}' \ 98 --aws-sigv4 "aws:amz:${BUCKET_REGION}:s3" \ 99 --user "${BUCKET_ACCESS_KEY}:${BUCKET_SECRET_KEY}" \ 100 -H "x-amz-content-sha256: ${sha}" \ 101 -X PUT --upload-file "${src}" \ 102 "${url}") || die "upload failed: ${key}" 103 case "${code}" in 104 2*) printf ' %s (%s)\n' "${key}" "${code}" ;; 105 *) die "upload of ${key} returned HTTP ${code}" ;; 106 esac 107 } 108 109 # Which .apk files to publish. 110 if [ "${INDEX_ONLY}" = "1" ]; then 111 FILES="" 112 elif [ -n "${PKGS}" ]; then 113 FILES="" 114 # One template can emit several .apks; publishing only the primary leaves 115 # its subpackages unreachable for anything that build-depends on them. 116 EXPANDED="" 117 for p in ${PKGS}; do 118 EXPANDED="${EXPANDED} ${p} $("${HERE}/deps.sh" subpkgs "${p}" 2>/dev/null || true)" 119 done 120 for p in ${EXPANDED}; do 121 found="" 122 for f in "${REPODIR}/${p}"-*.apk; do 123 [ -e "${f}" ] || continue 124 # `zlib-1.3.1-r0.apk` matches zlib, but `zlib-doc-...` must not, so 125 # require the character after the name to start a version field. 126 base=$(basename "${f}") 127 rest=${base#"${p}"-} 128 case "${rest}" in 129 [0-9]*) found="${found} ${f}" ;; 130 esac 131 done 132 [ -n "${found}" ] || die "no built .apk for '${p}' in ${REPODIR}" 133 FILES="${FILES}${found}" 134 done 135 else 136 FILES=$(ls "${REPODIR}"/*.apk 2>/dev/null || true) 137 [ -n "${FILES}" ] || die "no .apk files in ${REPODIR}" 138 fi 139 140 msg "publishing to ${BUCKET_NAME:-<dry-run>}/${PREFIX}" 141 for f in ${FILES}; do 142 put "${f}" "${PREFIX}/$(basename "${f}")" 143 done 144 145 # The index goes last, always. It is the file clients read to discover what 146 # exists, so publishing it before its packages would advertise packages that 147 # are not there yet; publishing it after is merely briefly stale. 148 IDX="${REPODIR}/APKINDEX.tar.gz" 149 if [ -f "${IDX}" ]; then 150 put "${IDX}" "${PREFIX}/APKINDEX.tar.gz" 151 else 152 [ "${INDEX_ONLY}" = "1" ] && die "no APKINDEX.tar.gz in ${REPODIR}; run ./mk/repo-index.sh" 153 msg "note: no APKINDEX.tar.gz to publish (run ./mk/repo-index.sh first)" 154 fi 155 156 msg "done"