sysroot.sh (9693B)
1 #!/bin/sh 2 # mk/sysroot.sh - seed build/sysroot from the published apk repo. 3 # 4 # ./mk/sysroot.sh seed for all packages (full graph) 5 # ./mk/sysroot.sh zlib seed only transitive makedepends of zlib 6 # ./mk/sysroot.sh --arch aarch64 zlib specific arch (default x86_64) 7 # ./mk/sysroot.sh --from repo+staging use repo + staging repos (for CI level-1 overlay) 8 # 9 # Why this exists: mk/build.sh:100 treats "a built .apk exists" as proof the 10 # sysroot was seeded. An ephemeral CI container building only zlib therefore 11 # had to rebuild all 38 packages. This script populates the sysroot by 12 # `apk --root build/sysroot --initdb add` from the published repo, resolving 13 # transitive makedepends via mk/deps.sh. It is the gate on per-package 14 # incremental builds. 15 # 16 # Requires: apk (host via build/host/bin/apk) and a repo to pull from. 17 # Repo URLs may be local dirs (build/repo/x86_64) or http(s) (Garage via Caddy). 18 # Supports overlay: staging/<run> layered over repo/x86_64 so a level-1 build 19 # compiles against the freshly built library from the same push. 20 set -eu 21 22 HERE=$(cd "$(dirname "$0")" && pwd) 23 ROOT=$(cd "${HERE}/.." && pwd) 24 25 # shellcheck disable=SC1091 26 . "${HERE}/channels.inc" 27 28 ARCH="x86_64" 29 REPOS="" 30 PKGS="" 31 CHANNEL="${UNOS_CHANNEL:-lts}" 32 # default repos: local build/repo/<arch> if it exists, else nothing (caller 33 # must supply --repo). CI supplies --repo https://repo.unos.finwo.net/x86_64 34 # and optionally --repo <staging-url>. 35 while [ $# -gt 0 ]; do 36 case "$1" in 37 --arch) ARCH="$2"; shift 2 ;; 38 --arch=*) ARCH="${1#--arch=}"; shift ;; 39 --repo) REPOS="${REPOS} $2"; shift 2 ;; 40 --repo=*) REPOS="${REPOS} ${1#--repo=}"; shift ;; 41 --channel) CHANNEL="$2"; shift 2 ;; 42 --channel=*) CHANNEL="${1#--channel=}"; shift ;; 43 -h|--help) 44 sed -n '1,45p' "$0" 45 exit 0 46 ;; 47 --) shift; break ;; 48 -*) echo "sysroot.sh: unknown option: $1" >&2; exit 1 ;; 49 *) PKGS="${PKGS} $1"; shift ;; 50 esac 51 done 52 # remaining args are also pkgs 53 for a in "$@"; do PKGS="${PKGS} $a"; done 54 55 ARCH=$(chan_arch "${ARCH}") 56 case "${ARCH}" in x86_64|aarch64) ;; *) echo "sysroot.sh: unsupported ARCH ${ARCH}" >&2; exit 1 ;; esac 57 58 export UNOS_CHANNEL="${CHANNEL}" 59 SYSROOT=$(chan_sysroot "${CHANNEL}" "${ARCH}") 60 # apk --initdb creates the root itself but not its parents. 61 mkdir -p "${SYSROOT}" 62 63 APK="${ROOT}/build/host/bin/apk" 64 if [ ! -x "${APK}" ]; then 65 APK="apk" 66 command -v apk >/dev/null 2>&1 || { 67 echo "sysroot.sh: apk not found (run mk/bootstrap-host.sh)" >&2 68 exit 1 69 } 70 fi 71 72 # Default repo: local build/repo/<channel>/<arch> if it exists 73 if [ -z "${REPOS}" ]; then 74 LOCALREPO=$(chan_repodir "${CHANNEL}" "${ARCH}") 75 if [ -d "${LOCALREPO}" ] && ls "${LOCALREPO}"/*.apk >/dev/null 2>&1; then 76 REPOS="${LOCALREPO}" 77 else 78 echo "sysroot.sh: no --repo given and no local ${LOCALREPO} with .apk files" >&2 79 echo " hint: --repo https://repo.unos.finwo.net or --repo /path/to/repo" >&2 80 exit 1 81 fi 82 fi 83 84 # Resolve transitive makedepends. 85 # deps.sh order <pkgs> already gives the topological closure including the 86 # requested packages themselves; filter to just the dependencies. 87 if [ -z "${PKGS}" ] || [ "${PKGS}" = " " ]; then 88 # all packages: seed everything (used for full sysroot) 89 ORDER=$("${ROOT}/mk/deps.sh" seed) || exit 1 90 else 91 # shellcheck disable=SC2086 92 ORDER=$("${ROOT}/mk/deps.sh" seed ${PKGS}) || exit 1 93 fi 94 95 # Build set of needed packages: transitive makedepends of requested pkgs. 96 # If PKGS is empty, we seeded all, but for sysroot we want only libs that 97 # other packages build against -- still safe to install all. 98 # For incremental, we install the closure minus the targets themselves if 99 # targets are leaves? Simpler: install the whole order - apk will handle 100 # already-installed. Filter to keep it small: if PKGS given, use order 101 # but the conductor will call with the full level's closure anyway. 102 PKGS_TO_INSTALL="" 103 for p in ${ORDER}; do 104 # Skip if this is a leaf requested package that hasn't been built yet and 105 # would not be in the repo. But if it's already in the repo, installing it 106 # is fine (provides headers). So we try to install everything in order; 107 # apk will skip what's not in the repo with an error - filter via repo. 108 PKGS_TO_INSTALL="${PKGS_TO_INSTALL} $p" 109 done 110 111 # When PKGS is given, the ORDER includes deps first then targets. Installing 112 # all is correct for level-1 overlay: level-1 needs the level-0 libs that are 113 # already in the repo plus the level-0 staging apks. 114 # If the target itself is not yet in any repo, apk add will fail; so we 115 # attempt and allow missing targets. 116 # Strategy: try to install all, but if apk fails due to missing pkg, retry 117 # without the leaf targets. 118 119 # The published APKINDEX lags within a pipeline run: it only gains a package 120 # once the index task reruns. Fetch each template's exact pin directly and 121 # index it locally; whatever is missed still resolves through APKINDEX below. 122 STAGE_BASE="${ROOT}/build/sysroot-stage/${CHANNEL}" 123 STAGE="${STAGE_BASE}/${ARCH}" 124 STAGE_ARGS="" 125 for r in ${REPOS}; do 126 case "${r}" in http://*|https://*) ;; *) continue ;; esac 127 command -v curl >/dev/null 2>&1 || break 128 mkdir -p "${STAGE}" 129 staged=0 130 queue="${PKGS_TO_INSTALL}" 131 seen="" 132 while [ -n "${queue}" ]; do 133 # shellcheck disable=SC2086 134 set -- ${queue} 135 n="$1"; shift; queue="$*" 136 137 # depends may carry a pin: linux-headers=6.18.51-r1 138 base="${n%%=*}" 139 case "${n}" in 140 *=*) pv="${n#*=}" ;; 141 *) pv=$("${ROOT}/mk/deps.sh" pkgver "${base}" 2>/dev/null) || continue ;; 142 esac 143 144 case " ${seen} " in *" ${base} "*) continue ;; esac 145 seen="${seen} ${base}" 146 147 apk_name="${base}-${pv}.apk" 148 if [ ! -f "${STAGE}/${apk_name}" ]; then 149 url=$(chan_url "${r}" "${CHANNEL}" "${ARCH}" "${apk_name}") 150 curl -fsSL -o "${STAGE}/${apk_name}" "${url}" 2>/dev/null || { 151 rm -f "${STAGE}/${apk_name}" 152 continue 153 } 154 echo " staged ${apk_name}" 155 fi 156 staged=$((staged + 1)) 157 158 queue="${queue} $("${ROOT}/mk/deps.sh" rtdeps "${base}" 2>/dev/null || true)" 159 done 160 if [ "${staged}" -gt 0 ]; then 161 ( cd "${STAGE}" && "${APK}" index --allow-untrusted -o APKINDEX.tar.gz ./*.apk >/dev/null ) 162 STAGE_ARGS=" --repository file://${STAGE_BASE}" 163 fi 164 break 165 done 166 167 # apk appends /<arch> to every --repository, so remote bases stop at the 168 # channel and local paths stop one above the arch directory. 169 REPO_ARGS="${STAGE_ARGS}" 170 for r in ${REPOS}; do 171 case "${r}" in 172 http://*|https://*) REPO_ARGS="${REPO_ARGS} --repository $(chan_repo_url "${r}" "${CHANNEL}")" ;; 173 file://*) REPO_ARGS="${REPO_ARGS} --repository ${r}" ;; 174 /*) REPO_ARGS="${REPO_ARGS} --repository file://$(dirname "${r}")" ;; 175 *) REPO_ARGS="${REPO_ARGS} --repository file://$(dirname "${ROOT}/${r}")" ;; 176 esac 177 done 178 179 # Allow apk to fetch from multiple repos; --allow-untrusted for local unsigned 180 # staging during dev. Published repo is signed; apk will verify via keys in 181 # the sysroot if present, but --allow-untrusted is harmless for seeding. 182 # Use --initdb if sysroot is empty. --no-scripts avoids ldconfig etc in sysroot. 183 184 # Non-root builds (local dev) need --usermode; CI runs as root in Docker. 185 USERMODE="" 186 if [ "$(id -u)" != "0" ]; then 187 USERMODE="--usermode" 188 fi 189 190 SYSROOT_EXISTS=0 191 [ -f "${SYSROOT}/lib/apk/db/installed" ] && SYSROOT_EXISTS=1 192 [ -f "${SYSROOT}/etc/apk/world" ] && SYSROOT_EXISTS=1 193 # Fallback: legacy build/sysroot via DESTDIR has no apk db at all -- treat as 194 # not-exists so we init it. Content already there will be preserved by apk 195 # (it merges). 196 197 echo "==> sysroot: ARCH=${ARCH} SYSROOT=${SYSROOT}" 198 echo "==> sysroot: repos:${REPOS}" 199 echo "==> sysroot: installing:${PKGS_TO_INSTALL}" 200 201 # shellcheck disable=SC2086 202 if [ "${SYSROOT_EXISTS}" = "0" ]; then 203 # shellcheck disable=SC2086 204 "${APK}" ${USERMODE} --root "${SYSROOT}" --initdb ${REPO_ARGS} --allow-untrusted --no-scripts add ${PKGS_TO_INSTALL} || { 205 rc=$? 206 # Retry without the requested leaf packages (they may not be in repo yet) 207 if [ -n "${PKGS}" ]; then 208 echo "==> sysroot: retry without leaf targets" 209 FILTERED="" 210 for p in ${PKGS_TO_INSTALL}; do 211 skip=0 212 for t in ${PKGS}; do [ "${p}" = "${t}" ] && skip=1 && break; done 213 [ "${skip}" = "0" ] && FILTERED="${FILTERED} $p" 214 done 215 if [ -z "${FILTERED}" ]; then 216 echo "==> sysroot: nothing to install after filtering (target not in repo yet)" 217 mkdir -p "${SYSROOT}/etc/apk" "${SYSROOT}/usr/lib/pkgconfig" 218 else 219 # shellcheck disable=SC2086 220 "${APK}" ${USERMODE} --root "${SYSROOT}" --initdb ${REPO_ARGS} --allow-untrusted --no-scripts add ${FILTERED} || exit $rc 221 fi 222 else 223 exit $rc 224 fi 225 } 226 else 227 # shellcheck disable=SC2086 228 "${APK}" ${USERMODE} --root "${SYSROOT}" ${REPO_ARGS} --allow-untrusted --no-scripts add ${PKGS_TO_INSTALL} || { 229 rc=$? 230 if [ -n "${PKGS}" ]; then 231 echo "==> sysroot: retry without leaf targets" 232 FILTERED="" 233 for p in ${PKGS_TO_INSTALL}; do 234 skip=0 235 for t in ${PKGS}; do [ "${p}" = "${t}" ] && skip=1 && break; done 236 [ "${skip}" = "0" ] && FILTERED="${FILTERED} $p" 237 done 238 if [ -z "${FILTERED}" ]; then 239 echo "==> sysroot: nothing to install after filtering" 240 else 241 # shellcheck disable=SC2086 242 "${APK}" ${USERMODE} --root "${SYSROOT}" ${REPO_ARGS} --allow-untrusted --no-scripts add ${FILTERED} || exit $rc 243 fi 244 else 245 exit $rc 246 fi 247 } 248 fi 249 250 # Fix up .pc files (same logic as mk/pc-fixup.sh, but arch-aware) 251 "${HERE}/pc-fixup.sh" "${SYSROOT}" 252 253 echo "==> sysroot: done ${SYSROOT} ($(find "${SYSROOT}" -type f | wc -l | tr -d ' ') files)"