build.sh (17993B)
1 #!/bin/sh 2 # mk/build.sh - UNOS package driver: template -> signed .apk (v2) 3 # 4 # Usage: ./mk/build.sh [--channel <name>] <pkgname> 5 # 6 # Pipeline: source template -> fetch+verify -> extract -> patches -> 7 # do_build/do_install into DESTDIR -> emit signed .apk into 8 # build/repo/<channel>/<arch>/ 9 # 10 # The whole of build/ is generated output (gitignored). Only mk/ is source. 11 set -eu 12 13 HERE=$(cd "$(dirname "$0")" && pwd) 14 ROOT=$(cd "${HERE}/.." && pwd) 15 16 # shellcheck disable=SC1091 17 . "${HERE}/channels.inc" 18 19 CHANNEL="${UNOS_CHANNEL:-lts}" 20 case "${1:-}" in 21 --channel) CHANNEL="${2:?--channel needs a value}"; shift 2 ;; 22 --channel=*) CHANNEL="${1#--channel=}"; shift ;; 23 esac 24 export UNOS_CHANNEL="${CHANNEL}" 25 26 PKG="${1:?usage: build.sh [--channel <c>] <pkgname>}" 27 PKGDIR=$(chan_pkgdir "${CHANNEL}") 28 29 TEMPLATE="${PKGDIR}/${PKG}/template" 30 [ -f "${TEMPLATE}" ] || { echo "build.sh: no template: ${TEMPLATE}" >&2; exit 1; } 31 32 # --- template variables (defaults; the template overrides) --- 33 pkgname= 34 version= 35 revision=0 36 short_desc= 37 maintainer= 38 license= 39 homepage= 40 distfiles= 41 checksum= 42 # srcfiles: first-party source fetched by immutable ref (tag) from our own 43 # forge. Deliberately NOT checksummed -- the tag is the pin and TLS is the 44 # transport. distfiles= stays reserved for third-party upstream releases, 45 # which must always carry a sha256, because there we are trusting a server 46 # nobody here controls. Keeping the two apart means "no checksum" can never 47 # silently apply to an upstream tarball. 48 srcfiles= 49 depends= 50 # makedepends: other UNOS packages whose headers/libs must be in build/sysroot 51 # before this one compiles. Determines build order (see mk/deps.sh). 52 # hostmakedepends: commands that must exist on the build host. Checked, never 53 # built by us. 54 # Both are declared in packages/README.md and were, until now, read by nothing. 55 makedepends= 56 hostmakedepends= 57 provides= 58 replaces= 59 subpackages= 60 61 FILESDIR="${PKGDIR}/${PKG}/files" 62 WORK="${ROOT}/build/work/${CHANNEL}/${PKG}" 63 SRCDEST="${ROOT}/build/work/sources" 64 DESTDIR="${WORK}/dest" 65 WRKSRC= 66 OUTDIR="${ROOT}/build/repo" 67 # shared build logic for templates; set before the template is sourced so 68 # templates can `. "${UNOS_MKDIR}/kernel.inc"` at the top level 69 UNOS_MKDIR="${HERE}" 70 export UNOS_MKDIR 71 # Build-time sysroot (generated, never committed): libraries that other 72 # packages build against install twice - once into DESTDIR (the shipped 73 # .apk) and once into SYSROOT (headers + .so + prefix-fixed .pc files). 74 # Consumers find it via PKG_CONFIG_PATH; host tools (muon) via PATH. 75 SYSROOT= 76 77 # --- helpers available to templates --- 78 msg() { printf '==> %s\n' "$*"; } 79 die() { printf 'build.sh: error: %s\n' "$*" >&2; exit 1; } 80 81 # vinstall <file> <mode> <targetdir> [name] 82 vinstall() { 83 [ $# -ge 3 ] || die "vinstall needs: file mode targetdir [name]" 84 _src=$1; _mode=$2; _dir=$3; _name=${4:-$(basename "$1")} 85 install -D -m "${_mode}" "${_src}" "${DESTDIR}/${_dir}/${_name}" 86 } 87 vmkdir() { install -d "${DESTDIR}/$1"; } 88 89 # shellcheck disable=SC1090 90 . "${TEMPLATE}" 91 92 [ -n "${pkgname}" ] || die "template sets no pkgname" 93 [ -n "${version}" ] || die "template sets no version" 94 [ -n "${short_desc}" ] || die "template sets no short_desc" 95 96 PKGVER="${version}-r${revision}" 97 # ARCH: target arch, default x86_64. Conductor dispatches per-arch; local 98 # `ARCH=aarch64 ./mk/build.sh <pkg>` or `./mk/build.sh <pkg> aarch64` both work. 99 # Normalise arm64 -> aarch64. 100 if [ $# -ge 2 ]; then 101 case "$2" in x86_64|aarch64|arm64) ARCH="$2";; *) ARCH="${ARCH:-x86_64}";; esac 102 else 103 ARCH="${ARCH:-x86_64}" 104 fi 105 ARCH=$(chan_arch "${ARCH}") 106 case "${ARCH}" in x86_64|aarch64) ;; *) die "unsupported ARCH: ${ARCH} (want x86_64 or aarch64)" ;; esac 107 OUTDIR=$(chan_repodir "${CHANNEL}" "${ARCH}") 108 SYSROOT=$(chan_sysroot "${CHANNEL}" "${ARCH}") 109 export ARCH 110 111 mkdir -p "${SRCDEST}" "${WORK}" "${OUTDIR}" "${SYSROOT}" 112 113 # Host-built tools (muon) must be on PATH before hostmakedepends is checked, 114 # otherwise a package declaring `muon` fails the check despite muon being 115 # present in build/host/bin. Exported again below with PKG_CONFIG_PATH; doing 116 # it here as well is harmless and keeps the check honest. 117 export PATH="${ROOT}/build/host/bin:${PATH}" 118 119 # --- build dependencies --------------------------------------------------- 120 # Checked before anything is fetched or compiled. A missing bison surfacing as 121 # a syntax error 200 lines into someone else's generated parser is the kind of 122 # failure that costs an afternoon; naming it up front costs nothing. 123 for _h in ${hostmakedepends}; do 124 command -v "${_h}" >/dev/null 2>&1 \ 125 || die "hostmakedepends: '${_h}' not found on PATH (required to build ${pkgname})" 126 done 127 128 for _m in ${makedepends}; do 129 "${HERE}/deps.sh" owner "${_m}" >/dev/null 2>&1 \ 130 || die "makedepends: '${_m}' is not a package or subpackage in channel '${CHANNEL}'" 131 # Presence of a built .apk is the signal that its sysroot seed has been 132 # installed; templates that other packages build against install twice 133 # (DESTDIR + SYSROOT), so one implies the other. 134 # When sysroot is seeded via mk/sysroot.sh (CI), the .apk may not be 135 # present locally but its headers are in SYSROOT; check there as fallback. 136 if ! ls "${OUTDIR}/${_m}"-*.apk >/dev/null 2>&1; then 137 # Fallback: if sysroot already has the dep's headers/.pc, allow it. 138 # This is the CI incremental path (sysroot seeded from published repo). 139 if [ -d "${SYSROOT}/usr/include" ] && [ -f "${SYSROOT}/usr/lib/pkgconfig/${_m}.pc" ]; then 140 msg "makedepends '${_m}': not built locally, but found in sysroot (CI seed)" 141 elif ls "${SYSROOT}/usr/include/${_m}" >/dev/null 2>&1; then 142 msg "makedepends '${_m}': not built locally, but found in sysroot" 143 elif command -v apk >/dev/null 2>&1 \ 144 && apk --root "${SYSROOT}" info -e "${_m}" 2>/dev/null | grep -qx "${_m}"; then 145 # No .pc and no include subdir of its own (glibc); ask the sysroot db. 146 msg "makedepends '${_m}': installed in sysroot apk db" 147 else 148 die "makedepends '${_m}' has not been built. 149 Build it first: ./mk/build.sh ${_m} 150 Or build in order: ./mk/build-all.sh ${pkgname} 151 Or seed sysroot: ./mk/sysroot.sh ${pkgname}" 152 fi 153 fi 154 done 155 156 # --- revision-bump guard --------------------------------------------------- 157 # Enforce "revision bump required" policy: if this exact version-rrev already 158 # exists in the repo, refuse unless UNOS_ALLOW_OVERWRITE=1 (local iteration). 159 if [ "${UNOS_ALLOW_OVERWRITE:-0}" != "1" ]; then 160 if [ -f "${OUTDIR}/${pkgname}-${PKGVER}.apk" ]; then 161 die "${pkgname}-${PKGVER}.apk already exists in ${OUTDIR}/ 162 bump revision in ${PKG}'s template before rebuilding 163 (or UNOS_ALLOW_OVERWRITE=1 ./mk/build.sh ${PKG} to overwrite locally)" 164 fi 165 fi 166 167 # fresh staging every build: templates must be idempotent AND ghost-free 168 rm -rf "${DESTDIR}" 169 mkdir -p "${DESTDIR}" 170 171 # --- fetch + verify --- 172 # A template uses one or the other, never both: whichever is set provides the 173 # primary tarball that WRKSRC is derived from, and having two sources would 174 # make that choice silent and arbitrary. 175 if [ -n "${distfiles}" ] && [ -n "${srcfiles}" ]; then 176 die "template sets both distfiles and srcfiles; use srcfiles for our own 177 tag-pinned source, distfiles for checksummed third-party releases" 178 fi 179 180 if [ -n "${distfiles}" ]; then 181 # checksums are space-separated, in the same order as distfiles 182 set -- ${checksum} 183 for url in ${distfiles}; do 184 want=${1:?template has distfiles but no checksum for ${url}} 185 shift 186 f=${SRCDEST}/$(basename "${url}") 187 if [ ! -f "${f}" ]; then 188 msg "fetching $(basename "${url}")" 189 curl -fL -o "${f}" "${url}" || wget -O "${f}" "${url}" 190 fi 191 echo "${want} ${f}" | sha256sum -c - || die "checksum mismatch: ${f}" 192 done 193 fi 194 195 if [ -n "${srcfiles}" ]; then 196 [ -z "${checksum}" ] || die "srcfiles is tag-pinned and must not carry a checksum 197 (a tag archive is regenerated by the forge and is not byte-stable)" 198 for url in ${srcfiles}; do 199 # Namespaced by package on purpose. These URLs end in the tag, so the 200 # basename is typically just 'v0.1.0.tar.gz' -- two first-party packages 201 # at the same version would otherwise silently share one cache entry and 202 # build each other's source. 203 f=${SRCDEST}/${PKG}-$(basename "${url}") 204 if [ ! -f "${f}" ]; then 205 msg "fetching $(basename "${url}") for ${PKG} (first-party, tag-pinned)" 206 curl -fL -o "${f}" "${url}" || wget -O "${f}" "${url}" 207 fi 208 done 209 fi 210 211 # --- extract (first tarball) + patches --- 212 if [ -n "${distfiles}" ] || [ -n "${srcfiles}" ]; then 213 if [ -n "${srcfiles}" ]; then 214 set -- ${srcfiles} 215 _tarball=${SRCDEST}/${PKG}-$(basename "$1") 216 else 217 set -- ${distfiles} 218 _tarball=${SRCDEST}/$(basename "$1") 219 fi 220 # Wipe first: a leftover tree from another tarball prefix makes the 221 # single-top-level-dir test below fail and WRKSRC silently wrong. 222 rm -rf "${WORK}/src" 223 mkdir -p "${WORK}/src" 224 tar -xf "${_tarball}" -C "${WORK}/src" 225 # WRKSRC: single top-level dir if the tarball has exactly one, else src/ 226 n=$(ls -A "${WORK}/src" | wc -l) 227 if [ "${n}" = "1" ] && [ -d "${WORK}/src/$(ls -A "${WORK}/src")" ]; then 228 WRKSRC=${WORK}/src/$(ls -A "${WORK}/src") 229 else 230 WRKSRC=${WORK}/src 231 fi 232 for p in "${PKGDIR}/${PKG}"/patches/*.patch; do 233 [ -e "${p}" ] || break 234 msg "applying $(basename "${p}")" 235 patch -d "${WRKSRC}" -p1 --no-backup-if-mismatch -i "${p}" 236 done 237 fi 238 export DESTDIR WRKSRC FILESDIR WORK SYSROOT UNOS_MKDIR 239 # host-built tools first (muon), target sysroot visible to pkg-config. 240 # Both are additive only; system cc/patch/curl/pkg-config resolve as before. 241 export PATH="${ROOT}/build/host/bin:${PATH}" 242 export PKG_CONFIG_PATH="${SYSROOT}/usr/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" 243 244 # A sysroot library links against sysroot glibc. Resolving it against the build 245 # host's libc instead fails on every symbol the host predates (__isoc23_* and 246 # friends) -- and fails as "library not detected", not as a version error. 247 # 248 # Gated on makedepends because that is exactly what sysroot.sh seeded: with no 249 # makedepends the sysroot may be empty, and --sysroot would then hide the host 250 # headers such a package legitimately builds against. 251 if [ -n "${makedepends}" ]; then 252 export CPPFLAGS="--sysroot=${SYSROOT} ${CPPFLAGS:-}" 253 export CFLAGS="--sysroot=${SYSROOT} ${CFLAGS:-}" 254 export CXXFLAGS="--sysroot=${SYSROOT} ${CXXFLAGS:-}" 255 export LDFLAGS="--sysroot=${SYSROOT} -L${SYSROOT}/usr/lib -Wl,-rpath-link,${SYSROOT}/usr/lib ${LDFLAGS:-}" 256 fi 257 258 # --- build + install phases (defaults: no-op) --- 259 command -v do_configure >/dev/null 2>&1 || do_configure() { :; } 260 command -v do_build >/dev/null 2>&1 || do_build() { :; } 261 command -v do_install >/dev/null 2>&1 || do_install() { :; } 262 msg "${pkgname}: do_configure" 263 do_configure 264 msg "${pkgname}: do_build" 265 do_build 266 msg "${pkgname}: do_install" 267 do_install 268 269 # --- sysroot .pc fixup (central convention; templates stop hand-rolling it) 270 # Delegates to mk/pc-fixup.sh so the same logic is reused by mk/sysroot.sh. 271 "${HERE}/pc-fixup.sh" "${SYSROOT}" 272 273 # Verify it took, for every .pc rather than the two packages that used to 274 # assert it themselves. Those assertions lived in do_install, which runs 275 # *before* this fixup, so they could only ever pass on a sysroot left dirty by 276 # an earlier build -- they passed locally for months and failed on the first 277 # clean tree. A consumer that picks up an unfixed .pc silently compiles 278 # against host headers, which is the failure this guards. 279 for _pc in "${SYSROOT}"/usr/lib/pkgconfig/*.pc "${SYSROOT}"/usr/lib64/pkgconfig/*.pc; do 280 [ -e "${_pc}" ] || continue 281 grep -q "^prefix=${SYSROOT}/usr\$" "${_pc}" \ 282 || die "sysroot .pc not fixed: ${_pc} (mk/pc-fixup.sh did not rewrite prefix=)" 283 done 284 285 # --- signing key: UNOS_SIGN_KEY > .sign-key > single key in ~/.unos-keys --- 286 # shellcheck disable=SC1090 287 . "${HERE}/sign-key.inc" 288 KEY=$(resolve_sign_key) || die "no signing key (mk/keymgmt.sh use)" 289 290 # --- emit signed .apk (v2): shell + pax-tar + openssl, no other languages --- 291 # fixed timestamp for the whole artifact (reproducible when pinned) 292 : "${SOURCE_DATE_EPOCH:=$(date +%s)}" 293 export SOURCE_DATE_EPOCH 294 295 # helper binary (C source is committed; the binary is output) 296 if [ ! -x "${HERE}/pax-tar" ] || [ "${HERE}/pax-tar.c" -nt "${HERE}/pax-tar" ]; then 297 msg "building pax-tar" 298 cc -std=c99 -O2 -Wall -Wextra -o "${HERE}/pax-tar" "${HERE}/pax-tar.c" 299 fi 300 301 # spick <relpath>... : move staged paths from DESTDIR into the current 302 # SUBDEST (split staging). Only valid inside a sub_<name>() function. 303 spick() { 304 [ -n "${SUBDEST:-}" ] || die "spick outside a subpackage split" 305 [ $# -ge 1 ] || die "spick needs paths" 306 for _p in "$@"; do 307 [ -e "${DESTDIR}/${_p}" ] || die "spick: no such staged path: ${_p}" 308 mkdir -p "${SUBDEST}/$(dirname "${_p}")" 309 mv "${DESTDIR}/${_p}" "${SUBDEST}/${_p}" 310 done 311 # moves leave empty parent dirs behind; prune them so the remainder 312 # assert below sees real leftovers only (never DESTDIR itself) 313 find "${DESTDIR}" -mindepth 1 -depth -type d -empty -delete 314 } 315 316 # emit_apk <stage> <pkgname> <desc> <depends> <provides> <replaces> <suffix> 317 # suffix "" = primary (scripts from FILESDIR/<s>); otherwise per-subpackage 318 # scripts from FILESDIR/<s>.<suffix>. 319 emit_apk() { 320 _stage=$1; _name=$2; _desc=$3; _depends=$4; _provides=$5; _replaces=$6; _sfx=$7 321 _ctrl="${WORK}/ctrl.${_name}" 322 rm -rf "${_ctrl}" 323 mkdir -p "${_ctrl}" 324 325 { 326 printf 'pkgname = %s\n' "${_name}" 327 printf 'pkgver = %s\n' "${PKGVER}" 328 printf 'pkgdesc = %s\n' "${_desc}" 329 printf 'url = %s\n' "${homepage}" 330 printf 'builddate = %s\n' "${SOURCE_DATE_EPOCH}" 331 printf 'packager = %s\n' "UNOS build driver (mk/build.sh)" 332 printf 'size = %s\n' "$(du -sb "${_stage}" | cut -f1)" 333 printf 'arch = %s\n' "${ARCH}" 334 printf 'origin = %s\n' "${pkgname}" 335 printf 'maintainer = %s\n' "${maintainer}" 336 printf 'license = %s\n' "${license}" 337 for d in ${_depends}; do printf 'depend = %s\n' "${d}"; done 338 for p in ${_provides}; do printf 'provides = %s\n' "${p}"; done 339 for r in ${_replaces}; do printf 'replaces = %s\n' "${r}"; done 340 } > "${_ctrl}/.PKGINFO" 341 342 for s in pre-install post-install pre-deinstall post-deinstall pre-upgrade post-upgrade trigger; do 343 if [ -z "${_sfx}" ]; then _f="${FILESDIR}/${s}"; else _f="${FILESDIR}/${s}.${_sfx}"; fi 344 if [ -f "${_f}" ]; then 345 cp "${_f}" "${_ctrl}/.${s}" 346 chmod 755 "${_ctrl}/.${s}" 347 fi 348 done 349 350 # data segment (pax-tar: sorted, root-owned, per-file SHA1 headers). 351 # The data tar KEEPS its end-of-tar markers: it terminates the archive. 352 "${HERE}/pax-tar" "${_stage}" "${WORK}/data.${_name}.tar" 353 gzip -n -9 -f "${WORK}/data.${_name}.tar" 354 datahash=$(sha256sum "${WORK}/data.${_name}.tar.gz" | cut -d' ' -f1) 355 printf 'datahash = %s\n' "${datahash}" >> "${_ctrl}/.PKGINFO" 356 357 # control segment (pax-tar without checksum records; host tar not trusted). 358 # The control tar must NOT carry end-of-tar markers: gunzipped, the package 359 # is one continuous tar stream (sig record, control files, data files) and 360 # only the data segment terminates it. This is the abuild-sign layout. 361 "${HERE}/pax-tar" --no-checksum "${_ctrl}" "${WORK}/control.${_name}.tar" 362 ctlsiz=$(stat -c%s "${WORK}/control.${_name}.tar") 363 head -c $((ctlsiz - 1024)) "${WORK}/control.${_name}.tar" > "${WORK}/controlnotr.${_name}.tar" 364 gzip -n -9 -f "${WORK}/controlnotr.${_name}.tar" 365 mv "${WORK}/controlnotr.${_name}.tar.gz" "${WORK}/control.${_name}.tar.gz" 366 367 # signature segment: RSA/SHA1 over the control gzip stream bytes, single tar 368 # record with NO end-of-tar blocks, prepended to control.tar.gz in place 369 # (abuild-sign layout: .apk = signed-control + data = 3 gzip members). 370 KEYNAME=$(basename "${KEY}" .rsa) 371 openssl dgst -sha1 -sign "${KEY}" -out "${WORK}/sig.${_name}.bin" "${WORK}/control.${_name}.tar.gz" 372 mkdir -p "${WORK}/sigdir.${_name}" 373 cp "${WORK}/sig.${_name}.bin" "${WORK}/sigdir.${_name}/.SIGN.RSA.${KEYNAME}.rsa.pub" 374 "${HERE}/pax-tar" --no-checksum "${WORK}/sigdir.${_name}" "${WORK}/sig.${_name}.tar" 375 sigsiz=$(stat -c%s "${WORK}/sig.${_name}.tar") 376 head -c $((sigsiz - 1024)) "${WORK}/sig.${_name}.tar" > "${WORK}/signo.${_name}.tar" 377 gzip -n -9 -f "${WORK}/signo.${_name}.tar" 378 cat "${WORK}/signo.${_name}.tar.gz" "${WORK}/control.${_name}.tar.gz" > "${WORK}/signed-control.${_name}.tar.gz" 379 380 cat "${WORK}/signed-control.${_name}.tar.gz" "${WORK}/data.${_name}.tar.gz" \ 381 > "${OUTDIR}/${_name}-${PKGVER}.apk" 382 383 msg "done: ${OUTDIR}/${_name}-${PKGVER}.apk" 384 } 385 386 # --- subpackages: one template, several .apks --- 387 # Template declares: subpackages="name ..." (hyphens fine), per-sub 388 # short_desc_<underscored>/depends_<underscored> (provides/replaces optional), 389 # and sub_<underscored>() splitting DESTDIR into SUBDEST via spick (move 390 # semantics: the tree is partitioned, never duplicated). Whatever no subpackage 391 # claimed stays with the primary; it is logged rather than asserted empty, 392 # because a typo'd spick path is visible in that line. 393 if [ -n "${subpackages}" ]; then 394 for _sub in ${subpackages}; do 395 _u=$(printf '%s' "${_sub}" | tr '-' '_') 396 command -v "sub_${_u}" >/dev/null 2>&1 || die "no sub_${_u}() for subpackage ${_sub}" 397 eval "_sd=\${short_desc_${_u}:-}" 398 [ -n "${_sd}" ] || die "template sets no short_desc_${_u}" 399 eval "_dd=\${depends_${_u}:-}" 400 eval "_pv=\${provides_${_u}:-}" 401 eval "_rp=\${replaces_${_u}:-}" 402 SUBDEST="${WORK}/split/${_sub}" 403 rm -rf "${SUBDEST}" 404 mkdir -p "${SUBDEST}" 405 export SUBDEST 406 msg "${pkgname}: split ${_sub}" 407 "sub_${_u}" 408 unset SUBDEST 409 msg "${pkgname}: packing ${_sub} ${PKGVER}" 410 emit_apk "${WORK}/split/${_sub}" "${_sub}" "${_sd}" "${_dd}" "${_pv}" "${_rp}" "${_sub}" 411 done 412 if [ -n "$(ls -A "${DESTDIR}")" ]; then 413 msg "${pkgname}: primary keeps $(ls -A "${DESTDIR}" | tr '\n' ' ')" 414 fi 415 fi 416 417 msg "${pkgname}: packing ${PKGVER}" 418 emit_apk "${DESTDIR}" "${pkgname}" "${short_desc}" "${depends}" "${provides}" "${replaces}" ""