build.sh (16503B)
1 #!/bin/sh 2 # mk/build.sh - UNOS package driver: template -> signed .apk (v2) 3 # 4 # Usage: ./mk/build.sh <pkgname> 5 # 6 # Pipeline: source template -> fetch+verify -> extract -> patches -> 7 # do_build/do_install into DESTDIR -> emit signed .apk into build/repo/ 8 # 9 # The whole of build/ is generated output (gitignored). Only mk/ is source. 10 set -eu 11 12 HERE=$(cd "$(dirname "$0")" && pwd) 13 ROOT=$(cd "${HERE}/.." && pwd) 14 PKG="${1:?usage: build.sh <pkgname>}" 15 16 TEMPLATE="${ROOT}/packages/${PKG}/template" 17 [ -f "${TEMPLATE}" ] || { echo "build.sh: no template: ${TEMPLATE}" >&2; exit 1; } 18 19 # --- template variables (defaults; the template overrides) --- 20 pkgname= 21 version= 22 revision=0 23 short_desc= 24 maintainer= 25 license= 26 homepage= 27 distfiles= 28 checksum= 29 # srcfiles: first-party source fetched by immutable ref (tag) from our own 30 # forge. Deliberately NOT checksummed -- the tag is the pin and TLS is the 31 # transport. distfiles= stays reserved for third-party upstream releases, 32 # which must always carry a sha256, because there we are trusting a server 33 # nobody here controls. Keeping the two apart means "no checksum" can never 34 # silently apply to an upstream tarball. 35 srcfiles= 36 depends= 37 # makedepends: other UNOS packages whose headers/libs must be in build/sysroot 38 # before this one compiles. Determines build order (see mk/deps.sh). 39 # hostmakedepends: commands that must exist on the build host. Checked, never 40 # built by us. 41 # Both are declared in packages/README.md and were, until now, read by nothing. 42 makedepends= 43 hostmakedepends= 44 provides= 45 replaces= 46 subpackages= 47 48 FILESDIR="${ROOT}/packages/${PKG}/files" 49 WORK="${ROOT}/build/work/${PKG}" 50 SRCDEST="${ROOT}/build/work/sources" 51 DESTDIR="${WORK}/dest" 52 WRKSRC= 53 OUTDIR="${ROOT}/build/repo" 54 # shared build logic for templates; set before the template is sourced so 55 # templates can `. "${UNOS_MKDIR}/kernel.inc"` at the top level 56 UNOS_MKDIR="${HERE}" 57 export UNOS_MKDIR 58 # Build-time sysroot (generated, never committed): libraries that other 59 # packages build against install twice - once into DESTDIR (the shipped 60 # .apk) and once into SYSROOT (headers + .so + prefix-fixed .pc files). 61 # Consumers find it via PKG_CONFIG_PATH; host tools (muon) via PATH. 62 SYSROOT="${ROOT}/build/sysroot" 63 64 # --- helpers available to templates --- 65 msg() { printf '==> %s\n' "$*"; } 66 die() { printf 'build.sh: error: %s\n' "$*" >&2; exit 1; } 67 68 # vinstall <file> <mode> <targetdir> [name] 69 vinstall() { 70 [ $# -ge 3 ] || die "vinstall needs: file mode targetdir [name]" 71 _src=$1; _mode=$2; _dir=$3; _name=${4:-$(basename "$1")} 72 install -D -m "${_mode}" "${_src}" "${DESTDIR}/${_dir}/${_name}" 73 } 74 vmkdir() { install -d "${DESTDIR}/$1"; } 75 76 # shellcheck disable=SC1090 77 . "${TEMPLATE}" 78 79 [ -n "${pkgname}" ] || die "template sets no pkgname" 80 [ -n "${version}" ] || die "template sets no version" 81 [ -n "${short_desc}" ] || die "template sets no short_desc" 82 83 PKGVER="${version}-r${revision}" 84 # ARCH: target arch, default x86_64. Conductor dispatches per-arch; local 85 # `ARCH=aarch64 ./mk/build.sh <pkg>` or `./mk/build.sh <pkg> aarch64` both work. 86 # Normalise arm64 -> aarch64. 87 if [ $# -ge 2 ]; then 88 case "$2" in x86_64|aarch64|arm64) ARCH="$2";; *) ARCH="${ARCH:-x86_64}";; esac 89 else 90 ARCH="${ARCH:-x86_64}" 91 fi 92 case "${ARCH}" in arm64) ARCH=aarch64 ;; esac 93 case "${ARCH}" in x86_64|aarch64) ;; *) die "unsupported ARCH: ${ARCH} (want x86_64 or aarch64)" ;; esac 94 OUTDIR="${ROOT}/build/repo/${ARCH}" 95 export ARCH 96 97 mkdir -p "${SRCDEST}" "${WORK}" "${OUTDIR}" "${SYSROOT}" 98 99 # Host-built tools (muon) must be on PATH before hostmakedepends is checked, 100 # otherwise a package declaring `muon` fails the check despite muon being 101 # present in build/host/bin. Exported again below with PKG_CONFIG_PATH; doing 102 # it here as well is harmless and keeps the check honest. 103 export PATH="${ROOT}/build/host/bin:${PATH}" 104 105 # --- build dependencies --------------------------------------------------- 106 # Checked before anything is fetched or compiled. A missing bison surfacing as 107 # a syntax error 200 lines into someone else's generated parser is the kind of 108 # failure that costs an afternoon; naming it up front costs nothing. 109 for _h in ${hostmakedepends}; do 110 command -v "${_h}" >/dev/null 2>&1 \ 111 || die "hostmakedepends: '${_h}' not found on PATH (required to build ${pkgname})" 112 done 113 114 for _m in ${makedepends}; do 115 [ -f "${ROOT}/packages/${_m}/template" ] \ 116 || die "makedepends: '${_m}' is not a package under packages/" 117 # Presence of a built .apk is the signal that its sysroot seed has been 118 # installed; templates that other packages build against install twice 119 # (DESTDIR + SYSROOT), so one implies the other. 120 # When sysroot is seeded via mk/sysroot.sh (CI), the .apk may not be 121 # present locally but its headers are in SYSROOT; check there as fallback. 122 if ! ls "${OUTDIR}/${_m}"-*.apk >/dev/null 2>&1; then 123 # Fallback: if sysroot already has the dep's headers/.pc, allow it. 124 # This is the CI incremental path (sysroot seeded from published repo). 125 if [ -d "${SYSROOT}/usr/include" ] && [ -f "${SYSROOT}/usr/lib/pkgconfig/${_m}.pc" ]; then 126 msg "makedepends '${_m}': not built locally, but found in sysroot (CI seed)" 127 elif ls "${SYSROOT}/usr/include/${_m}" >/dev/null 2>&1; then 128 msg "makedepends '${_m}': not built locally, but found in sysroot" 129 else 130 die "makedepends '${_m}' has not been built. 131 Build it first: ./mk/build.sh ${_m} 132 Or build in order: ./mk/build-all.sh ${pkgname} 133 Or seed sysroot: ./mk/sysroot.sh ${pkgname}" 134 fi 135 fi 136 done 137 138 # --- revision-bump guard --------------------------------------------------- 139 # Enforce "revision bump required" policy: if this exact version-rrev already 140 # exists in the repo, refuse unless UNOS_ALLOW_OVERWRITE=1 (local iteration). 141 if [ "${UNOS_ALLOW_OVERWRITE:-0}" != "1" ]; then 142 if [ -f "${OUTDIR}/${pkgname}-${PKGVER}.apk" ]; then 143 die "${pkgname}-${PKGVER}.apk already exists in ${OUTDIR}/ 144 bump revision in packages/${PKG}/template before rebuilding 145 (or UNOS_ALLOW_OVERWRITE=1 ./mk/build.sh ${PKG} to overwrite locally)" 146 fi 147 fi 148 149 # fresh staging every build: templates must be idempotent AND ghost-free 150 rm -rf "${DESTDIR}" 151 mkdir -p "${DESTDIR}" 152 153 # --- fetch + verify --- 154 # A template uses one or the other, never both: whichever is set provides the 155 # primary tarball that WRKSRC is derived from, and having two sources would 156 # make that choice silent and arbitrary. 157 if [ -n "${distfiles}" ] && [ -n "${srcfiles}" ]; then 158 die "template sets both distfiles and srcfiles; use srcfiles for our own 159 tag-pinned source, distfiles for checksummed third-party releases" 160 fi 161 162 if [ -n "${distfiles}" ]; then 163 # checksums are space-separated, in the same order as distfiles 164 set -- ${checksum} 165 for url in ${distfiles}; do 166 want=${1:?template has distfiles but no checksum for ${url}} 167 shift 168 f=${SRCDEST}/$(basename "${url}") 169 if [ ! -f "${f}" ]; then 170 msg "fetching $(basename "${url}")" 171 curl -fL -o "${f}" "${url}" || wget -O "${f}" "${url}" 172 fi 173 echo "${want} ${f}" | sha256sum -c - || die "checksum mismatch: ${f}" 174 done 175 fi 176 177 if [ -n "${srcfiles}" ]; then 178 [ -z "${checksum}" ] || die "srcfiles is tag-pinned and must not carry a checksum 179 (a tag archive is regenerated by the forge and is not byte-stable)" 180 for url in ${srcfiles}; do 181 # Namespaced by package on purpose. These URLs end in the tag, so the 182 # basename is typically just 'v0.1.0.tar.gz' -- two first-party packages 183 # at the same version would otherwise silently share one cache entry and 184 # build each other's source. 185 f=${SRCDEST}/${PKG}-$(basename "${url}") 186 if [ ! -f "${f}" ]; then 187 msg "fetching $(basename "${url}") for ${PKG} (first-party, tag-pinned)" 188 curl -fL -o "${f}" "${url}" || wget -O "${f}" "${url}" 189 fi 190 done 191 fi 192 193 # --- extract (first tarball) + patches --- 194 if [ -n "${distfiles}" ] || [ -n "${srcfiles}" ]; then 195 if [ -n "${srcfiles}" ]; then 196 set -- ${srcfiles} 197 _tarball=${SRCDEST}/${PKG}-$(basename "$1") 198 else 199 set -- ${distfiles} 200 _tarball=${SRCDEST}/$(basename "$1") 201 fi 202 mkdir -p "${WORK}/src" 203 tar -xf "${_tarball}" -C "${WORK}/src" 204 # WRKSRC: single top-level dir if the tarball has exactly one, else src/ 205 n=$(ls -A "${WORK}/src" | wc -l) 206 if [ "${n}" = "1" ] && [ -d "${WORK}/src/$(ls -A "${WORK}/src")" ]; then 207 WRKSRC=${WORK}/src/$(ls -A "${WORK}/src") 208 else 209 WRKSRC=${WORK}/src 210 fi 211 for p in "${ROOT}/packages/${PKG}"/patches/*.patch; do 212 [ -e "${p}" ] || break 213 msg "applying $(basename "${p}")" 214 patch -d "${WRKSRC}" -p1 --no-backup-if-mismatch -i "${p}" 215 done 216 fi 217 export DESTDIR WRKSRC FILESDIR WORK SYSROOT UNOS_MKDIR 218 # host-built tools first (muon), target sysroot visible to pkg-config. 219 # Both are additive only; system cc/patch/curl/pkg-config resolve as before. 220 export PATH="${ROOT}/build/host/bin:${PATH}" 221 export PKG_CONFIG_PATH="${SYSROOT}/usr/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" 222 223 # --- build + install phases (defaults: no-op) --- 224 command -v do_configure >/dev/null 2>&1 || do_configure() { :; } 225 command -v do_build >/dev/null 2>&1 || do_build() { :; } 226 command -v do_install >/dev/null 2>&1 || do_install() { :; } 227 msg "${pkgname}: do_configure" 228 do_configure 229 msg "${pkgname}: do_build" 230 do_build 231 msg "${pkgname}: do_install" 232 do_install 233 234 # --- sysroot .pc fixup (central convention; templates stop hand-rolling it) 235 # Delegates to mk/pc-fixup.sh so the same logic is reused by mk/sysroot.sh. 236 "${HERE}/pc-fixup.sh" "${SYSROOT}" 237 238 # Verify it took, for every .pc rather than the two packages that used to 239 # assert it themselves. Those assertions lived in do_install, which runs 240 # *before* this fixup, so they could only ever pass on a sysroot left dirty by 241 # an earlier build -- they passed locally for months and failed on the first 242 # clean tree. A consumer that picks up an unfixed .pc silently compiles 243 # against host headers, which is the failure this guards. 244 for _pc in "${SYSROOT}"/usr/lib/pkgconfig/*.pc "${SYSROOT}"/usr/lib64/pkgconfig/*.pc; do 245 [ -e "${_pc}" ] || continue 246 grep -q "^prefix=${SYSROOT}/usr\$" "${_pc}" \ 247 || die "sysroot .pc not fixed: ${_pc} (mk/pc-fixup.sh did not rewrite prefix=)" 248 done 249 250 # --- signing key: UNOS_SIGN_KEY > .sign-key > single key in ~/.unos-keys --- 251 # shellcheck disable=SC1090 252 . "${HERE}/sign-key.inc" 253 KEY=$(resolve_sign_key) || die "no signing key (mk/keymgmt.sh use)" 254 255 # --- emit signed .apk (v2): shell + pax-tar + openssl, no other languages --- 256 # fixed timestamp for the whole artifact (reproducible when pinned) 257 : "${SOURCE_DATE_EPOCH:=$(date +%s)}" 258 export SOURCE_DATE_EPOCH 259 260 # helper binary (C source is committed; the binary is output) 261 if [ ! -x "${HERE}/pax-tar" ] || [ "${HERE}/pax-tar.c" -nt "${HERE}/pax-tar" ]; then 262 msg "building pax-tar" 263 cc -std=c99 -O2 -Wall -Wextra -o "${HERE}/pax-tar" "${HERE}/pax-tar.c" 264 fi 265 266 # spick <relpath>... : move staged paths from DESTDIR into the current 267 # SUBDEST (split staging). Only valid inside a sub_<name>() function. 268 spick() { 269 [ -n "${SUBDEST:-}" ] || die "spick outside a subpackage split" 270 [ $# -ge 1 ] || die "spick needs paths" 271 for _p in "$@"; do 272 [ -e "${DESTDIR}/${_p}" ] || die "spick: no such staged path: ${_p}" 273 mkdir -p "${SUBDEST}/$(dirname "${_p}")" 274 mv "${DESTDIR}/${_p}" "${SUBDEST}/${_p}" 275 done 276 # moves leave empty parent dirs behind; prune them so the remainder 277 # assert below sees real leftovers only (never DESTDIR itself) 278 find "${DESTDIR}" -mindepth 1 -depth -type d -empty -delete 279 } 280 281 # emit_apk <stage> <pkgname> <desc> <depends> <provides> <replaces> <suffix> 282 # suffix "" = primary (scripts from FILESDIR/<s>); otherwise per-subpackage 283 # scripts from FILESDIR/<s>.<suffix>. 284 emit_apk() { 285 _stage=$1; _name=$2; _desc=$3; _depends=$4; _provides=$5; _replaces=$6; _sfx=$7 286 _ctrl="${WORK}/ctrl.${_name}" 287 rm -rf "${_ctrl}" 288 mkdir -p "${_ctrl}" 289 290 { 291 printf 'pkgname = %s\n' "${_name}" 292 printf 'pkgver = %s\n' "${PKGVER}" 293 printf 'pkgdesc = %s\n' "${_desc}" 294 printf 'url = %s\n' "${homepage}" 295 printf 'builddate = %s\n' "${SOURCE_DATE_EPOCH}" 296 printf 'packager = %s\n' "UNOS build driver (mk/build.sh)" 297 printf 'size = %s\n' "$(find "${_stage}" -type f -printf '%s\n' | awk '{s+=$1} END {print s+0}')" 298 printf 'arch = %s\n' "${ARCH}" 299 printf 'origin = %s\n' "${pkgname}" 300 printf 'maintainer = %s\n' "${maintainer}" 301 printf 'license = %s\n' "${license}" 302 for d in ${_depends}; do printf 'depend = %s\n' "${d}"; done 303 for p in ${_provides}; do printf 'provides = %s\n' "${p}"; done 304 for r in ${_replaces}; do printf 'replaces = %s\n' "${r}"; done 305 } > "${_ctrl}/.PKGINFO" 306 307 for s in pre-install post-install pre-deinstall post-deinstall pre-upgrade post-upgrade trigger; do 308 if [ -z "${_sfx}" ]; then _f="${FILESDIR}/${s}"; else _f="${FILESDIR}/${s}.${_sfx}"; fi 309 if [ -f "${_f}" ]; then 310 cp "${_f}" "${_ctrl}/.${s}" 311 chmod 755 "${_ctrl}/.${s}" 312 fi 313 done 314 315 # data segment (pax-tar: sorted, root-owned, per-file SHA1 headers). 316 # The data tar KEEPS its end-of-tar markers: it terminates the archive. 317 "${HERE}/pax-tar" "${_stage}" "${WORK}/data.${_name}.tar" 318 gzip -n -9 -f "${WORK}/data.${_name}.tar" 319 datahash=$(sha256sum "${WORK}/data.${_name}.tar.gz" | cut -d' ' -f1) 320 printf 'datahash = %s\n' "${datahash}" >> "${_ctrl}/.PKGINFO" 321 322 # control segment (pax-tar without checksum records; host tar not trusted). 323 # The control tar must NOT carry end-of-tar markers: gunzipped, the package 324 # is one continuous tar stream (sig record, control files, data files) and 325 # only the data segment terminates it. This is the abuild-sign layout. 326 "${HERE}/pax-tar" --no-checksum "${_ctrl}" "${WORK}/control.${_name}.tar" 327 ctlsiz=$(stat -c%s "${WORK}/control.${_name}.tar") 328 head -c $((ctlsiz - 1024)) "${WORK}/control.${_name}.tar" > "${WORK}/controlnotr.${_name}.tar" 329 gzip -n -9 -f "${WORK}/controlnotr.${_name}.tar" 330 mv "${WORK}/controlnotr.${_name}.tar.gz" "${WORK}/control.${_name}.tar.gz" 331 332 # signature segment: RSA/SHA1 over the control gzip stream bytes, single tar 333 # record with NO end-of-tar blocks, prepended to control.tar.gz in place 334 # (abuild-sign layout: .apk = signed-control + data = 3 gzip members). 335 KEYNAME=$(basename "${KEY}" .rsa) 336 openssl dgst -sha1 -sign "${KEY}" -out "${WORK}/sig.${_name}.bin" "${WORK}/control.${_name}.tar.gz" 337 mkdir -p "${WORK}/sigdir.${_name}" 338 cp "${WORK}/sig.${_name}.bin" "${WORK}/sigdir.${_name}/.SIGN.RSA.${KEYNAME}.rsa.pub" 339 "${HERE}/pax-tar" --no-checksum "${WORK}/sigdir.${_name}" "${WORK}/sig.${_name}.tar" 340 sigsiz=$(stat -c%s "${WORK}/sig.${_name}.tar") 341 head -c $((sigsiz - 1024)) "${WORK}/sig.${_name}.tar" > "${WORK}/signo.${_name}.tar" 342 gzip -n -9 -f "${WORK}/signo.${_name}.tar" 343 cat "${WORK}/signo.${_name}.tar.gz" "${WORK}/control.${_name}.tar.gz" > "${WORK}/signed-control.${_name}.tar.gz" 344 345 cat "${WORK}/signed-control.${_name}.tar.gz" "${WORK}/data.${_name}.tar.gz" \ 346 > "${OUTDIR}/${_name}-${PKGVER}.apk" 347 348 msg "done: ${OUTDIR}/${_name}-${PKGVER}.apk" 349 } 350 351 # --- subpackages: one template, several .apks --- 352 # Template declares: subpackages="name ..." (hyphens fine), per-sub 353 # short_desc_<underscored>/depends_<underscored> (provides/replaces optional), 354 # and sub_<underscored>() splitting DESTDIR into SUBDEST via spick (move 355 # semantics: the tree is partitioned, never duplicated). After all splits 356 # DESTDIR must be empty - the primary packs the remainder, so for meta 357 # primaries this asserts the meta carries no files by construction. 358 if [ -n "${subpackages}" ]; then 359 for _sub in ${subpackages}; do 360 _u=$(printf '%s' "${_sub}" | tr '-' '_') 361 command -v "sub_${_u}" >/dev/null 2>&1 || die "no sub_${_u}() for subpackage ${_sub}" 362 eval "_sd=\${short_desc_${_u}:-}" 363 [ -n "${_sd}" ] || die "template sets no short_desc_${_u}" 364 eval "_dd=\${depends_${_u}:-}" 365 eval "_pv=\${provides_${_u}:-}" 366 eval "_rp=\${replaces_${_u}:-}" 367 SUBDEST="${WORK}/split/${_sub}" 368 rm -rf "${SUBDEST}" 369 mkdir -p "${SUBDEST}" 370 export SUBDEST 371 msg "${pkgname}: split ${_sub}" 372 "sub_${_u}" 373 unset SUBDEST 374 msg "${pkgname}: packing ${_sub} ${PKGVER}" 375 emit_apk "${WORK}/split/${_sub}" "${_sub}" "${_sd}" "${_dd}" "${_pv}" "${_rp}" "${_sub}" 376 done 377 if [ -n "$(ls -A "${DESTDIR}")" ]; then 378 die "unpartitioned files remain for primary ${pkgname}: $(ls -A "${DESTDIR}" | head)" 379 fi 380 fi 381 382 msg "${pkgname}: packing ${PKGVER}" 383 emit_apk "${DESTDIR}" "${pkgname}" "${short_desc}" "${depends}" "${provides}" "${replaces}" ""