commit 048d4fed616c6848bfbf090593a22196d27b14bc
parent 824a5e90651d6d841539f90b24aa6b0ba0c2328e
Author: finwo <finwo@pm.me>
Date: Sat, 19 Sep 2026 16:41:03 +0200
Restrict jobs to refs with only:, and publish images on main and tags
Diffstat:
13 files changed, 723 insertions(+), 12 deletions(-)
diff --git a/.conductor.yml b/.conductor.yml
@@ -0,0 +1,81 @@
+# The pipeline conductor runs on itself.
+#
+# The suite is split in two because most of it needs nothing but node,
+# while the parts that exercise object storage and OIDC start real
+# containers and so need a worker offering the docker feature. Setting
+# CONDUCTOR_TEST_NO_DOCKER skips those rather than failing them, which
+# keeps the fast job honest about what it did and did not check.
+
+version: 1
+visibility: public
+
+defaults:
+ image: node:24-bookworm-slim
+ timeout: 15m
+
+jobs:
+ # Style rules that are cheap to break and cheap to check: ASCII only,
+ # no CRLF, no trailing whitespace.
+ style:
+ script:
+ - npm install --no-audit --no-fund
+ - node --test test/ascii.test.js
+ cache:
+ key: npm
+ paths: [.npm]
+
+ # Everything that runs without a docker socket.
+ test:
+ script:
+ - npm install --no-audit --no-fund
+ - npm test
+ env:
+ CONDUCTOR_TEST_NO_DOCKER: '1'
+ cache:
+ key: npm
+ paths: [.npm]
+
+ # The same suite with the container-backed tests switched on, so the S3
+ # and OIDC paths are exercised against real servers rather than skipped.
+ integration:
+ requires: [docker]
+ script:
+ - npm install --no-audit --no-fund
+ - npm test
+ cache:
+ key: npm
+ paths: [.npm]
+
+ # Both images have to build before anything claims a release is possible.
+ # Host architecture only, since this runs on every branch and the point
+ # is to catch a broken Dockerfile early rather than to ship anything.
+ images:
+ needs: [style, test]
+ requires: [docker]
+ image: docker:28-cli
+ script:
+ - docker build -f deploy/Dockerfile -t conductor:$CONDUCTOR_SHA .
+ - docker build -f deploy/Dockerfile.worker -t conductor-worker:$CONDUCTOR_SHA .
+
+ # The whole stack, built from the images and exercised with a real
+ # pipeline. Catches what unit tests cannot: that the thing people
+ # actually deploy starts, registers a worker and runs a job.
+ smoke:
+ needs: [images]
+ requires: [docker]
+ image: docker:28-cli
+ script:
+ - apk add --no-cache git openssl curl
+ - ./deploy/smoke.sh
+
+ # Pushes to Docker Hub. Restricted to main and release tags, so a branch
+ # build never moves :latest. Without the rule this job would run on every
+ # push, which is the whole reason only: exists.
+ publish:
+ needs: [smoke, integration]
+ requires: [docker]
+ image: docker:28-cli
+ only:
+ refs: [refs/heads/main, 'refs/tags/v*']
+ script:
+ - ./deploy/publish.sh
diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ Under construction. Working today:
- the worker: containerised jobs, services, features, caches, cleanup
- accounts, OIDC, project ownership, visibility and project variables
- the web interface, server rendered with htmx
- - container images, compose manifests and a deployment smoke test
+ - container images for amd64, arm64 and riscv64, published on release
Installation
------------
@@ -44,6 +44,9 @@ With docker:
docker compose -f deploy/docker-compose.yml up -d conductor
```
+Images are published as `finwo/conductor` and `finwo/conductor-worker`
+for amd64, arm64 and riscv64.
+
From source, requiring node 24 or newer and no native modules:
```sh
diff --git a/deploy/publish.sh b/deploy/publish.sh
@@ -0,0 +1,149 @@
+#!/bin/sh
+# deploy/publish.sh - build and push the images to Docker Hub
+#
+# Run by the publish job in .conductor.yml, which restricts it to main and
+# to release tags. Safe to run by hand for a dry run.
+#
+# What gets pushed, from CONDUCTOR_REF:
+#
+# refs/heads/main finwo/conductor:latest and :<short sha>
+# refs/tags/v1.2.0 finwo/conductor:1.2.0, :1.2 and :latest
+#
+# A tag publishes :latest as well, so :latest follows releases once there
+# are any, and follows main until then.
+#
+# Needs REGISTRY_USERNAME and REGISTRY_TOKEN as project variables. They are
+# masked in the log by the conductor, but this still keeps them off the
+# command line, where they would show up in a process list.
+#
+# DRY_RUN=1 build for the host architecture only, and push nothing
+
+set -eu
+
+: "${CONDUCTOR_REF:?CONDUCTOR_REF is not set}"
+: "${CONDUCTOR_SHA:?CONDUCTOR_SHA is not set}"
+
+DRY_RUN="${DRY_RUN:-0}"
+CONDUCTOR_IMAGE="${CONDUCTOR_IMAGE:-finwo/conductor}"
+WORKER_IMAGE="${WORKER_IMAGE:-finwo/conductor-worker}"
+
+# riscv64 has no node image and no static docker CLI, which is why both
+# Dockerfiles are built on Alpine. Everything here is available there.
+PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64,linux/riscv64}"
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+SHORT=$(printf '%s' "${CONDUCTOR_SHA}" | cut -c1-12)
+
+log() { printf '\n== %s\n' "$*"; }
+fail() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; }
+
+# Turns a ref into the list of tags it should publish.
+#
+# A version tag also publishes the major.minor series, so v1.2.1 moves
+# :1.2 as well. Deliberately no bare major: :1 moving across minor
+# releases tends to surprise people more than it helps them.
+tags_for_ref() {
+ ref=$1
+ case "${ref}" in
+ refs/heads/main)
+ printf 'latest\n%s\n' "${SHORT}"
+ ;;
+ refs/tags/v*)
+ version=${ref#refs/tags/v}
+ case "${version}" in
+ *[!0-9.]*)
+ fail "tag ${ref} is not a plain version, refusing to guess what to publish"
+ ;;
+ esac
+ printf '%s\n' "${version}"
+ # v1.2.3 -> 1.2, but v1.2 stays as it is.
+ series=$(printf '%s' "${version}" | cut -d. -f1-2)
+ [ "${series}" = "${version}" ] || printf '%s\n' "${series}"
+ printf 'latest\n'
+ ;;
+ *)
+ fail "ref ${ref} does not publish anything; the only: rule in .conductor.yml should have prevented this"
+ ;;
+ esac
+}
+
+TAGS=$(tags_for_ref "${CONDUCTOR_REF}")
+
+log "publishing from ${CONDUCTOR_REF} at ${SHORT}"
+printf 'tags:\n'
+printf ' %s\n' ${TAGS}
+printf 'platforms: %s\n' "${PLATFORMS}"
+
+# Collects --tag arguments for one image.
+tag_args() {
+ image=$1
+ for tag in ${TAGS}; do
+ printf -- '--tag\n%s:%s\n' "${image}" "${tag}"
+ done
+}
+
+if [ "${DRY_RUN}" = 1 ]; then
+ log "dry run: building for this architecture only, pushing nothing"
+ for spec in "deploy/Dockerfile ${CONDUCTOR_IMAGE}" "deploy/Dockerfile.worker ${WORKER_IMAGE}"; do
+ # shellcheck disable=SC2086
+ set -- ${spec}
+ dockerfile=$1
+ image=$2
+ printf '\n-- %s\n' "${image}"
+ docker build -f "${ROOT}/${dockerfile}" -t "${image}:dry-run" "${ROOT}" >/dev/null
+ printf 'would push:\n'
+ for tag in ${TAGS}; do printf ' %s:%s\n' "${image}" "${tag}"; done
+ done
+ printf '\nPASSED (dry run)\n'
+ exit 0
+fi
+
+: "${REGISTRY_USERNAME:?REGISTRY_USERNAME is not set}"
+: "${REGISTRY_TOKEN:?REGISTRY_TOKEN is not set}"
+
+log "registering emulators"
+# arm64 and riscv64 are emulated unless the worker is that architecture.
+# Without this, buildx cannot run the foreign binaries a build needs.
+docker run --privileged --rm tonistiigi/binfmt:qemu-v9.2.2 --install all >/dev/null
+
+log "preparing the builder"
+# The default builder cannot do more than one platform at a time.
+docker buildx inspect conductor-publish >/dev/null 2>&1 \
+ || docker buildx create --name conductor-publish --driver docker-container --bootstrap >/dev/null
+docker buildx use conductor-publish
+
+log "signing in"
+printf '%s' "${REGISTRY_TOKEN}" | docker login --username "${REGISTRY_USERNAME}" --password-stdin >/dev/null
+
+# Whatever happens next, do not leave the credentials behind on a worker
+# that is shared with other people's jobs.
+cleanup() { docker logout >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+log "building and pushing ${CONDUCTOR_IMAGE}"
+# shellcheck disable=SC2046
+docker buildx build \
+ --platform "${PLATFORMS}" \
+ --file "${ROOT}/deploy/Dockerfile" \
+ $(tag_args "${CONDUCTOR_IMAGE}" | tr '\n' ' ') \
+ --push \
+ "${ROOT}"
+
+log "building and pushing ${WORKER_IMAGE}"
+# shellcheck disable=SC2046
+docker buildx build \
+ --platform "${PLATFORMS}" \
+ --file "${ROOT}/deploy/Dockerfile.worker" \
+ $(tag_args "${WORKER_IMAGE}" | tr '\n' ' ') \
+ --push \
+ "${ROOT}"
+
+log "verifying the manifests"
+for image in "${CONDUCTOR_IMAGE}" "${WORKER_IMAGE}"; do
+ first=$(printf '%s' "${TAGS}" | head -1)
+ docker buildx imagetools inspect "${image}:${first}" \
+ | grep -i platform \
+ || fail "no platforms reported for ${image}:${first}"
+done
+
+printf '\nPASSED\n'
diff --git a/deploy/smoke.sh b/deploy/smoke.sh
@@ -64,6 +64,11 @@ jobs:
publish:
needs: [build]
script: ['echo published']
+ release:
+ needs: [build]
+ only:
+ refs: [refs/heads/main]
+ script: ['echo released']
PIPELINE
echo 'smoke test repository' > "${WORK}/repo/README.md"
git -C "${WORK}/repo" init -q -b main
@@ -177,4 +182,36 @@ printf 'artifact contents: %s\n' "${CONTENT}"
log "checking the interface"
curl -fsS "http://127.0.0.1:${PORT}/" | grep -q 'conductor' || fail "the interface did not render"
+# The release job is restricted to main. A push to anything else must not
+# produce it at all, rather than produce it and skip it, since a skipped
+# job fails the run.
+log "checking that a branch push leaves the restricted job out"
+BRANCH_BODY=$(printf '{"sha":"%s","ref":"refs/heads/feature"}' "${SHA}")
+BRANCH_SIG=$(printf '%s' "${BRANCH_BODY}" | openssl dgst -sha256 -hmac "${SECRET}" | sed 's/^.*[= ]//')
+BRANCH=$(curl -fsS -X POST "http://127.0.0.1:${PORT}/api/trigger/demo" \
+ -H 'Content-Type: application/json' \
+ -H "X-Hub-Signature-256: sha256=${BRANCH_SIG}" \
+ -d "${BRANCH_BODY}")
+printf '%s\n' "${BRANCH}"
+
+printf '%s' "${BRANCH}" | grep -q '"jobs":2' \
+ || fail "expected 2 jobs on a branch, got: ${BRANCH}"
+
+BRANCH_RUN=$(printf '%s' "${BRANCH}" | sed 's/.*"run_id":"\([^"]*\)".*/\1/')
+i=0
+while [ "${i}" -lt 90 ]; do
+ BRANCH_STATE=$(curl -fsS "http://127.0.0.1:${PORT}/api/runs/${BRANCH_RUN}" | sed 's/.*"state":"\([^"]*\)".*/\1/')
+ case "${BRANCH_STATE}" in
+ success|failed|cancelled) break ;;
+ esac
+ i=$((i + 1))
+ sleep 1
+done
+
+curl -fsS "http://127.0.0.1:${PORT}/api/runs/${BRANCH_RUN}" | grep -q '"release"' \
+ && fail "the release job should not exist on a branch run"
+
+printf 'branch run finished as %s without the release job\n' "${BRANCH_STATE}"
+[ "${BRANCH_STATE}" = success ] || fail "the branch run ended as ${BRANCH_STATE}"
+
printf '\nPASSED\n'
diff --git a/docs/deployment.md b/docs/deployment.md
@@ -9,11 +9,11 @@ Quick start
-----------
```sh
-docker compose -f deploy/docker-compose.yml up -d --build
+docker compose -f deploy/docker-compose.yml up -d
```
-That brings up a conductor on port 8080 with sqlite on a volume, and one
-worker. The compose file needs a worker token, so the first run is
+That pulls the published images and brings up a conductor on port 8080
+with sqlite on a volume, and one worker. The compose file needs a worker token, so the first run is
two steps:
```sh
@@ -42,16 +42,25 @@ log and artifact came back.
Images
------
-| Image | Contents |
-| -------------------- | ----------------------------------------------- |
-| `deploy/Dockerfile` | the conductor: node, git, the application |
-| `deploy/Dockerfile.worker` | the worker: node, git, tar, the docker CLI |
+| Image | Contents |
+| ------------------------- | ------------------------------------------- |
+| `finwo/conductor` | the conductor: node, git, the application |
+| `finwo/conductor-worker` | the worker: node, git, tar, the docker CLI |
+
+Published for `linux/amd64`, `linux/arm64` and `linux/riscv64`. Tags:
+`latest` follows releases, `1.2.3` and `1.2` pin a version, and a twelve
+character commit id pins an exact build from main.
+
+Both are built on Alpine rather than the official node image, which is
+only published for amd64, arm64 and ppc64le. Alpine 3.23 carries node 24
+on every architecture here, and a docker client too, so the worker no
+longer fetches a static binary that has no riscv64 build at all.
-Both build from the repository root:
+To build them yourself:
```sh
-docker build -f deploy/Dockerfile -t conductor .
-docker build -f deploy/Dockerfile.worker -t conductor-worker .
+docker build -f deploy/Dockerfile -t finwo/conductor .
+docker build -f deploy/Dockerfile.worker -t finwo/conductor-worker .
```
The worker image installs `yaml` so its configuration file may be YAML; the
@@ -179,6 +188,49 @@ docker compose -f deploy/docker-compose.yml pull
docker compose -f deploy/docker-compose.yml up -d
```
+Pin a version rather than tracking `latest` if an unattended restart
+picking up a new release would be unwelcome:
+
+```sh
+CONDUCTOR_IMAGE=finwo/conductor:1.2.3 \
+CONDUCTOR_WORKER_IMAGE=finwo/conductor-worker:1.2.3 \
+ docker compose -f deploy/docker-compose.yml up -d
+```
+
Workers can be upgraded independently and in any order. The worker API is
the boundary between them, and a worker that goes away mid-job has that job
requeued after `scheduler.heartbeat_timeout`.
+
+Publishing
+----------
+
+The images are built and pushed by the project's own pipeline, from
+`deploy/publish.sh`. It runs only on `main` and on `v*` tags, enforced by
+an `only:` rule in `.conductor.yml` rather than by a check inside the
+script, so a branch build cannot move `latest` even by accident.
+
+| Ref | Tags pushed |
+| ------------------ | ------------------------------------ |
+| `refs/heads/main` | `latest`, `<short sha>` |
+| `refs/tags/v1.2.3` | `1.2.3`, `1.2`, `latest` |
+
+A tag that is not a plain version, `v1.2.3-rc1` for instance, is refused
+rather than guessed at.
+
+Pushing needs `REGISTRY_USERNAME` and `REGISTRY_TOKEN` as project
+variables. They are masked in job logs, and passed to `docker login` on
+standard input rather than on the command line so they stay out of the
+process list. The job logs out again on the way out, whether or not it
+succeeded, so nothing is left behind on a shared worker.
+
+The foreign architectures are emulated with QEMU, registered per run with
+`tonistiigi/binfmt`. A riscv64 build under emulation is slow; giving the
+pool a worker of that architecture makes it native instead, since the
+worker declares its own architectures when it polls.
+
+To see what a release would push without pushing it:
+
+```sh
+CONDUCTOR_REF=refs/tags/v1.2.3 CONDUCTOR_SHA=$(git rev-parse HEAD) \
+ DRY_RUN=1 deploy/publish.sh
+```
diff --git a/docs/pipeline.md b/docs/pipeline.md
@@ -66,6 +66,7 @@ Jobs
| `allow_failure` | boolean | `false` | yes |
| `timeout` | duration | server default | yes |
| `max_attempts` | integer, 1 to 10 | `1` | yes |
+| `only` | mapping | none | no |
`image` and `script` are the only required fields, and `image` may come from
`defaults`.
@@ -92,6 +93,48 @@ exceed seven days; `artifacts.expire` may not exceed a year.
Fanning out
-----------
+### only
+
+Restricts a job to certain refs. Without it a job runs on every push.
+
+```yaml
+publish:
+ needs: [build]
+ only:
+ refs: [refs/heads/main, 'refs/tags/v*']
+ script:
+ - ./deploy/publish.sh
+```
+
+Patterns match the whole ref, so `refs/heads/main` rather than `main`.
+A `*` stands for any run of characters; everything else is literal, so a
+dot is a dot rather than any character.
+
+A job that does not match is left out of the run entirely, not recorded
+as skipped. A skipped job means a dependency collapsed and fails the run,
+which is the wrong reading for a publish step that was never meant to run
+on this branch.
+
+Anything depending on an excluded job is excluded with it. Dropping the
+dependency instead would let a job run without something it declared it
+needed, which is the worse surprise:
+
+```yaml
+# On a branch, none of these three run.
+publish:
+ only: { refs: [refs/heads/main] }
+ script: ['make publish']
+announce:
+ needs: [publish]
+ script: ['make announce']
+```
+
+A run triggered without a ref matches no pattern, so restricted jobs stay
+out rather than being handed a ref they were never written for.
+
+The rule is not inheritable from `defaults`. A restriction that silently
+applied to every job is hard to spot when the symptom is an empty run.
+
### arch
Expands the job into one job per architecture, and restricts each to a worker
diff --git a/src/conductor/scheduler.js b/src/conductor/scheduler.js
@@ -113,6 +113,10 @@ export function createScheduler({ cfg, db, git, logs, storage, projects, variabl
source: project.config_path,
defaultTimeout: cfg.scheduler.default_job_timeout,
defaultAttempts: 1,
+ // Decides which jobs this push runs at all, so a publish step
+ // restricted to main is absent from a branch run rather than
+ // recorded against it.
+ ref: ref ?? '',
});
const info = await git.commitInfo(project, headSha).catch(() => ({ subject: null }));
diff --git a/src/lib/pipeline/expand.js b/src/lib/pipeline/expand.js
@@ -149,6 +149,9 @@ export function expandPipeline(pipeline, options = {}) {
allow_failure: job.allow_failure,
timeout: job.timeout ?? defaultTimeout,
max_attempts: job.max_attempts ?? defaultAttempts,
+ // Every instance of a matrix job inherits the ref rule, so a
+ // restricted job stays restricted across all of its expansions.
+ only: job.only ?? null,
needs: [],
});
}
diff --git a/src/lib/pipeline/index.js b/src/lib/pipeline/index.js
@@ -8,17 +8,25 @@
import { parsePipeline } from './parse.js';
import { expandPipeline } from './expand.js';
import { topologicalOrder, depths, CycleError } from './dag.js';
+import { selectForRef } from './only.js';
import { PipelineError } from './schema.js';
export { parsePipeline } from './parse.js';
export { expandPipeline, interpolate } from './expand.js';
export { topologicalOrder, depths, transitiveDependents, runnable, CycleError } from './dag.js';
+export { refMatches, jobRunsOnRef, selectForRef } from './only.js';
export { PipelineError } from './schema.js';
export { SUPPORTED_VERSION, VISIBILITIES } from './parse.js';
export function compilePipeline(text, options = {}) {
const pipeline = parsePipeline(text, options);
- const jobs = expandPipeline(pipeline, options);
+ const expanded = expandPipeline(pipeline, options);
+
+ // Jobs restricted to other refs drop out before the graph is checked, so
+ // ordering and depth describe the run that will actually happen. Without
+ // a ref nothing is filtered, which keeps validation of a pipeline
+ // separate from deciding what one push will run.
+ const jobs = options.ref === undefined ? expanded : selectForRef(expanded, options.ref);
try {
topologicalOrder(jobs);
diff --git a/src/lib/pipeline/only.js b/src/lib/pipeline/only.js
@@ -0,0 +1,51 @@
+// src/lib/pipeline/only.js - restricting jobs to particular refs
+//
+// A job carrying an `only` rule runs when the ref that triggered the run
+// matches one of its patterns, and is left out of the run entirely when it
+// does not. Left out rather than recorded as skipped, because the
+// scheduler treats a skipped job as a reason to fail the run: that is the
+// right reading when a dependency collapsed, and the wrong one for a
+// publish step that was never meant to run on this branch.
+//
+// Anything depending on an excluded job is excluded with it. The
+// alternative, quietly dropping the dependency, would let a job run
+// without something it declared it needed, which is a worse surprise than
+// the job not running.
+
+import { transitiveDependents } from './dag.js';
+
+// Glob matching over a whole ref. Only * is special, standing for any run
+// of characters including none; everything else is literal. Deliberately
+// not a regular expression, since these come out of a repository and are
+// read by people who are not thinking about regular expression syntax.
+export function refMatches(pattern, ref) {
+ if (typeof pattern !== 'string' || typeof ref !== 'string') return false;
+ if (!pattern.includes('*')) return pattern === ref;
+
+ const source = pattern
+ .split('*')
+ .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+ .join('.*');
+ return new RegExp(`^${source}$`).test(ref);
+}
+
+export function jobRunsOnRef(job, ref) {
+ if (!job.only) return true;
+
+ // A run with no ref at all, which a manual or api trigger may produce,
+ // cannot match a ref pattern. Restricted jobs stay out rather than
+ // being handed a ref they were never written for.
+ if (typeof ref !== 'string' || ref === '') return false;
+
+ return job.only.refs.some((pattern) => refMatches(pattern, ref));
+}
+
+// The jobs that belong in a run for this ref, in their original order.
+export function selectForRef(jobs, ref) {
+ const excluded = new Set(jobs.filter((job) => !jobRunsOnRef(job, ref)).map((job) => job.name));
+ if (excluded.size === 0) return jobs;
+
+ for (const name of transitiveDependents(jobs, [...excluded])) excluded.add(name);
+
+ return jobs.filter((job) => !excluded.has(job.name));
+}
diff --git a/src/lib/pipeline/parse.js b/src/lib/pipeline/parse.js
@@ -31,6 +31,7 @@ export const VISIBILITIES = ['public', 'private'];
const JOB_KEYS = [
'image', 'script', 'needs', 'arch', 'matrix', 'requires', 'services',
'env', 'artifacts', 'cache', 'allow_failure', 'timeout', 'max_attempts',
+ 'only',
];
// Fields a job may inherit from defaults.
@@ -39,6 +40,7 @@ const DEFAULT_KEYS = [
'allow_failure', 'timeout', 'max_attempts',
];
+const ONLY_KEYS = ['refs'];
const ARTIFACT_KEYS = ['paths', 'expire', 'when'];
const ARTIFACT_WHEN = ['on_success', 'on_failure', 'always'];
const CACHE_KEYS = ['key', 'paths'];
@@ -203,9 +205,46 @@ function parseJob(problems, path, raw, defaults) {
max_attempts: raw.max_attempts === undefined
? defaults.max_attempts
: asInteger(problems, `${path}.max_attempts`, raw.max_attempts, { min: 1, max: 10 }),
+ only: parseOnly(problems, `${path}.only`, raw.only),
};
}
+// Restricts a job to certain refs. Deliberately not inheritable from
+// defaults: a rule that silently applied to every job is the kind of thing
+// that stops a pipeline running at all and takes an afternoon to find.
+//
+// only:
+// refs: [refs/heads/main, 'refs/tags/v*']
+//
+// Patterns match the whole ref, so refs/heads/main rather than main, with
+// * standing for any run of characters. Absent means the job always runs.
+function parseOnly(problems, path, raw) {
+ if (raw === undefined || raw === null) return null;
+ if (!isPlainObject(raw)) {
+ problems.add(path, 'must be a mapping, for example: only: { refs: [refs/heads/main] }');
+ return null;
+ }
+
+ for (const key of Object.keys(raw)) {
+ if (!ONLY_KEYS.includes(key)) {
+ problems.add(`${path}.${key}`, `unknown key, expected one of: ${ONLY_KEYS.join(', ')}`);
+ }
+ }
+
+ if (raw.refs === undefined) {
+ problems.add(path, 'needs a refs list, otherwise it restricts nothing');
+ return null;
+ }
+
+ const refs = asStringList(problems, `${path}.refs`, raw.refs, { max: 64 });
+ if (refs.length === 0) {
+ problems.add(`${path}.refs`, 'must list at least one ref pattern');
+ return null;
+ }
+
+ return { refs };
+}
+
// A need is either a job name, or a mapping for the cases where the default
// dimension matching is not what is wanted.
function parseNeeds(problems, path, raw) {
diff --git a/test/only.test.js b/test/only.test.js
@@ -0,0 +1,160 @@
+// test/only.test.js - restricting jobs to particular refs
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { compilePipeline, refMatches, PipelineError } from '../src/lib/pipeline/index.js';
+
+const PIPELINE = `
+version: 1
+defaults:
+ image: alpine:3
+jobs:
+ build:
+ script: ['make']
+ publish:
+ needs: [build]
+ only:
+ refs: [refs/heads/main, 'refs/tags/v*']
+ script: ['make publish']
+`;
+
+function names(text, ref) {
+ return compilePipeline(text, { ref }).jobs.map((j) => j.name).sort();
+}
+
+test('a pattern without a star matches exactly', () => {
+ assert.ok(refMatches('refs/heads/main', 'refs/heads/main'));
+ assert.ok(!refMatches('refs/heads/main', 'refs/heads/main-2'));
+ assert.ok(!refMatches('refs/heads/main', 'refs/heads/mai'));
+
+ // A pattern is matched against the whole ref, so a branch name on its
+ // own is not enough.
+ assert.ok(!refMatches('main', 'refs/heads/main'));
+});
+
+test('a star matches any run of characters', () => {
+ assert.ok(refMatches('refs/tags/v*', 'refs/tags/v1.2.0'));
+ assert.ok(refMatches('refs/tags/v*', 'refs/tags/v'));
+ assert.ok(!refMatches('refs/tags/v*', 'refs/heads/v1'));
+ assert.ok(refMatches('*', 'refs/heads/anything'));
+ assert.ok(refMatches('refs/*/main', 'refs/heads/main'));
+});
+
+test('the rest of a pattern is literal, not a regular expression', () => {
+ // A dot is a dot. Were this compiled straight to a regular expression,
+ // v1.2.0 would also match v1x2x0.
+ assert.ok(refMatches('refs/tags/v1.2.0', 'refs/tags/v1.2.0'));
+ assert.ok(!refMatches('refs/tags/v1.2.0', 'refs/tags/v1x2x0'));
+
+ // And these would be quantifiers rather than characters.
+ assert.ok(refMatches('refs/heads/fix+', 'refs/heads/fix+'));
+ assert.ok(!refMatches('refs/heads/fix+', 'refs/heads/fixx'));
+});
+
+test('a restricted job is left out of runs for other refs', () => {
+ assert.deepEqual(names(PIPELINE, 'refs/heads/main'), ['build', 'publish']);
+ assert.deepEqual(names(PIPELINE, 'refs/tags/v1.2.0'), ['build', 'publish']);
+ assert.deepEqual(names(PIPELINE, 'refs/heads/feature'), ['build']);
+});
+
+test('it is left out rather than recorded as skipped', () => {
+ // The scheduler reads a skipped job as a reason to fail the run, so a
+ // publish step that was never meant to run here must not appear at all.
+ const { jobs } = compilePipeline(PIPELINE, { ref: 'refs/heads/feature' });
+ assert.ok(!jobs.some((j) => j.name === 'publish'));
+});
+
+test('jobs depending on an excluded job are excluded too', () => {
+ const text = `
+version: 1
+defaults:
+ image: alpine:3
+jobs:
+ build:
+ script: ['make']
+ publish:
+ needs: [build]
+ only:
+ refs: [refs/heads/main]
+ script: ['make publish']
+ announce:
+ needs: [publish]
+ script: ['make announce']
+`;
+ // announce carries no rule of its own, but running it without publish
+ // would mean running it without something it declared it needed.
+ assert.deepEqual(names(text, 'refs/heads/feature'), ['build']);
+ assert.deepEqual(names(text, 'refs/heads/main'), ['announce', 'build', 'publish']);
+});
+
+test('depth and ordering describe the run that will happen', () => {
+ const { jobs } = compilePipeline(PIPELINE, { ref: 'refs/heads/feature' });
+ assert.equal(jobs.length, 1);
+ assert.equal(jobs[0].name, 'build');
+ assert.equal(jobs[0].depth, 0);
+});
+
+test('without a ref nothing is filtered, so a pipeline can still be validated', () => {
+ const { jobs } = compilePipeline(PIPELINE);
+ assert.deepEqual(jobs.map((j) => j.name).sort(), ['build', 'publish']);
+});
+
+test('a restricted job cannot match a run that has no ref', () => {
+ assert.deepEqual(names(PIPELINE, ''), ['build']);
+});
+
+test('a malformed rule is reported rather than ignored', () => {
+ const cases = [
+ ['only: refs/heads/main', 'must be a mapping'],
+ ['only:\n branches: [main]', 'unknown key'],
+ ['only:\n refs: []', 'at least one'],
+ ['only: {}', 'needs a refs list'],
+ ];
+
+ for (const [fragment, expected] of cases) {
+ const text = `
+version: 1
+defaults:
+ image: alpine:3
+jobs:
+ publish:
+ script: ['make publish']
+ ${fragment}
+`;
+ assert.throws(
+ () => compilePipeline(text, { ref: 'refs/heads/main' }),
+ (error) => {
+ assert.ok(error instanceof PipelineError, `${fragment}: expected PipelineError`);
+ const text = error.errors.map((e) => e.message).join('; ');
+ assert.match(text, new RegExp(expected), `${fragment}: got ${text}`);
+ return true;
+ },
+ `expected ${JSON.stringify(fragment)} to be rejected`,
+ );
+ }
+});
+
+test('the rule is not inherited from defaults', () => {
+ // Inheriting it would restrict every job at once, which is a hard
+ // mistake to spot when the symptom is an empty run.
+ const text = `
+version: 1
+defaults:
+ image: alpine:3
+ only:
+ refs: [refs/heads/main]
+jobs:
+ build:
+ script: ['make']
+`;
+ assert.throws(
+ () => compilePipeline(text, { ref: 'refs/heads/main' }),
+ (error) => {
+ assert.ok(error instanceof PipelineError);
+ const message = error.errors.map((e) => `${e.path}: ${e.message}`).join('; ');
+ assert.match(message, /defaults\.only/, `expected defaults.only to be rejected, got ${message}`);
+ return true;
+ },
+ );
+});
diff --git a/test/self-pipeline.test.js b/test/self-pipeline.test.js
@@ -0,0 +1,81 @@
+// test/self-pipeline.test.js - the pipeline this project runs on itself
+//
+// .conductor.yml is not covered by examples.test.js, and it is the one
+// pipeline whose breakage stops the project from building at all. It also
+// encodes decisions that are easy to undo by accident: which jobs may run
+// on any worker, and which need a docker socket.
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { compilePipeline } from '../src/lib/pipeline/index.js';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+async function selfPipeline(arches = ['x86_64'], ref = 'refs/heads/main') {
+ const text = await fs.readFile(path.join(ROOT, '.conductor.yml'), 'utf8');
+ return compilePipeline(text, { source: '.conductor.yml', arches, ref });
+}
+
+test('the pipeline this project runs on itself compiles', async () => {
+ const { jobs } = await selfPipeline();
+ assert.ok(jobs.length > 0, '.conductor.yml produced no jobs');
+});
+
+test('its jobs declare the features they actually need', async () => {
+ const { jobs } = await selfPipeline();
+ const byName = new Map(jobs.map((job) => [job.name, job]));
+
+ for (const name of ['style', 'test', 'integration', 'images', 'smoke', 'publish']) {
+ assert.ok(byName.has(name), `expected a ${name} job`);
+ }
+
+ // The fast job has to actually skip the container-backed tests, or it
+ // fails on any worker without a docker socket.
+ assert.equal(byName.get('test').env.CONDUCTOR_TEST_NO_DOCKER, '1');
+
+ // Anything starting containers has to say so, otherwise it gets handed
+ // to a worker that cannot run it.
+ for (const name of ['integration', 'images', 'smoke', 'publish']) {
+ assert.ok(
+ byName.get(name).requires.includes('docker'),
+ `${name} starts containers and must require the docker feature`,
+ );
+ }
+
+ // Conversely, the jobs meant to run anywhere must not demand features.
+ for (const name of ['style', 'test']) {
+ assert.deepEqual(byName.get(name).requires, [], `${name} should run on any worker`);
+ }
+});
+
+test('publishing is restricted to main and release tags', async () => {
+ // The cost of getting this wrong is a branch build overwriting :latest
+ // on Docker Hub, which is not something a test should leave to trust.
+ const onMain = await selfPipeline();
+ assert.ok(onMain.jobs.some((j) => j.name === 'publish'));
+
+ for (const ref of ['refs/heads/feature', 'refs/heads/main-2', 'refs/pull/7/head', '']) {
+ const { jobs } = compilePipeline(
+ await fs.readFile(path.join(ROOT, '.conductor.yml'), 'utf8'),
+ { source: '.conductor.yml', arches: ['x86_64'], ref },
+ );
+ assert.ok(
+ !jobs.some((j) => j.name === 'publish'),
+ `publish must not run for ${JSON.stringify(ref)}`,
+ );
+ // The rest of the pipeline still has to run on a branch.
+ assert.ok(jobs.some((j) => j.name === 'test'), `test should still run for ${JSON.stringify(ref)}`);
+ }
+
+ for (const ref of ['refs/heads/main', 'refs/tags/v1.2.0']) {
+ const { jobs } = compilePipeline(
+ await fs.readFile(path.join(ROOT, '.conductor.yml'), 'utf8'),
+ { source: '.conductor.yml', arches: ['x86_64'], ref },
+ );
+ assert.ok(jobs.some((j) => j.name === 'publish'), `publish should run for ${ref}`);
+ }
+});
