commit 5d11ffb12e6835d1516157fbe2e0c39446c4717d
parent ece142938b360219e241e7c2fb0eb938dab8e37c
Author: finwo <finwo@pm.me>
Date: Fri, 18 Sep 2026 23:34:16 +0200
Pipeline parsing, matrix expansion and job graph
Diffstat:
7 files changed, 1446 insertions(+), 0 deletions(-)
diff --git a/src/lib/pipeline/dag.js b/src/lib/pipeline/dag.js
@@ -0,0 +1,91 @@
+// src/lib/pipeline/dag.js - dependency graph checks and traversal
+//
+// The prototype stored a single integer level per job and treated the lowest
+// queued level as the next runnable job. That serialized every run and, on
+// failure, only skipped direct dependents, so a transitive dependent could
+// still be dispatched with its dependency missing. Both problems come from
+// throwing the graph away, so the graph is kept here and used directly.
+
+export class CycleError extends Error {
+ constructor(cycle) {
+ super(`pipeline contains a dependency cycle: ${cycle.join(' -> ')}`);
+ this.name = 'CycleError';
+ this.cycle = cycle;
+ }
+}
+
+// Returns the jobs in an order where every dependency precedes its
+// dependents. Throws CycleError naming the cycle when none exists.
+export function topologicalOrder(jobs) {
+ const byName = new Map(jobs.map((j) => [j.name, j]));
+ const state = new Map();
+ const order = [];
+ const stack = [];
+
+ function visit(name) {
+ const current = state.get(name);
+ if (current === 'done') return;
+ if (current === 'active') {
+ const from = stack.indexOf(name);
+ throw new CycleError([...stack.slice(from), name]);
+ }
+
+ state.set(name, 'active');
+ stack.push(name);
+ for (const dep of byName.get(name).needs) {
+ if (byName.has(dep)) visit(dep);
+ }
+ stack.pop();
+ state.set(name, 'done');
+ order.push(name);
+ }
+
+ // Sorted so that an unchanged pipeline always produces the same order.
+ for (const name of [...byName.keys()].sort()) visit(name);
+ return order.map((name) => byName.get(name));
+}
+
+// Depth of each job, where a job with no dependencies is 0. Used only for
+// display; scheduling reads the edges, never the depth.
+export function depths(jobs) {
+ const byName = new Map(jobs.map((j) => [j.name, j]));
+ const out = new Map();
+ for (const job of topologicalOrder(jobs)) {
+ const deps = job.needs.filter((d) => byName.has(d));
+ out.set(job.name, deps.length === 0 ? 0 : Math.max(...deps.map((d) => out.get(d))) + 1);
+ }
+ return out;
+}
+
+// Every job reachable by following dependents from the given names. This is
+// what a failure must skip: direct dependents alone leave transitive ones
+// runnable against a dependency that never produced anything.
+export function transitiveDependents(jobs, startNames) {
+ const dependents = new Map(jobs.map((j) => [j.name, []]));
+ for (const job of jobs) {
+ for (const dep of job.needs) {
+ if (dependents.has(dep)) dependents.get(dep).push(job.name);
+ }
+ }
+
+ const seen = new Set();
+ const queue = [...startNames];
+ while (queue.length > 0) {
+ const name = queue.shift();
+ for (const child of dependents.get(name) ?? []) {
+ if (seen.has(child)) continue;
+ seen.add(child);
+ queue.push(child);
+ }
+ }
+ return seen;
+}
+
+// Jobs whose dependencies have all reached a satisfying state. Passed the
+// current state of every job by name.
+export function runnable(jobs, stateByName, { satisfied = ['success'] } = {}) {
+ return jobs.filter((job) => {
+ if (stateByName.get(job.name) !== 'queued') return false;
+ return job.needs.every((dep) => satisfied.includes(stateByName.get(dep)));
+ });
+}
diff --git a/src/lib/pipeline/expand.js b/src/lib/pipeline/expand.js
@@ -0,0 +1,221 @@
+// src/lib/pipeline/expand.js - matrix and architecture expansion
+//
+// Turns each job template into one concrete job per combination of its
+// dimensions, then resolves needs between the concrete jobs.
+//
+// Dimension matching is the part worth understanding. When a job declares
+// needs on a template that shares a dimension with it, the dependency is
+// matched on that dimension rather than fanned out across all of it. So an
+// aarch64 package job needs the aarch64 build, not every build. Dimensions
+// the dependency has but the dependent does not are fanned out, which is
+// what you want when a single publish job waits for every architecture.
+// Write `{ job: name, match: all }` to opt out.
+
+import { Problems, MATRIX_KEY_PATTERN } from './schema.js';
+
+// Matches ${{ arch }} and ${{ matrix.pkg }}, tolerating inner whitespace.
+const INTERPOLATION = /\$\{\{\s*([^}]*?)\s*\}\}/g;
+
+// jobs.name is 191 characters in the mysql schema.
+export const MAX_JOB_NAME = 191;
+
+export function interpolate(problems, path, text, context) {
+ if (typeof text !== 'string') return text;
+
+ return text.replace(INTERPOLATION, (whole, expr) => {
+ if (expr === 'arch') {
+ if (context.arch === null) {
+ problems.add(path, 'refers to ${{ arch }} but the job declares no arch');
+ return whole;
+ }
+ return context.arch;
+ }
+
+ const matrixMatch = /^matrix\.([A-Za-z_][A-Za-z0-9_]*)$/.exec(expr);
+ if (matrixMatch) {
+ const key = matrixMatch[1];
+ if (!Object.hasOwn(context.matrix, key)) {
+ const known = Object.keys(context.matrix);
+ problems.add(
+ path,
+ `refers to \${{ matrix.${key} }} which the job does not define` +
+ (known.length > 0 ? `; available: ${known.join(', ')}` : '')
+ );
+ return whole;
+ }
+ return context.matrix[key];
+ }
+
+ problems.add(path, `unknown expression \${{ ${expr} }}; expected arch or matrix.<name>`);
+ return whole;
+ });
+}
+
+function cartesian(dimensions) {
+ let rows = [{}];
+ for (const [key, values] of dimensions) {
+ const next = [];
+ for (const row of rows) {
+ for (const value of values) next.push({ ...row, [key]: value });
+ }
+ rows = next;
+ }
+ return rows;
+}
+
+// build:arch=x86_64,pkg=musl
+function instanceName(baseName, dims) {
+ const entries = Object.entries(dims);
+ if (entries.length === 0) return baseName;
+ return `${baseName}:${entries.map(([k, v]) => `${k}=${v}`).join(',')}`;
+}
+
+function dimensionsOf(job) {
+ const dims = [];
+ if (job.arch.length > 0) dims.push(['arch', job.arch]);
+ for (const [key, values] of Object.entries(job.matrix)) dims.push([key, values]);
+ return dims;
+}
+
+export function expandPipeline(pipeline, options = {}) {
+ const problems = new Problems();
+ const defaultTimeout = options.defaultTimeout ?? 3600;
+ const defaultAttempts = options.defaultAttempts ?? 1;
+
+ // Pass one: build every concrete job.
+ const byBase = new Map();
+ const instances = [];
+
+ for (const [baseName, job] of Object.entries(pipeline.jobs)) {
+ const dims = dimensionsOf(job);
+ const combos = cartesian(dims);
+ const made = [];
+
+ for (const combo of combos) {
+ const arch = combo.arch ?? null;
+ const matrix = { ...combo };
+ delete matrix.arch;
+
+ const name = instanceName(baseName, combo);
+ const path = `jobs.${baseName}`;
+ if (name.length > MAX_JOB_NAME) {
+ problems.add(path, `expands to a job name longer than ${MAX_JOB_NAME} characters: ${name}`);
+ continue;
+ }
+
+ const context = { arch, matrix };
+ const env = {};
+ for (const [key, value] of Object.entries(job.env)) {
+ env[key] = interpolate(problems, `${path}.env.${key}`, value, context);
+ }
+ // Dimension values are exposed to the script, so a matrix job rarely
+ // needs interpolation at all.
+ if (arch !== null) env.ARCH = arch;
+ for (const [key, value] of Object.entries(matrix)) {
+ if (MATRIX_KEY_PATTERN.test(key)) env[`MATRIX_${key.toUpperCase()}`] = value;
+ }
+
+ made.push({
+ name,
+ baseName,
+ arch,
+ matrix,
+ dims: combo,
+ image: interpolate(problems, `${path}.image`, job.image, context),
+ script: job.script.map((line, i) => interpolate(problems, `${path}.script[${i}]`, line, context)),
+ requires: job.requires.map((r, i) => interpolate(problems, `${path}.requires[${i}]`, r, context)),
+ services: job.services.map((service, i) => ({
+ image: interpolate(problems, `${path}.services[${i}].image`, service.image, context),
+ alias: service.alias,
+ env: Object.fromEntries(Object.entries(service.env).map(([k, v]) => [
+ k, interpolate(problems, `${path}.services[${i}].env.${k}`, v, context),
+ ])),
+ entrypoint: service.entrypoint,
+ command: service.command,
+ })),
+ env,
+ artifacts: job.artifacts
+ ? {
+ ...job.artifacts,
+ paths: job.artifacts.paths.map((p, i) => interpolate(problems, `${path}.artifacts.paths[${i}]`, p, context)),
+ }
+ : null,
+ cache: job.cache
+ ? {
+ key: interpolate(problems, `${path}.cache.key`, job.cache.key, context),
+ paths: job.cache.paths.map((p, i) => interpolate(problems, `${path}.cache.paths[${i}]`, p, context)),
+ }
+ : null,
+ allow_failure: job.allow_failure,
+ timeout: job.timeout ?? defaultTimeout,
+ max_attempts: job.max_attempts ?? defaultAttempts,
+ needs: [],
+ });
+ }
+
+ byBase.set(baseName, { job, dimensionNames: dims.map(([k]) => k), instances: made });
+ instances.push(...made);
+ }
+
+ const byName = new Map(instances.map((i) => [i.name, i]));
+
+ // Pass two: resolve needs now that every concrete job exists.
+ for (const [baseName, entry] of byBase) {
+ for (const instance of entry.instances) {
+ const resolved = new Set();
+
+ for (const need of entry.job.needs) {
+ const path = `jobs.${baseName}.needs`;
+
+ // An exact concrete job name wins, which is the escape hatch for
+ // depending on one specific combination.
+ if (byName.has(need.job)) {
+ if (need.job === instance.name) {
+ problems.add(path, `job ${JSON.stringify(baseName)} cannot depend on itself`);
+ } else {
+ resolved.add(need.job);
+ }
+ continue;
+ }
+
+ const target = byBase.get(need.job);
+ if (!target) {
+ const known = [...byBase.keys()].filter((n) => n !== baseName);
+ problems.add(
+ path,
+ `refers to unknown job ${JSON.stringify(need.job)}` +
+ (known.length > 0 ? `; defined jobs are ${known.join(', ')}` : '')
+ );
+ continue;
+ }
+ if (need.job === baseName) {
+ problems.add(path, `job ${JSON.stringify(baseName)} cannot depend on itself`);
+ continue;
+ }
+
+ let candidates = target.instances;
+ if (need.match === 'shared') {
+ const shared = target.dimensionNames.filter((d) => Object.hasOwn(instance.dims, d));
+ if (shared.length > 0) {
+ candidates = candidates.filter((c) => shared.every((d) => c.dims[d] === instance.dims[d]));
+ if (candidates.length === 0) {
+ problems.add(
+ path,
+ `no instance of ${JSON.stringify(need.job)} matches ${instance.name} on ` +
+ `${shared.join(', ')}; add the missing value, or use { job: ${need.job}, match: all }`
+ );
+ continue;
+ }
+ }
+ }
+
+ for (const candidate of candidates) resolved.add(candidate.name);
+ }
+
+ instance.needs = [...resolved].sort();
+ }
+ }
+
+ problems.throwIfAny(pipeline.source);
+ return instances;
+}
diff --git a/src/lib/pipeline/index.js b/src/lib/pipeline/index.js
@@ -0,0 +1,36 @@
+// src/lib/pipeline/index.js - pipeline compilation
+//
+// compilePipeline turns the text of a .conductor.yml into the concrete job
+// graph the scheduler stores. Every failure mode raises PipelineError with
+// the full list of problems, so a bad pipeline is reported once rather than
+// one mistake per push.
+
+import { parsePipeline } from './parse.js';
+import { expandPipeline } from './expand.js';
+import { topologicalOrder, depths, CycleError } from './dag.js';
+import { PipelineError } from './schema.js';
+
+export { parsePipeline } from './parse.js';
+export { expandPipeline, interpolate } from './expand.js';
+export { topologicalOrder, depths, transitiveDependents, runnable, CycleError } from './dag.js';
+export { PipelineError } from './schema.js';
+export { SUPPORTED_VERSION } from './parse.js';
+
+export function compilePipeline(text, options = {}) {
+ const pipeline = parsePipeline(text, options);
+ const jobs = expandPipeline(pipeline, options);
+
+ try {
+ topologicalOrder(jobs);
+ } catch (e) {
+ if (e instanceof CycleError) {
+ throw new PipelineError([{ path: 'jobs', message: e.message }], pipeline.source);
+ }
+ throw e;
+ }
+
+ const depth = depths(jobs);
+ for (const job of jobs) job.depth = depth.get(job.name);
+
+ return { version: pipeline.version, source: pipeline.source, jobs };
+}
diff --git a/src/lib/pipeline/parse.js b/src/lib/pipeline/parse.js
@@ -0,0 +1,373 @@
+// src/lib/pipeline/parse.js - reads and validates a .conductor.yml
+//
+// Produces a normalized document: every job has every field filled in from
+// defaults, so later stages never have to ask whether something was set.
+// Nothing here expands matrices or resolves dependencies; see expand.js.
+
+import YAML from 'yaml';
+import {
+ PipelineError,
+ Problems,
+ isPlainObject,
+ checkUnknown,
+ asString,
+ asName,
+ asBoolean,
+ asInteger,
+ asStringList,
+ asEnvMap,
+ asDuration,
+ typeName,
+ MATRIX_KEY_PATTERN,
+ NAME_PATTERN,
+} from './schema.js';
+
+const TOP_KEYS = ['version', 'defaults', 'jobs'];
+
+const JOB_KEYS = [
+ 'image', 'script', 'needs', 'arch', 'matrix', 'requires', 'services',
+ 'env', 'artifacts', 'cache', 'allow_failure', 'timeout', 'max_attempts',
+];
+
+// Fields a job may inherit from defaults.
+const DEFAULT_KEYS = [
+ 'image', 'arch', 'requires', 'env', 'services', 'cache',
+ 'allow_failure', 'timeout', 'max_attempts',
+];
+
+const ARTIFACT_KEYS = ['paths', 'expire', 'when'];
+const ARTIFACT_WHEN = ['on_success', 'on_failure', 'always'];
+const CACHE_KEYS = ['key', 'paths'];
+const SERVICE_KEYS = ['image', 'alias', 'env', 'entrypoint', 'command'];
+
+export const SUPPORTED_VERSION = 1;
+
+export function parsePipeline(text, options = {}) {
+ const source = options.source || '.conductor.yml';
+ const problems = new Problems();
+
+ let doc;
+ try {
+ doc = YAML.parse(text, { prettyErrors: true });
+ } catch (e) {
+ throw new PipelineError([{ path: '', message: `not valid YAML: ${e.message}` }], source);
+ }
+
+ if (doc === null || doc === undefined) {
+ throw new PipelineError([{ path: '', message: 'file is empty' }], source);
+ }
+ if (!isPlainObject(doc)) {
+ throw new PipelineError([{ path: '', message: `expected a mapping at the top level, got ${typeName(doc)}` }], source);
+ }
+
+ checkUnknown(problems, '', doc, TOP_KEYS);
+
+ if (doc.version === undefined) {
+ problems.add('version', `is required; this conductor understands version ${SUPPORTED_VERSION}`);
+ } else if (doc.version !== SUPPORTED_VERSION) {
+ problems.add('version', `unsupported version ${JSON.stringify(doc.version)}, expected ${SUPPORTED_VERSION}`);
+ }
+
+ const defaults = parseDefaults(problems, doc.defaults);
+
+ if (doc.jobs === undefined) {
+ problems.add('jobs', 'is required');
+ } else if (!isPlainObject(doc.jobs)) {
+ problems.add('jobs', `expected a mapping of job names, got ${typeName(doc.jobs)}`);
+ } else if (Object.keys(doc.jobs).length === 0) {
+ problems.add('jobs', 'must define at least one job');
+ }
+
+ const jobs = {};
+ if (isPlainObject(doc.jobs)) {
+ for (const [name, raw] of Object.entries(doc.jobs)) {
+ const path = `jobs.${name}`;
+ if (!NAME_PATTERN.test(name) || name.length > 100) {
+ problems.add(path, 'job name must start with a letter or digit and contain only letters, digits, underscore, dot and hyphen');
+ continue;
+ }
+ if (!isPlainObject(raw)) {
+ problems.add(path, `expected a mapping, got ${typeName(raw)}`);
+ continue;
+ }
+ jobs[name] = parseJob(problems, path, raw, defaults);
+ }
+ }
+
+ problems.throwIfAny(source);
+ return { version: SUPPORTED_VERSION, defaults, jobs, source };
+}
+
+function parseDefaults(problems, raw) {
+ const empty = {
+ image: undefined,
+ arch: undefined,
+ requires: [],
+ env: {},
+ services: [],
+ cache: null,
+ allow_failure: false,
+ timeout: undefined,
+ max_attempts: undefined,
+ };
+ if (raw === undefined) return empty;
+ if (!isPlainObject(raw)) {
+ problems.add('defaults', `expected a mapping, got ${typeName(raw)}`);
+ return empty;
+ }
+
+ checkUnknown(problems, 'defaults', raw, DEFAULT_KEYS);
+
+ return {
+ image: raw.image === undefined ? undefined : asString(problems, 'defaults.image', raw.image),
+ arch: raw.arch === undefined ? undefined : parseArch(problems, 'defaults.arch', raw.arch),
+ requires: asStringList(problems, 'defaults.requires', raw.requires),
+ env: asEnvMap(problems, 'defaults.env', raw.env),
+ services: parseServices(problems, 'defaults.services', raw.services),
+ cache: parseCache(problems, 'defaults.cache', raw.cache),
+ allow_failure: raw.allow_failure === undefined ? false : asBoolean(problems, 'defaults.allow_failure', raw.allow_failure) ?? false,
+ timeout: raw.timeout === undefined ? undefined : asDuration(problems, 'defaults.timeout', raw.timeout),
+ max_attempts: raw.max_attempts === undefined ? undefined : asInteger(problems, 'defaults.max_attempts', raw.max_attempts, { min: 1, max: 10 }),
+ };
+}
+
+function parseArch(problems, path, raw) {
+ const list = asStringList(problems, path, raw, { max: 32 });
+ const out = [];
+ list.forEach((value, i) => {
+ if (!NAME_PATTERN.test(value)) {
+ problems.add(`${path}[${i}]`, `${JSON.stringify(value)} is not a valid architecture name`);
+ return;
+ }
+ if (out.includes(value)) {
+ problems.add(`${path}[${i}]`, `duplicate architecture ${JSON.stringify(value)}`);
+ return;
+ }
+ out.push(value);
+ });
+ return out;
+}
+
+function parseJob(problems, path, raw, defaults) {
+ checkUnknown(problems, path, raw, JOB_KEYS);
+
+ const image = raw.image === undefined ? defaults.image : asString(problems, `${path}.image`, raw.image);
+ if (image === undefined) {
+ problems.add(`${path}.image`, 'is required; set it on the job or under defaults');
+ }
+
+ if (raw.script === undefined) {
+ problems.add(`${path}.script`, 'is required');
+ }
+ const script = asStringList(problems, `${path}.script`, raw.script, { max: 65536 });
+ if (raw.script !== undefined && script.length === 0) {
+ problems.add(`${path}.script`, 'must contain at least one command');
+ }
+
+ const arch = raw.arch === undefined ? (defaults.arch ?? []) : parseArch(problems, `${path}.arch`, raw.arch);
+ const matrix = parseMatrix(problems, `${path}.matrix`, raw.matrix);
+
+ if (matrix.arch !== undefined) {
+ problems.add(`${path}.matrix.arch`, 'use the arch key instead of a matrix dimension named arch');
+ }
+
+ return {
+ image,
+ script,
+ needs: parseNeeds(problems, `${path}.needs`, raw.needs),
+ arch,
+ matrix,
+ requires: raw.requires === undefined ? defaults.requires : asStringList(problems, `${path}.requires`, raw.requires, { max: 64 }),
+ services: raw.services === undefined ? defaults.services : parseServices(problems, `${path}.services`, raw.services),
+ env: { ...defaults.env, ...asEnvMap(problems, `${path}.env`, raw.env) },
+ artifacts: parseArtifacts(problems, `${path}.artifacts`, raw.artifacts),
+ cache: raw.cache === undefined ? defaults.cache : parseCache(problems, `${path}.cache`, raw.cache),
+ allow_failure: raw.allow_failure === undefined
+ ? defaults.allow_failure
+ : asBoolean(problems, `${path}.allow_failure`, raw.allow_failure) ?? false,
+ timeout: raw.timeout === undefined ? defaults.timeout : asDuration(problems, `${path}.timeout`, raw.timeout),
+ max_attempts: raw.max_attempts === undefined
+ ? defaults.max_attempts
+ : asInteger(problems, `${path}.max_attempts`, raw.max_attempts, { min: 1, max: 10 }),
+ };
+}
+
+// A need is either a job name, or a mapping for the cases where the default
+// dimension matching is not what is wanted.
+function parseNeeds(problems, path, raw) {
+ if (raw === undefined) return [];
+ const list = Array.isArray(raw) ? raw : [raw];
+ const out = [];
+
+ list.forEach((item, i) => {
+ const itemPath = `${path}[${i}]`;
+ if (typeof item === 'string') {
+ out.push({ job: item, match: 'shared' });
+ return;
+ }
+ if (!isPlainObject(item)) {
+ problems.add(itemPath, `expected a job name or a mapping, got ${typeName(item)}`);
+ return;
+ }
+ checkUnknown(problems, itemPath, item, ['job', 'match']);
+ const job = asString(problems, `${itemPath}.job`, item.job);
+ const match = item.match === undefined ? 'shared' : asString(problems, `${itemPath}.match`, item.match);
+ if (match !== undefined && !['shared', 'all'].includes(match)) {
+ problems.add(`${itemPath}.match`, `expected shared or all, got ${JSON.stringify(match)}`);
+ return;
+ }
+ if (job !== undefined) out.push({ job, match: match ?? 'shared' });
+ });
+
+ return out;
+}
+
+function parseMatrix(problems, path, raw) {
+ if (raw === undefined) return {};
+ if (!isPlainObject(raw)) {
+ problems.add(path, `expected a mapping of dimension names to value lists, got ${typeName(raw)}`);
+ return {};
+ }
+
+ const out = {};
+ for (const [key, value] of Object.entries(raw)) {
+ const keyPath = `${path}.${key}`;
+ if (!MATRIX_KEY_PATTERN.test(key)) {
+ problems.add(keyPath, 'dimension name must be a valid environment variable name');
+ continue;
+ }
+ if (!Array.isArray(value)) {
+ problems.add(keyPath, `expected a list of values, got ${typeName(value)}`);
+ continue;
+ }
+ if (value.length === 0) {
+ problems.add(keyPath, 'must contain at least one value');
+ continue;
+ }
+
+ const values = [];
+ value.forEach((item, i) => {
+ if (typeof item !== 'string' && typeof item !== 'number' && typeof item !== 'boolean') {
+ problems.add(`${keyPath}[${i}]`, `expected a scalar value, got ${typeName(item)}`);
+ return;
+ }
+ const s = String(item);
+ if (!NAME_PATTERN.test(s)) {
+ problems.add(
+ `${keyPath}[${i}]`,
+ `${JSON.stringify(s)} may only contain letters, digits, underscore, dot and hyphen, ` +
+ 'because it becomes part of the job name'
+ );
+ return;
+ }
+ if (values.includes(s)) {
+ problems.add(`${keyPath}[${i}]`, `duplicate value ${JSON.stringify(s)}`);
+ return;
+ }
+ values.push(s);
+ });
+ out[key] = values;
+ }
+ return out;
+}
+
+function parseServices(problems, path, raw) {
+ if (raw === undefined) return [];
+ if (!Array.isArray(raw)) {
+ problems.add(path, `expected a list of services, got ${typeName(raw)}`);
+ return [];
+ }
+
+ const out = [];
+ const aliases = new Set();
+ raw.forEach((item, i) => {
+ const itemPath = `${path}[${i}]`;
+ const service = typeof item === 'string' ? { image: item } : item;
+ if (!isPlainObject(service)) {
+ problems.add(itemPath, `expected an image name or a mapping, got ${typeName(item)}`);
+ return;
+ }
+ checkUnknown(problems, itemPath, service, SERVICE_KEYS);
+
+ const image = asString(problems, `${itemPath}.image`, service.image);
+ if (image === undefined) return;
+
+ // Default alias is the image name without registry, path or tag, which
+ // is what a job would naturally use as a hostname.
+ const derived = image.split('/').pop().split(':')[0];
+ const alias = service.alias === undefined
+ ? derived
+ : asName(problems, `${itemPath}.alias`, service.alias, { max: 63 });
+ if (alias === undefined) return;
+ if (aliases.has(alias)) {
+ problems.add(`${itemPath}.alias`, `duplicate service alias ${JSON.stringify(alias)}`);
+ return;
+ }
+ aliases.add(alias);
+
+ out.push({
+ image,
+ alias,
+ env: asEnvMap(problems, `${itemPath}.env`, service.env),
+ entrypoint: asStringList(problems, `${itemPath}.entrypoint`, service.entrypoint),
+ command: asStringList(problems, `${itemPath}.command`, service.command),
+ });
+ });
+ return out;
+}
+
+function parseArtifacts(problems, path, raw) {
+ if (raw === undefined) return null;
+
+ // The common case is a bare list of paths.
+ const spec = Array.isArray(raw) ? { paths: raw } : raw;
+ if (!isPlainObject(spec)) {
+ problems.add(path, `expected a list of paths or a mapping, got ${typeName(raw)}`);
+ return null;
+ }
+ checkUnknown(problems, path, spec, ARTIFACT_KEYS);
+
+ const paths = asStringList(problems, `${path}.paths`, spec.paths, { max: 512 });
+ if (paths.length === 0) {
+ problems.add(`${path}.paths`, 'must list at least one path');
+ return null;
+ }
+ paths.forEach((p, i) => {
+ if (p.startsWith('/')) problems.add(`${path}.paths[${i}]`, 'must be relative to the workspace');
+ });
+
+ const when = spec.when === undefined ? 'on_success' : asString(problems, `${path}.when`, spec.when);
+ if (when !== undefined && !ARTIFACT_WHEN.includes(when)) {
+ problems.add(`${path}.when`, `expected one of ${ARTIFACT_WHEN.join(', ')}, got ${JSON.stringify(when)}`);
+ }
+
+ return {
+ paths,
+ when: ARTIFACT_WHEN.includes(when) ? when : 'on_success',
+ expire: spec.expire === undefined ? null : asDuration(problems, `${path}.expire`, spec.expire, { max: 365 * 24 * 3600 }),
+ };
+}
+
+function parseCache(problems, path, raw) {
+ if (raw === undefined || raw === null) return null;
+ const spec = Array.isArray(raw) ? { paths: raw } : raw;
+ if (!isPlainObject(spec)) {
+ problems.add(path, `expected a list of paths or a mapping, got ${typeName(raw)}`);
+ return null;
+ }
+ checkUnknown(problems, path, spec, CACHE_KEYS);
+
+ const paths = asStringList(problems, `${path}.paths`, spec.paths, { max: 512 });
+ if (paths.length === 0) {
+ problems.add(`${path}.paths`, 'must list at least one path');
+ return null;
+ }
+ paths.forEach((p, i) => {
+ if (p.startsWith('/')) problems.add(`${path}.paths[${i}]`, 'must be relative to the workspace');
+ });
+
+ return {
+ key: spec.key === undefined ? 'default' : asString(problems, `${path}.key`, spec.key, { max: 128 }),
+ paths,
+ };
+}
diff --git a/src/lib/pipeline/schema.js b/src/lib/pipeline/schema.js
@@ -0,0 +1,195 @@
+// src/lib/pipeline/schema.js - validation primitives for pipeline documents
+//
+// A .conductor.yml is written by hand and frequently by someone who is not
+// watching the conductor log, so validation collects every problem and
+// reports them together with a path, rather than failing on the first one.
+// Unknown keys are errors, not warnings: a silently ignored typo in a
+// pipeline is a bad afternoon.
+
+export class PipelineError extends Error {
+ constructor(errors, source) {
+ const list = errors.map((e) => ` ${e.path}: ${e.message}`).join('\n');
+ super(`invalid pipeline${source ? ` in ${source}` : ''}:\n${list}`);
+ this.name = 'PipelineError';
+ this.errors = errors;
+ this.source = source;
+ }
+}
+
+// Collects errors so that one pass reports everything wrong with a document.
+export class Problems {
+ constructor() {
+ this.items = [];
+ }
+
+ add(path, message) {
+ this.items.push({ path, message });
+ return undefined;
+ }
+
+ get length() {
+ return this.items.length;
+ }
+
+ throwIfAny(source) {
+ if (this.items.length > 0) throw new PipelineError(this.items, source);
+ }
+}
+
+export function isPlainObject(v) {
+ return v !== null && typeof v === 'object' && !Array.isArray(v);
+}
+
+export function checkUnknown(problems, path, value, allowed) {
+ for (const key of Object.keys(value)) {
+ if (!allowed.includes(key)) {
+ const hint = suggest(key, allowed);
+ problems.add(`${path}.${key}`, `unknown key${hint ? `, did you mean ${hint}` : ''}`);
+ }
+ }
+}
+
+// Cheap edit distance, only to improve the message on a near miss.
+function suggest(key, allowed) {
+ let best = null;
+ let bestScore = Infinity;
+ for (const candidate of allowed) {
+ const score = distance(key, candidate);
+ if (score < bestScore) {
+ bestScore = score;
+ best = candidate;
+ }
+ }
+ return bestScore <= 2 ? best : null;
+}
+
+function distance(a, b) {
+ const rows = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
+ for (let j = 0; j <= b.length; j += 1) rows[0][j] = j;
+ for (let i = 1; i <= a.length; i += 1) {
+ for (let j = 1; j <= b.length; j += 1) {
+ rows[i][j] = Math.min(
+ rows[i - 1][j] + 1,
+ rows[i][j - 1] + 1,
+ rows[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)
+ );
+ }
+ }
+ return rows[a.length][b.length];
+}
+
+export function asString(problems, path, value, { max = 4096 } = {}) {
+ if (typeof value !== 'string') return problems.add(path, `expected a string, got ${typeName(value)}`);
+ if (value.length === 0) return problems.add(path, 'must not be empty');
+ if (value.length > max) return problems.add(path, `must be at most ${max} characters`);
+ return value;
+}
+
+export function asBoolean(problems, path, value) {
+ if (typeof value !== 'boolean') return problems.add(path, `expected true or false, got ${typeName(value)}`);
+ return value;
+}
+
+export function asInteger(problems, path, value, { min = 1, max = Number.MAX_SAFE_INTEGER } = {}) {
+ if (typeof value !== 'number' || !Number.isInteger(value)) {
+ return problems.add(path, `expected an integer, got ${typeName(value)}`);
+ }
+ if (value < min || value > max) return problems.add(path, `must be between ${min} and ${max}`);
+ return value;
+}
+
+// Accepts a single string as a one element list, which is how people
+// naturally write a single value.
+export function asStringList(problems, path, value, opts = {}) {
+ if (value === undefined) return [];
+ const list = Array.isArray(value) ? value : [value];
+ const out = [];
+ list.forEach((item, i) => {
+ const s = asString(problems, `${path}[${i}]`, item, opts);
+ if (s !== undefined) out.push(s);
+ });
+ return out;
+}
+
+// Scalar values only. YAML readily produces numbers and booleans here, and
+// silently stringifying them hides mistakes, so they are converted but
+// anything structured is rejected.
+export function asEnvMap(problems, path, value) {
+ if (value === undefined) return {};
+ if (!isPlainObject(value)) {
+ problems.add(path, `expected a mapping of names to values, got ${typeName(value)}`);
+ return {};
+ }
+ const out = {};
+ for (const [key, raw] of Object.entries(value)) {
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) {
+ problems.add(`${path}.${key}`, 'is not a valid environment variable name');
+ continue;
+ }
+ if (raw === null) {
+ out[key] = '';
+ } else if (typeof raw === 'string' || typeof raw === 'number' || typeof raw === 'boolean') {
+ out[key] = String(raw);
+ } else {
+ problems.add(`${path}.${key}`, `expected a scalar value, got ${typeName(raw)}`);
+ }
+ }
+ return out;
+}
+
+export const MAX_DURATION = 7 * 24 * 60 * 60;
+
+// Accepts 90, '90s', '30m', '1h', or a composite such as '1h30m'.
+export function asDuration(problems, path, value, { max = MAX_DURATION } = {}) {
+ let seconds;
+
+ if (typeof value === 'number') {
+ if (!Number.isInteger(value)) return problems.add(path, 'expected a whole number of seconds');
+ seconds = value;
+ } else if (typeof value === 'string') {
+ const text = value.trim();
+ if (/^\d+$/.test(text)) {
+ seconds = parseInt(text, 10);
+ } else {
+ const matches = [...text.matchAll(/(\d+)([smh])/g)];
+ const consumed = matches.reduce((n, m) => n + m[0].length, 0);
+ if (matches.length === 0 || consumed !== text.length) {
+ return problems.add(path, `expected a duration such as 90s, 30m or 1h30m, got ${JSON.stringify(value)}`);
+ }
+ const unit = { s: 1, m: 60, h: 3600 };
+ seconds = matches.reduce((n, m) => n + parseInt(m[1], 10) * unit[m[2]], 0);
+ }
+ } else {
+ return problems.add(path, `expected a duration, got ${typeName(value)}`);
+ }
+
+ if (seconds < 1) return problems.add(path, 'must be at least 1 second');
+ if (seconds > max) return problems.add(path, `must be at most ${max} seconds`);
+ return seconds;
+}
+
+export function typeName(v) {
+ if (v === null) return 'null';
+ if (Array.isArray(v)) return 'a list';
+ if (v === undefined) return 'nothing';
+ if (typeof v === 'object') return 'a mapping';
+ return `a ${typeof v}`;
+}
+
+// Job and matrix value names appear in generated job names, storage keys and
+// environment variables, so the character set is deliberately narrow.
+export const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
+export const MATRIX_KEY_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+export function asName(problems, path, value, { max = 100 } = {}) {
+ const s = asString(problems, path, value, { max });
+ if (s === undefined) return undefined;
+ if (!NAME_PATTERN.test(s)) {
+ return problems.add(
+ path,
+ `${JSON.stringify(s)} must start with a letter or digit and contain only ` +
+ 'letters, digits, underscore, dot and hyphen'
+ );
+ }
+ return s;
+}
diff --git a/test/dag.test.js b/test/dag.test.js
@@ -0,0 +1,110 @@
+// test/dag.test.js - dependency graph traversal
+//
+// These cover the two scheduling defects carried over from the prototype:
+// serialized dispatch, and a failure cascade that only skipped direct
+// dependents.
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { topologicalOrder, depths, transitiveDependents, runnable, CycleError } from '../src/lib/pipeline/dag.js';
+
+// a -> b -> d, a -> c -> d, plus an unrelated island.
+const GRAPH = [
+ { name: 'a', needs: [] },
+ { name: 'b', needs: ['a'] },
+ { name: 'c', needs: ['a'] },
+ { name: 'd', needs: ['b', 'c'] },
+ { name: 'island', needs: [] },
+];
+
+function states(overrides = {}) {
+ const map = new Map(GRAPH.map((j) => [j.name, 'queued']));
+ for (const [k, v] of Object.entries(overrides)) map.set(k, v);
+ return map;
+}
+
+test('topological order places dependencies first', () => {
+ const order = topologicalOrder(GRAPH).map((j) => j.name);
+ assert.ok(order.indexOf('a') < order.indexOf('b'));
+ assert.ok(order.indexOf('a') < order.indexOf('c'));
+ assert.ok(order.indexOf('b') < order.indexOf('d'));
+ assert.ok(order.indexOf('c') < order.indexOf('d'));
+});
+
+test('topological order is deterministic', () => {
+ const first = topologicalOrder(GRAPH).map((j) => j.name);
+ const shuffled = [...GRAPH].reverse();
+ assert.deepEqual(topologicalOrder(shuffled).map((j) => j.name), first);
+});
+
+test('a cycle is detected and named', () => {
+ const cyclic = [
+ { name: 'x', needs: ['z'] },
+ { name: 'y', needs: ['x'] },
+ { name: 'z', needs: ['y'] },
+ ];
+ assert.throws(() => topologicalOrder(cyclic), CycleError);
+ try {
+ topologicalOrder(cyclic);
+ } catch (e) {
+ // The reported path returns to where it started.
+ assert.equal(e.cycle[0], e.cycle[e.cycle.length - 1]);
+ }
+});
+
+test('depth reflects the longest path, not insertion order', () => {
+ const d = depths(GRAPH);
+ assert.equal(d.get('a'), 0);
+ assert.equal(d.get('island'), 0);
+ assert.equal(d.get('b'), 1);
+ assert.equal(d.get('d'), 2);
+});
+
+test('independent jobs are runnable at the same time', () => {
+ // The prototype could only ever return one job here.
+ const ready = runnable(GRAPH, states()).map((j) => j.name);
+ assert.deepEqual(ready.sort(), ['a', 'island']);
+});
+
+test('a job becomes runnable only once every dependency succeeds', () => {
+ assert.deepEqual(
+ runnable(GRAPH, states({ a: 'success', b: 'success' })).map((j) => j.name).sort(),
+ ['c', 'island']
+ );
+ assert.deepEqual(
+ runnable(GRAPH, states({ a: 'success', b: 'success', c: 'success', island: 'success' })).map((j) => j.name),
+ ['d']
+ );
+});
+
+test('a running dependency does not release its dependents', () => {
+ assert.equal(runnable(GRAPH, states({ a: 'running' })).some((j) => j.name === 'b'), false);
+});
+
+test('failure skips transitive dependents, not just direct ones', () => {
+ // This is the prototype bug: it would have skipped only b and c, leaving
+ // d queued and dispatchable with its inputs missing.
+ const skipped = transitiveDependents(GRAPH, ['a']);
+ assert.deepEqual([...skipped].sort(), ['b', 'c', 'd']);
+ assert.equal(skipped.has('island'), false);
+});
+
+test('a skipped dependency never releases a dependent', () => {
+ assert.equal(runnable(GRAPH, states({ a: 'failed' })).some((j) => j.name === 'b'), false);
+ assert.equal(runnable(GRAPH, states({ a: 'skipped' })).some((j) => j.name === 'b'), false);
+});
+
+test('allowed failures can satisfy a dependency when asked to', () => {
+ const ready = runnable(GRAPH, states({ a: 'failed' }), { satisfied: ['success', 'failed'] });
+ assert.deepEqual(ready.map((j) => j.name).sort(), ['b', 'c', 'island']);
+});
+
+test('transitive dependents of a leaf is empty', () => {
+ assert.equal(transitiveDependents(GRAPH, ['d']).size, 0);
+});
+
+test('graph helpers tolerate dependencies outside the set', () => {
+ const partial = [{ name: 'only', needs: ['absent'] }];
+ assert.deepEqual(topologicalOrder(partial).map((j) => j.name), ['only']);
+ assert.equal(depths(partial).get('only'), 0);
+});
diff --git a/test/pipeline.test.js b/test/pipeline.test.js
@@ -0,0 +1,420 @@
+// test/pipeline.test.js - parsing, validation and expansion of .conductor.yml
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { compilePipeline, parsePipeline, PipelineError } from '../src/lib/pipeline/index.js';
+
+const MINIMAL = `
+version: 1
+jobs:
+ build:
+ image: alpine
+ script: [make]
+`;
+
+function errorPaths(fn) {
+ try {
+ fn();
+ } catch (e) {
+ assert.ok(e instanceof PipelineError, `expected PipelineError, got ${e}`);
+ return e.errors.map((x) => x.path);
+ }
+ throw new Error('expected the pipeline to be rejected');
+}
+
+function byName(pipeline) {
+ return Object.fromEntries(pipeline.jobs.map((j) => [j.name, j]));
+}
+
+test('a minimal pipeline compiles', () => {
+ const p = compilePipeline(MINIMAL);
+ assert.equal(p.jobs.length, 1);
+ assert.equal(p.jobs[0].name, 'build');
+ assert.deepEqual(p.jobs[0].script, ['make']);
+ assert.equal(p.jobs[0].depth, 0);
+});
+
+test('version is required and pinned', () => {
+ assert.deepEqual(errorPaths(() => compilePipeline('jobs:\n a:\n image: x\n script: [y]\n')), ['version']);
+ assert.deepEqual(errorPaths(() => compilePipeline('version: 2\njobs:\n a:\n image: x\n script: [y]\n')), ['version']);
+});
+
+test('empty and malformed documents are rejected clearly', () => {
+ assert.throws(() => compilePipeline(''), /file is empty/);
+ assert.throws(() => compilePipeline('- a\n- b\n'), /expected a mapping at the top level/);
+ assert.throws(() => compilePipeline('version: 1\njobs: {\n'), /not valid YAML/);
+});
+
+test('image and script are required', () => {
+ const paths = errorPaths(() => compilePipeline('version: 1\njobs:\n a: {}\n'));
+ assert.deepEqual(paths.sort(), ['jobs.a.image', 'jobs.a.script']);
+});
+
+test('image may come from defaults', () => {
+ const p = compilePipeline('version: 1\ndefaults:\n image: alpine\njobs:\n a:\n script: [x]\n');
+ assert.equal(p.jobs[0].image, 'alpine');
+});
+
+test('unknown keys are rejected, with a suggestion when close', () => {
+ try {
+ compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n scripts: [z]\n');
+ throw new Error('expected rejection');
+ } catch (e) {
+ assert.match(e.message, /jobs\.a\.scripts: unknown key, did you mean script/);
+ }
+});
+
+test('all problems are reported at once', () => {
+ const paths = errorPaths(() => compilePipeline(`
+version: 1
+jobs:
+ a:
+ script: [x]
+ b:
+ image: y
+`));
+ assert.ok(paths.includes('jobs.a.image'));
+ assert.ok(paths.includes('jobs.b.script'));
+});
+
+test('durations accept seconds, units and composites', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ a: { image: x, script: [y], timeout: 90 }
+ b: { image: x, script: [y], timeout: 30m }
+ c: { image: x, script: [y], timeout: 1h30m }
+`);
+ const jobs = byName(p);
+ assert.equal(jobs.a.timeout, 90);
+ assert.equal(jobs.b.timeout, 1800);
+ assert.equal(jobs.c.timeout, 5400);
+});
+
+test('a malformed duration is rejected', () => {
+ assert.deepEqual(
+ errorPaths(() => compilePipeline('version: 1\njobs:\n a: { image: x, script: [y], timeout: soon }\n')),
+ ['jobs.a.timeout']
+ );
+});
+
+test('arch expands into one job per architecture', () => {
+ const p = compilePipeline('version: 1\njobs:\n b:\n image: x\n script: [y]\n arch: [x86_64, aarch64]\n');
+ assert.deepEqual(p.jobs.map((j) => j.name).sort(), ['b:arch=aarch64', 'b:arch=x86_64']);
+ assert.equal(byName(p)['b:arch=x86_64'].env.ARCH, 'x86_64');
+});
+
+test('a matrix expands to the cartesian product', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ p:
+ image: x
+ script: [y]
+ matrix:
+ pkg: [musl, busybox]
+ mode: [debug, release]
+`);
+ assert.equal(p.jobs.length, 4);
+ // Dimensions keep their declaration order, so names are predictable.
+ assert.deepEqual(p.jobs.map((j) => j.name).sort(), [
+ 'p:pkg=busybox,mode=debug',
+ 'p:pkg=busybox,mode=release',
+ 'p:pkg=musl,mode=debug',
+ 'p:pkg=musl,mode=release',
+ ]);
+ const one = byName(p)['p:pkg=musl,mode=debug'];
+ assert.equal(one.env.MATRIX_PKG, 'musl');
+ assert.equal(one.env.MATRIX_MODE, 'debug');
+});
+
+test('needs match on shared dimensions rather than fanning out', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ build:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ package:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ matrix:
+ pkg: [musl]
+ needs: [build]
+`);
+ const jobs = byName(p);
+ assert.deepEqual(jobs['package:arch=x86_64,pkg=musl'].needs, ['build:arch=x86_64']);
+ assert.deepEqual(jobs['package:arch=aarch64,pkg=musl'].needs, ['build:arch=aarch64']);
+});
+
+test('a job without the shared dimension depends on every instance', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ build:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ publish:
+ image: x
+ script: [y]
+ needs: [build]
+`);
+ assert.deepEqual(byName(p).publish.needs, ['build:arch=aarch64', 'build:arch=x86_64']);
+});
+
+test('match all opts out of dimension matching', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ build:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ check:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ needs:
+ - job: build
+ match: all
+`);
+ assert.deepEqual(byName(p)['check:arch=x86_64'].needs, ['build:arch=aarch64', 'build:arch=x86_64']);
+});
+
+test('needs may name one concrete instance', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ build:
+ image: x
+ script: [y]
+ arch: [x86_64, aarch64]
+ docs:
+ image: x
+ script: [y]
+ needs: ['build:arch=x86_64']
+`);
+ assert.deepEqual(byName(p).docs.needs, ['build:arch=x86_64']);
+});
+
+test('an unsatisfiable dimension match is reported, not silently dropped', () => {
+ try {
+ compilePipeline(`
+version: 1
+jobs:
+ build:
+ image: x
+ script: [y]
+ arch: [x86_64]
+ package:
+ image: x
+ script: [y]
+ arch: [riscv64]
+ needs: [build]
+`);
+ throw new Error('expected rejection');
+ } catch (e) {
+ assert.match(e.message, /no instance of "build" matches package:arch=riscv64 on arch/);
+ assert.match(e.message, /match: all/);
+ }
+});
+
+test('needs on an unknown job lists what is defined', () => {
+ try {
+ compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n needs: [nope]\n');
+ throw new Error('expected rejection');
+ } catch (e) {
+ assert.match(e.message, /refers to unknown job "nope"/);
+ }
+});
+
+test('self dependency is rejected', () => {
+ assert.throws(
+ () => compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n needs: [a]\n'),
+ /cannot depend on itself/
+ );
+});
+
+test('a dependency cycle names the cycle', () => {
+ assert.throws(() => compilePipeline(`
+version: 1
+jobs:
+ a: { image: x, script: [s], needs: [c] }
+ b: { image: x, script: [s], needs: [a] }
+ c: { image: x, script: [s], needs: [b] }
+`), /dependency cycle: (a -> c -> b -> a|b -> a -> c -> b|c -> b -> a -> c)/);
+});
+
+test('interpolation substitutes arch and matrix values', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ b:
+ image: 'builder:\${{ arch }}'
+ arch: [x86_64]
+ matrix:
+ pkg: [musl]
+ script: ['build \${{ matrix.pkg }} for \${{ arch }}']
+ env:
+ TAG: '\${{ matrix.pkg }}-\${{ arch }}'
+`);
+ const job = p.jobs[0];
+ assert.equal(job.image, 'builder:x86_64');
+ assert.deepEqual(job.script, ['build musl for x86_64']);
+ assert.equal(job.env.TAG, 'musl-x86_64');
+});
+
+test('interpolating an undefined dimension is an error', () => {
+ try {
+ compilePipeline('version: 1\njobs:\n a:\n image: x\n script: ["\${{ matrix.nope }}"]\n');
+ throw new Error('expected rejection');
+ } catch (e) {
+ assert.match(e.message, /matrix\.nope \}\} which the job does not define/);
+ }
+});
+
+test('referring to arch without declaring one is an error', () => {
+ assert.throws(
+ () => compilePipeline('version: 1\njobs:\n a:\n image: "x:\${{ arch }}"\n script: [y]\n'),
+ /the job declares no arch/
+ );
+});
+
+test('matrix values must be safe for use in a job name', () => {
+ assert.deepEqual(
+ errorPaths(() => compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n matrix:\n k: ["has space"]\n')),
+ ['jobs.a.matrix.k[0]']
+ );
+});
+
+test('arch may not also be a matrix dimension', () => {
+ assert.deepEqual(
+ errorPaths(() => compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n matrix:\n arch: [x86_64]\n')),
+ ['jobs.a.matrix.arch']
+ );
+});
+
+test('services get a derived alias and reject duplicates', () => {
+ const p = compilePipeline(`
+version: 1
+jobs:
+ a:
+ image: x
+ script: [y]
+ services:
+ - docker:dind
+ - image: registry.example.com/team/postgres:16
+ alias: db
+`);
+ assert.deepEqual(p.jobs[0].services.map((s) => s.alias), ['docker', 'db']);
+
+ assert.throws(() => compilePipeline(`
+version: 1
+jobs:
+ a:
+ image: x
+ script: [y]
+ services: [docker:dind, docker:24-dind]
+`), /duplicate service alias/);
+});
+
+test('artifacts accept a bare list and reject absolute paths', () => {
+ const p = compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n artifacts: [dist/**]\n');
+ assert.deepEqual(p.jobs[0].artifacts.paths, ['dist/**']);
+ assert.equal(p.jobs[0].artifacts.when, 'on_success');
+
+ assert.deepEqual(
+ errorPaths(() => compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n artifacts: [/etc/passwd]\n')),
+ ['jobs.a.artifacts.paths[0]']
+ );
+});
+
+test('artifact when is constrained', () => {
+ assert.deepEqual(
+ errorPaths(() => compilePipeline('version: 1\njobs:\n a:\n image: x\n script: [y]\n artifacts:\n paths: [d]\n when: maybe\n')),
+ ['jobs.a.artifacts.when']
+ );
+});
+
+test('defaults are inherited and overridden per job', () => {
+ const p = compilePipeline(`
+version: 1
+defaults:
+ image: base
+ timeout: 10m
+ env:
+ SHARED: yes
+ requires: [docker]
+jobs:
+ a:
+ script: [x]
+ b:
+ image: custom
+ timeout: 1h
+ script: [x]
+ env:
+ OWN: no
+`);
+ const jobs = byName(p);
+ assert.equal(jobs.a.image, 'base');
+ assert.equal(jobs.a.timeout, 600);
+ assert.deepEqual(jobs.a.requires, ['docker']);
+ assert.equal(jobs.b.image, 'custom');
+ assert.equal(jobs.b.timeout, 3600);
+ assert.equal(jobs.b.env.SHARED, 'yes');
+ assert.equal(jobs.b.env.OWN, 'no');
+});
+
+test('env values must be scalars with valid names', () => {
+ const paths = errorPaths(() => compilePipeline(`
+version: 1
+jobs:
+ a:
+ image: x
+ script: [y]
+ env:
+ 'bad name': 1
+ GOOD: { nested: true }
+`));
+ assert.deepEqual(paths.sort(), ['jobs.a.env.GOOD', 'jobs.a.env.bad name']);
+});
+
+test('parsePipeline normalizes without expanding', () => {
+ const doc = parsePipeline(MINIMAL);
+ assert.equal(doc.version, 1);
+ assert.deepEqual(Object.keys(doc.jobs), ['build']);
+ assert.deepEqual(doc.jobs.build.needs, []);
+});
+
+test('a realistic distribution pipeline produces the expected graph', () => {
+ const p = compilePipeline(`
+version: 1
+defaults:
+ image: debian:bookworm-slim
+jobs:
+ lint:
+ script: [make lint]
+ build:
+ arch: [x86_64, aarch64]
+ script: ['./mk/build.sh $ARCH']
+ package:
+ needs: [build]
+ arch: [x86_64, aarch64]
+ matrix:
+ pkg: [musl, busybox]
+ requires: [sign-key]
+ script: ['./mk/pkg.sh $MATRIX_PKG $ARCH']
+ publish:
+ needs: [package, lint]
+ script: [./mk/publish.sh]
+`);
+
+ assert.equal(p.jobs.length, 1 + 2 + 4 + 1);
+ const jobs = byName(p);
+ assert.equal(jobs.publish.needs.length, 5);
+ assert.equal(jobs.publish.depth, 2);
+ assert.deepEqual(jobs['package:arch=aarch64,pkg=musl'].requires, ['sign-key']);
+});
