parse.js - conductor - CI task system

parse.js (15926B)
      1 // src/lib/pipeline/parse.js - reads and validates a .conductor.yml
      2 //
      3 // Produces a normalized document: every task has every field filled in from
      4 // defaults, so later stages never have to ask whether something was set.
      5 // Nothing here expands matrices or resolves dependencies; see expand.js.
      6 
      7 import YAML from 'yaml';
      8 import {
      9   PipelineError,
     10   Problems,
     11   isPlainObject,
     12   checkUnknown,
     13   asString,
     14   asName,
     15   asBoolean,
     16   asInteger,
     17   asStringList,
     18   asEnvMap,
     19   asDuration,
     20   typeName,
     21   MATRIX_KEY_PATTERN,
     22   NAME_PATTERN,
     23 } from './schema.js';
     24 
     25 const TOP_KEYS = ['version', 'workdir', 'defaults', 'tasks'];
     26 
     27 // Where the tree is put inside a task container, and the directory every
     28 // script starts in. Unset means the project's setting stands, and failing
     29 // that the server default.
     30 export const DEFAULT_WORKDIR = '/work';
     31 
     32 // An absolute path of ordinary directory names. The tree is delivered by
     33 // unpacking an archive at the container's root, so the path is built from
     34 // the archive's own entries: anything that could climb out of it, or that
     35 // is not a plain name, is refused rather than sanitised.
     36 export function parseWorkdir(problems, path, raw) {
     37   const value = asString(problems, path, raw);
     38   if (typeof value !== 'string') return null;
     39 
     40   if (!value.startsWith('/')) {
     41     problems.add(path, `must be an absolute path, got ${JSON.stringify(value)}`);
     42     return null;
     43   }
     44 
     45   const segments = value.split('/').filter(Boolean);
     46   if (segments.length === 0) {
     47     problems.add(path, 'must name a directory, not the container root');
     48     return null;
     49   }
     50 
     51   for (const segment of segments) {
     52     if (segment === '.' || segment === '..') {
     53       problems.add(path, `must not contain ${JSON.stringify(segment)}`);
     54       return null;
     55     }
     56     if (!/^[A-Za-z0-9._-]+$/.test(segment)) {
     57       problems.add(path, `${JSON.stringify(segment)} is not a usable directory name`);
     58       return null;
     59     }
     60   }
     61 
     62   return `/${segments.join('/')}`;
     63 }
     64 
     65 const JOB_KEYS = [
     66   'image', 'script', 'needs', 'after', 'arch', 'matrix', 'requires', 'services',
     67   'env', 'artifacts', 'allow_failure', 'timeout', 'max_attempts',
     68   'only',
     69 ];
     70 
     71 // Fields a task may inherit from defaults.
     72 const DEFAULT_KEYS = [
     73   'image', 'arch', 'requires', 'env', 'services',
     74   'allow_failure', 'timeout', 'max_attempts',
     75 ];
     76 
     77 const ONLY_KEYS = ['refs'];
     78 const ARTIFACT_KEYS = ['paths', 'expire', 'when'];
     79 const ARTIFACT_WHEN = ['on_success', 'on_failure', 'always'];
     80 
     81 const SERVICE_KEYS = ['image', 'alias', 'env', 'entrypoint', 'command'];
     82 
     83 export const SUPPORTED_VERSION = 1;
     84 
     85 export function parsePipeline(text, options = {}) {
     86   const source = options.source || '.conductor.yml';
     87   const problems = new Problems();
     88 
     89   let doc;
     90   try {
     91     doc = YAML.parse(text, { prettyErrors: true });
     92   } catch (e) {
     93     throw new PipelineError([{ path: '', message: `not valid YAML: ${e.message}` }], source);
     94   }
     95 
     96   if (doc === null || doc === undefined) {
     97     throw new PipelineError([{ path: '', message: 'file is empty' }], source);
     98   }
     99   if (!isPlainObject(doc)) {
    100     throw new PipelineError([{ path: '', message: `expected a mapping at the top level, got ${typeName(doc)}` }], source);
    101   }
    102 
    103   // The key was called jobs before a job became the thing that contains
    104   // these. Named outright rather than left to checkUnknown, whose nearest
    105   // match would be a guess at a word that used to be correct.
    106   if (doc.jobs !== undefined && doc.tasks === undefined) {
    107     throw new PipelineError(
    108       [{ path: 'jobs', message: 'was renamed to tasks; a job is now the pipeline run that these tasks belong to' }],
    109       source
    110     );
    111   }
    112 
    113   // Named outright; checkUnknown would only offer a nearest match.
    114   if (doc.visibility !== undefined) {
    115     throw new PipelineError(
    116       [{ path: 'visibility', message: 'is no longer set by the pipeline; it is a project setting' }],
    117       source
    118     );
    119   }
    120 
    121   checkUnknown(problems, '', doc, TOP_KEYS);
    122 
    123   if (doc.version === undefined) {
    124     problems.add('version', `is required; this conductor understands version ${SUPPORTED_VERSION}`);
    125   } else if (doc.version !== SUPPORTED_VERSION) {
    126     problems.add('version', `unsupported version ${JSON.stringify(doc.version)}, expected ${SUPPORTED_VERSION}`);
    127   }
    128 
    129   const defaults = parseDefaults(problems, doc.defaults);
    130 
    131   if (doc.tasks === undefined) {
    132     problems.add('tasks', 'is required');
    133   } else if (!isPlainObject(doc.tasks)) {
    134     problems.add('tasks', `expected a mapping of task names, got ${typeName(doc.tasks)}`);
    135   } else if (Object.keys(doc.tasks).length === 0) {
    136     problems.add('tasks', 'must define at least one task');
    137   }
    138 
    139   const tasks = {};
    140   if (isPlainObject(doc.tasks)) {
    141     for (const [name, raw] of Object.entries(doc.tasks)) {
    142       const path = `tasks.${name}`;
    143       if (!NAME_PATTERN.test(name) || name.length > 100) {
    144         problems.add(path, 'task name must start with a letter or digit and contain only letters, digits, underscore, dot and hyphen');
    145         continue;
    146       }
    147       if (!isPlainObject(raw)) {
    148         problems.add(path, `expected a mapping, got ${typeName(raw)}`);
    149         continue;
    150       }
    151       tasks[name] = parseTask(problems, path, raw, defaults);
    152     }
    153   }
    154 
    155   const workdir = doc.workdir === undefined ? null : parseWorkdir(problems, 'workdir', doc.workdir);
    156 
    157   problems.throwIfAny(source);
    158   return { version: SUPPORTED_VERSION, workdir, defaults, tasks, source };
    159 }
    160 
    161 function parseDefaults(problems, raw) {
    162   const empty = {
    163     image: undefined,
    164     arch: undefined,
    165     requires: [],
    166     env: {},
    167     services: [],
    168     allow_failure: false,
    169     timeout: undefined,
    170     max_attempts: undefined,
    171   };
    172   if (raw === undefined) return empty;
    173   if (!isPlainObject(raw)) {
    174     problems.add('defaults', `expected a mapping, got ${typeName(raw)}`);
    175     return empty;
    176   }
    177 
    178   checkUnknown(problems, 'defaults', raw, DEFAULT_KEYS);
    179 
    180   return {
    181     image: raw.image === undefined ? undefined : asString(problems, 'defaults.image', raw.image),
    182     arch: raw.arch === undefined ? undefined : parseArch(problems, 'defaults.arch', raw.arch),
    183     requires: asStringList(problems, 'defaults.requires', raw.requires),
    184     env: asEnvMap(problems, 'defaults.env', raw.env),
    185     services: parseServices(problems, 'defaults.services', raw.services),
    186     allow_failure: raw.allow_failure === undefined ? false : asBoolean(problems, 'defaults.allow_failure', raw.allow_failure) ?? false,
    187     timeout: raw.timeout === undefined ? undefined : asDuration(problems, 'defaults.timeout', raw.timeout),
    188     max_attempts: raw.max_attempts === undefined ? undefined : asInteger(problems, 'defaults.max_attempts', raw.max_attempts, { min: 1, max: 10 }),
    189   };
    190 }
    191 
    192 function parseArch(problems, path, raw) {
    193   const list = asStringList(problems, path, raw, { max: 32 });
    194   const out = [];
    195   list.forEach((value, i) => {
    196     if (!NAME_PATTERN.test(value)) {
    197       problems.add(`${path}[${i}]`, `${JSON.stringify(value)} is not a valid architecture name`);
    198       return;
    199     }
    200     if (out.includes(value)) {
    201       problems.add(`${path}[${i}]`, `duplicate architecture ${JSON.stringify(value)}`);
    202       return;
    203     }
    204     out.push(value);
    205   });
    206   return out;
    207 }
    208 
    209 function parseTask(problems, path, raw, defaults) {
    210   checkUnknown(problems, path, raw, JOB_KEYS);
    211 
    212   const image = raw.image === undefined ? defaults.image : asString(problems, `${path}.image`, raw.image);
    213   if (image === undefined) {
    214     problems.add(`${path}.image`, 'is required; set it on the task or under defaults');
    215   }
    216 
    217   if (raw.script === undefined) {
    218     problems.add(`${path}.script`, 'is required');
    219   }
    220   const script = asStringList(problems, `${path}.script`, raw.script, { max: 65536 });
    221   if (raw.script !== undefined && script.length === 0) {
    222     problems.add(`${path}.script`, 'must contain at least one command');
    223   }
    224 
    225   const arch = raw.arch === undefined ? (defaults.arch ?? []) : parseArch(problems, `${path}.arch`, raw.arch);
    226   const matrix = parseMatrix(problems, `${path}.matrix`, raw.matrix);
    227 
    228   if (matrix.arch !== undefined) {
    229     problems.add(`${path}.matrix.arch`, 'use the arch key instead of a matrix dimension named arch');
    230   }
    231 
    232   return {
    233     image,
    234     script,
    235     needs: parseDeps(problems, `${path}.needs`, raw.needs),
    236     after: parseDeps(problems, `${path}.after`, raw.after),
    237     arch,
    238     matrix,
    239     requires: raw.requires === undefined ? defaults.requires : asStringList(problems, `${path}.requires`, raw.requires, { max: 64 }),
    240     services: raw.services === undefined ? defaults.services : parseServices(problems, `${path}.services`, raw.services),
    241     env: { ...defaults.env, ...asEnvMap(problems, `${path}.env`, raw.env) },
    242     artifacts: parseArtifacts(problems, `${path}.artifacts`, raw.artifacts),
    243     allow_failure: raw.allow_failure === undefined
    244       ? defaults.allow_failure
    245       : asBoolean(problems, `${path}.allow_failure`, raw.allow_failure) ?? false,
    246     timeout: raw.timeout === undefined ? defaults.timeout : asDuration(problems, `${path}.timeout`, raw.timeout),
    247     max_attempts: raw.max_attempts === undefined
    248       ? defaults.max_attempts
    249       : asInteger(problems, `${path}.max_attempts`, raw.max_attempts, { min: 1, max: 10 }),
    250     only: parseOnly(problems, `${path}.only`, raw.only),
    251   };
    252 }
    253 
    254 // Restricts a task to certain refs. Deliberately not inheritable from
    255 // defaults: a rule that silently applied to every task is the kind of thing
    256 // that stops a pipeline running at all and takes an afternoon to find.
    257 //
    258 //   only:
    259 //     refs: [refs/heads/main, 'refs/tags/v*']
    260 //
    261 // Patterns match the whole ref, so refs/heads/main rather than main, with
    262 // * standing for any run of characters. Absent means the task always runs.
    263 function parseOnly(problems, path, raw) {
    264   if (raw === undefined || raw === null) return null;
    265   if (!isPlainObject(raw)) {
    266     problems.add(path, 'must be a mapping, for example: only: { refs: [refs/heads/main] }');
    267     return null;
    268   }
    269 
    270   for (const key of Object.keys(raw)) {
    271     if (!ONLY_KEYS.includes(key)) {
    272       problems.add(`${path}.${key}`, `unknown key, expected one of: ${ONLY_KEYS.join(', ')}`);
    273     }
    274   }
    275 
    276   if (raw.refs === undefined) {
    277     problems.add(path, 'needs a refs list, otherwise it restricts nothing');
    278     return null;
    279   }
    280 
    281   const refs = asStringList(problems, `${path}.refs`, raw.refs, { max: 64 });
    282   if (refs.length === 0) {
    283     problems.add(`${path}.refs`, 'must list at least one ref pattern');
    284     return null;
    285   }
    286 
    287   return { refs };
    288 }
    289 
    290 // A dependency is either a task name, or a mapping for the cases where the
    291 // default dimension matching is not what is wanted. `needs` and `after` take
    292 // the same forms and differ only in what the scheduler makes of the edge.
    293 function parseDeps(problems, path, raw) {
    294   if (raw === undefined) return [];
    295   const list = Array.isArray(raw) ? raw : [raw];
    296   const out = [];
    297 
    298   list.forEach((item, i) => {
    299     const itemPath = `${path}[${i}]`;
    300     if (typeof item === 'string') {
    301       out.push({ task: item, match: 'shared' });
    302       return;
    303     }
    304     if (!isPlainObject(item)) {
    305       problems.add(itemPath, `expected a task name or a mapping, got ${typeName(item)}`);
    306       return;
    307     }
    308     checkUnknown(problems, itemPath, item, ['task', 'match']);
    309     const task = asString(problems, `${itemPath}.task`, item.task);
    310     const match = item.match === undefined ? 'shared' : asString(problems, `${itemPath}.match`, item.match);
    311     if (match !== undefined && !['shared', 'all'].includes(match)) {
    312       problems.add(`${itemPath}.match`, `expected shared or all, got ${JSON.stringify(match)}`);
    313       return;
    314     }
    315     if (task !== undefined) out.push({ task, match: match ?? 'shared' });
    316   });
    317 
    318   return out;
    319 }
    320 
    321 function parseMatrix(problems, path, raw) {
    322   if (raw === undefined) return {};
    323   if (!isPlainObject(raw)) {
    324     problems.add(path, `expected a mapping of dimension names to value lists, got ${typeName(raw)}`);
    325     return {};
    326   }
    327 
    328   const out = {};
    329   for (const [key, value] of Object.entries(raw)) {
    330     const keyPath = `${path}.${key}`;
    331     if (!MATRIX_KEY_PATTERN.test(key)) {
    332       problems.add(keyPath, 'dimension name must be a valid environment variable name');
    333       continue;
    334     }
    335     if (!Array.isArray(value)) {
    336       problems.add(keyPath, `expected a list of values, got ${typeName(value)}`);
    337       continue;
    338     }
    339     if (value.length === 0) {
    340       problems.add(keyPath, 'must contain at least one value');
    341       continue;
    342     }
    343 
    344     const values = [];
    345     value.forEach((item, i) => {
    346       if (typeof item !== 'string' && typeof item !== 'number' && typeof item !== 'boolean') {
    347         problems.add(`${keyPath}[${i}]`, `expected a scalar value, got ${typeName(item)}`);
    348         return;
    349       }
    350       const s = String(item);
    351       if (!NAME_PATTERN.test(s)) {
    352         problems.add(
    353           `${keyPath}[${i}]`,
    354           `${JSON.stringify(s)} may only contain letters, digits, underscore, dot and hyphen, ` +
    355           'because it becomes part of the task name'
    356         );
    357         return;
    358       }
    359       if (values.includes(s)) {
    360         problems.add(`${keyPath}[${i}]`, `duplicate value ${JSON.stringify(s)}`);
    361         return;
    362       }
    363       values.push(s);
    364     });
    365     out[key] = values;
    366   }
    367   return out;
    368 }
    369 
    370 function parseServices(problems, path, raw) {
    371   if (raw === undefined) return [];
    372   if (!Array.isArray(raw)) {
    373     problems.add(path, `expected a list of services, got ${typeName(raw)}`);
    374     return [];
    375   }
    376 
    377   const out = [];
    378   const aliases = new Set();
    379   raw.forEach((item, i) => {
    380     const itemPath = `${path}[${i}]`;
    381     const service = typeof item === 'string' ? { image: item } : item;
    382     if (!isPlainObject(service)) {
    383       problems.add(itemPath, `expected an image name or a mapping, got ${typeName(item)}`);
    384       return;
    385     }
    386     checkUnknown(problems, itemPath, service, SERVICE_KEYS);
    387 
    388     const image = asString(problems, `${itemPath}.image`, service.image);
    389     if (image === undefined) return;
    390 
    391     // Default alias is the image name without registry, path or tag, which
    392     // is what a task would naturally use as a hostname.
    393     const derived = image.split('/').pop().split(':')[0];
    394     const alias = service.alias === undefined
    395       ? derived
    396       : asName(problems, `${itemPath}.alias`, service.alias, { max: 63 });
    397     if (alias === undefined) return;
    398     if (aliases.has(alias)) {
    399       problems.add(`${itemPath}.alias`, `duplicate service alias ${JSON.stringify(alias)}`);
    400       return;
    401     }
    402     aliases.add(alias);
    403 
    404     out.push({
    405       image,
    406       alias,
    407       env: asEnvMap(problems, `${itemPath}.env`, service.env),
    408       entrypoint: asStringList(problems, `${itemPath}.entrypoint`, service.entrypoint),
    409       command: asStringList(problems, `${itemPath}.command`, service.command),
    410     });
    411   });
    412   return out;
    413 }
    414 
    415 function parseArtifacts(problems, path, raw) {
    416   if (raw === undefined) return null;
    417 
    418   // The common case is a bare list of paths.
    419   const spec = Array.isArray(raw) ? { paths: raw } : raw;
    420   if (!isPlainObject(spec)) {
    421     problems.add(path, `expected a list of paths or a mapping, got ${typeName(raw)}`);
    422     return null;
    423   }
    424   checkUnknown(problems, path, spec, ARTIFACT_KEYS);
    425 
    426   const paths = asStringList(problems, `${path}.paths`, spec.paths, { max: 512 });
    427   if (paths.length === 0) {
    428     problems.add(`${path}.paths`, 'must list at least one path');
    429     return null;
    430   }
    431   paths.forEach((p, i) => {
    432     if (p.startsWith('/')) problems.add(`${path}.paths[${i}]`, 'must be relative to the workspace');
    433   });
    434 
    435   const when = spec.when === undefined ? 'on_success' : asString(problems, `${path}.when`, spec.when);
    436   if (when !== undefined && !ARTIFACT_WHEN.includes(when)) {
    437     problems.add(`${path}.when`, `expected one of ${ARTIFACT_WHEN.join(', ')}, got ${JSON.stringify(when)}`);
    438   }
    439 
    440   return {
    441     paths,
    442     when: ARTIFACT_WHEN.includes(when) ? when : 'on_success',
    443     expire: spec.expire === undefined ? null : asDuration(problems, `${path}.expire`, spec.expire, { max: 365 * 24 * 3600 }),
    444   };
    445 }
    446 
    447
	conductor CI task system
	git clone git://git.finwo.net/app/conductor
	Log \| Files \| Refs \| README \| LICENSE