parse.js - conductor - CI task system

parse.js (16081B)
      1 // src/lib/pipeline/parse.js - reads and validates a .conductor.yml
      2 //
      3 // Produces a normalized document: every task has every field filled in from
      4 // defaults, so later stages never have to ask whether something was set.
      5 // Nothing here expands matrices or resolves dependencies; see expand.js.
      6 
      7 import YAML from 'yaml';
      8 import {
      9   PipelineError,
     10   Problems,
     11   isPlainObject,
     12   checkUnknown,
     13   asString,
     14   asName,
     15   asBoolean,
     16   asInteger,
     17   asStringList,
     18   asEnvMap,
     19   asDuration,
     20   typeName,
     21   MATRIX_KEY_PATTERN,
     22   NAME_PATTERN,
     23 } from './schema.js';
     24 
     25 const TOP_KEYS = ['version', 'visibility', 'workdir', 'defaults', 'tasks'];
     26 
     27 // A repository decides whether its own build results are readable without
     28 // signing in. Unset means the project's setting stands.
     29 export const VISIBILITIES = ['public', 'private'];
     30 
     31 // Where the tree is put inside a task container, and the directory every
     32 // script starts in. Unset means the project's setting stands, and failing
     33 // that the server default.
     34 export const DEFAULT_WORKDIR = '/work';
     35 
     36 // An absolute path of ordinary directory names. The tree is delivered by
     37 // unpacking an archive at the container's root, so the path is built from
     38 // the archive's own entries: anything that could climb out of it, or that
     39 // is not a plain name, is refused rather than sanitised.
     40 export function parseWorkdir(problems, path, raw) {
     41   const value = asString(problems, path, raw);
     42   if (typeof value !== 'string') return null;
     43 
     44   if (!value.startsWith('/')) {
     45     problems.add(path, `must be an absolute path, got ${JSON.stringify(value)}`);
     46     return null;
     47   }
     48 
     49   const segments = value.split('/').filter(Boolean);
     50   if (segments.length === 0) {
     51     problems.add(path, 'must name a directory, not the container root');
     52     return null;
     53   }
     54 
     55   for (const segment of segments) {
     56     if (segment === '.' || segment === '..') {
     57       problems.add(path, `must not contain ${JSON.stringify(segment)}`);
     58       return null;
     59     }
     60     if (!/^[A-Za-z0-9._-]+$/.test(segment)) {
     61       problems.add(path, `${JSON.stringify(segment)} is not a usable directory name`);
     62       return null;
     63     }
     64   }
     65 
     66   return `/${segments.join('/')}`;
     67 }
     68 
     69 const JOB_KEYS = [
     70   'image', 'script', 'needs', 'arch', 'matrix', 'requires', 'services',
     71   'env', 'artifacts', 'allow_failure', 'timeout', 'max_attempts',
     72   'only',
     73 ];
     74 
     75 // Fields a task may inherit from defaults.
     76 const DEFAULT_KEYS = [
     77   'image', 'arch', 'requires', 'env', 'services',
     78   'allow_failure', 'timeout', 'max_attempts',
     79 ];
     80 
     81 const ONLY_KEYS = ['refs'];
     82 const ARTIFACT_KEYS = ['paths', 'expire', 'when'];
     83 const ARTIFACT_WHEN = ['on_success', 'on_failure', 'always'];
     84 
     85 const SERVICE_KEYS = ['image', 'alias', 'env', 'entrypoint', 'command'];
     86 
     87 export const SUPPORTED_VERSION = 1;
     88 
     89 export function parsePipeline(text, options = {}) {
     90   const source = options.source || '.conductor.yml';
     91   const problems = new Problems();
     92 
     93   let doc;
     94   try {
     95     doc = YAML.parse(text, { prettyErrors: true });
     96   } catch (e) {
     97     throw new PipelineError([{ path: '', message: `not valid YAML: ${e.message}` }], source);
     98   }
     99 
    100   if (doc === null || doc === undefined) {
    101     throw new PipelineError([{ path: '', message: 'file is empty' }], source);
    102   }
    103   if (!isPlainObject(doc)) {
    104     throw new PipelineError([{ path: '', message: `expected a mapping at the top level, got ${typeName(doc)}` }], source);
    105   }
    106 
    107   // The key was called jobs before a job became the thing that contains
    108   // these. Named outright rather than left to checkUnknown, whose nearest
    109   // match would be a guess at a word that used to be correct.
    110   if (doc.jobs !== undefined && doc.tasks === undefined) {
    111     throw new PipelineError(
    112       [{ path: 'jobs', message: 'was renamed to tasks; a job is now the pipeline run that these tasks belong to' }],
    113       source
    114     );
    115   }
    116 
    117   checkUnknown(problems, '', doc, TOP_KEYS);
    118 
    119   if (doc.version === undefined) {
    120     problems.add('version', `is required; this conductor understands version ${SUPPORTED_VERSION}`);
    121   } else if (doc.version !== SUPPORTED_VERSION) {
    122     problems.add('version', `unsupported version ${JSON.stringify(doc.version)}, expected ${SUPPORTED_VERSION}`);
    123   }
    124 
    125   let visibility = null;
    126   if (doc.visibility !== undefined) {
    127     const value = asString(problems, 'visibility', doc.visibility);
    128     if (value !== undefined && !VISIBILITIES.includes(value)) {
    129       problems.add('visibility', `expected one of ${VISIBILITIES.join(', ')}, got ${JSON.stringify(value)}`);
    130     } else if (value !== undefined) {
    131       visibility = value;
    132     }
    133   }
    134 
    135   const defaults = parseDefaults(problems, doc.defaults);
    136 
    137   if (doc.tasks === undefined) {
    138     problems.add('tasks', 'is required');
    139   } else if (!isPlainObject(doc.tasks)) {
    140     problems.add('tasks', `expected a mapping of task names, got ${typeName(doc.tasks)}`);
    141   } else if (Object.keys(doc.tasks).length === 0) {
    142     problems.add('tasks', 'must define at least one task');
    143   }
    144 
    145   const tasks = {};
    146   if (isPlainObject(doc.tasks)) {
    147     for (const [name, raw] of Object.entries(doc.tasks)) {
    148       const path = `tasks.${name}`;
    149       if (!NAME_PATTERN.test(name) || name.length > 100) {
    150         problems.add(path, 'task name must start with a letter or digit and contain only letters, digits, underscore, dot and hyphen');
    151         continue;
    152       }
    153       if (!isPlainObject(raw)) {
    154         problems.add(path, `expected a mapping, got ${typeName(raw)}`);
    155         continue;
    156       }
    157       tasks[name] = parseTask(problems, path, raw, defaults);
    158     }
    159   }
    160 
    161   const workdir = doc.workdir === undefined ? null : parseWorkdir(problems, 'workdir', doc.workdir);
    162 
    163   problems.throwIfAny(source);
    164   return { version: SUPPORTED_VERSION, visibility, workdir, defaults, tasks, source };
    165 }
    166 
    167 function parseDefaults(problems, raw) {
    168   const empty = {
    169     image: undefined,
    170     arch: undefined,
    171     requires: [],
    172     env: {},
    173     services: [],
    174     allow_failure: false,
    175     timeout: undefined,
    176     max_attempts: undefined,
    177   };
    178   if (raw === undefined) return empty;
    179   if (!isPlainObject(raw)) {
    180     problems.add('defaults', `expected a mapping, got ${typeName(raw)}`);
    181     return empty;
    182   }
    183 
    184   checkUnknown(problems, 'defaults', raw, DEFAULT_KEYS);
    185 
    186   return {
    187     image: raw.image === undefined ? undefined : asString(problems, 'defaults.image', raw.image),
    188     arch: raw.arch === undefined ? undefined : parseArch(problems, 'defaults.arch', raw.arch),
    189     requires: asStringList(problems, 'defaults.requires', raw.requires),
    190     env: asEnvMap(problems, 'defaults.env', raw.env),
    191     services: parseServices(problems, 'defaults.services', raw.services),
    192     allow_failure: raw.allow_failure === undefined ? false : asBoolean(problems, 'defaults.allow_failure', raw.allow_failure) ?? false,
    193     timeout: raw.timeout === undefined ? undefined : asDuration(problems, 'defaults.timeout', raw.timeout),
    194     max_attempts: raw.max_attempts === undefined ? undefined : asInteger(problems, 'defaults.max_attempts', raw.max_attempts, { min: 1, max: 10 }),
    195   };
    196 }
    197 
    198 function parseArch(problems, path, raw) {
    199   const list = asStringList(problems, path, raw, { max: 32 });
    200   const out = [];
    201   list.forEach((value, i) => {
    202     if (!NAME_PATTERN.test(value)) {
    203       problems.add(`${path}[${i}]`, `${JSON.stringify(value)} is not a valid architecture name`);
    204       return;
    205     }
    206     if (out.includes(value)) {
    207       problems.add(`${path}[${i}]`, `duplicate architecture ${JSON.stringify(value)}`);
    208       return;
    209     }
    210     out.push(value);
    211   });
    212   return out;
    213 }
    214 
    215 function parseTask(problems, path, raw, defaults) {
    216   checkUnknown(problems, path, raw, JOB_KEYS);
    217 
    218   const image = raw.image === undefined ? defaults.image : asString(problems, `${path}.image`, raw.image);
    219   if (image === undefined) {
    220     problems.add(`${path}.image`, 'is required; set it on the task or under defaults');
    221   }
    222 
    223   if (raw.script === undefined) {
    224     problems.add(`${path}.script`, 'is required');
    225   }
    226   const script = asStringList(problems, `${path}.script`, raw.script, { max: 65536 });
    227   if (raw.script !== undefined && script.length === 0) {
    228     problems.add(`${path}.script`, 'must contain at least one command');
    229   }
    230 
    231   const arch = raw.arch === undefined ? (defaults.arch ?? []) : parseArch(problems, `${path}.arch`, raw.arch);
    232   const matrix = parseMatrix(problems, `${path}.matrix`, raw.matrix);
    233 
    234   if (matrix.arch !== undefined) {
    235     problems.add(`${path}.matrix.arch`, 'use the arch key instead of a matrix dimension named arch');
    236   }
    237 
    238   return {
    239     image,
    240     script,
    241     needs: parseNeeds(problems, `${path}.needs`, raw.needs),
    242     arch,
    243     matrix,
    244     requires: raw.requires === undefined ? defaults.requires : asStringList(problems, `${path}.requires`, raw.requires, { max: 64 }),
    245     services: raw.services === undefined ? defaults.services : parseServices(problems, `${path}.services`, raw.services),
    246     env: { ...defaults.env, ...asEnvMap(problems, `${path}.env`, raw.env) },
    247     artifacts: parseArtifacts(problems, `${path}.artifacts`, raw.artifacts),
    248     allow_failure: raw.allow_failure === undefined
    249       ? defaults.allow_failure
    250       : asBoolean(problems, `${path}.allow_failure`, raw.allow_failure) ?? false,
    251     timeout: raw.timeout === undefined ? defaults.timeout : asDuration(problems, `${path}.timeout`, raw.timeout),
    252     max_attempts: raw.max_attempts === undefined
    253       ? defaults.max_attempts
    254       : asInteger(problems, `${path}.max_attempts`, raw.max_attempts, { min: 1, max: 10 }),
    255     only: parseOnly(problems, `${path}.only`, raw.only),
    256   };
    257 }
    258 
    259 // Restricts a task to certain refs. Deliberately not inheritable from
    260 // defaults: a rule that silently applied to every task is the kind of thing
    261 // that stops a pipeline running at all and takes an afternoon to find.
    262 //
    263 //   only:
    264 //     refs: [refs/heads/main, 'refs/tags/v*']
    265 //
    266 // Patterns match the whole ref, so refs/heads/main rather than main, with
    267 // * standing for any run of characters. Absent means the task always runs.
    268 function parseOnly(problems, path, raw) {
    269   if (raw === undefined || raw === null) return null;
    270   if (!isPlainObject(raw)) {
    271     problems.add(path, 'must be a mapping, for example: only: { refs: [refs/heads/main] }');
    272     return null;
    273   }
    274 
    275   for (const key of Object.keys(raw)) {
    276     if (!ONLY_KEYS.includes(key)) {
    277       problems.add(`${path}.${key}`, `unknown key, expected one of: ${ONLY_KEYS.join(', ')}`);
    278     }
    279   }
    280 
    281   if (raw.refs === undefined) {
    282     problems.add(path, 'needs a refs list, otherwise it restricts nothing');
    283     return null;
    284   }
    285 
    286   const refs = asStringList(problems, `${path}.refs`, raw.refs, { max: 64 });
    287   if (refs.length === 0) {
    288     problems.add(`${path}.refs`, 'must list at least one ref pattern');
    289     return null;
    290   }
    291 
    292   return { refs };
    293 }
    294 
    295 // A need is either a task name, or a mapping for the cases where the default
    296 // dimension matching is not what is wanted.
    297 function parseNeeds(problems, path, raw) {
    298   if (raw === undefined) return [];
    299   const list = Array.isArray(raw) ? raw : [raw];
    300   const out = [];
    301 
    302   list.forEach((item, i) => {
    303     const itemPath = `${path}[${i}]`;
    304     if (typeof item === 'string') {
    305       out.push({ task: item, match: 'shared' });
    306       return;
    307     }
    308     if (!isPlainObject(item)) {
    309       problems.add(itemPath, `expected a task name or a mapping, got ${typeName(item)}`);
    310       return;
    311     }
    312     checkUnknown(problems, itemPath, item, ['task', 'match']);
    313     const task = asString(problems, `${itemPath}.task`, item.task);
    314     const match = item.match === undefined ? 'shared' : asString(problems, `${itemPath}.match`, item.match);
    315     if (match !== undefined && !['shared', 'all'].includes(match)) {
    316       problems.add(`${itemPath}.match`, `expected shared or all, got ${JSON.stringify(match)}`);
    317       return;
    318     }
    319     if (task !== undefined) out.push({ task, match: match ?? 'shared' });
    320   });
    321 
    322   return out;
    323 }
    324 
    325 function parseMatrix(problems, path, raw) {
    326   if (raw === undefined) return {};
    327   if (!isPlainObject(raw)) {
    328     problems.add(path, `expected a mapping of dimension names to value lists, got ${typeName(raw)}`);
    329     return {};
    330   }
    331 
    332   const out = {};
    333   for (const [key, value] of Object.entries(raw)) {
    334     const keyPath = `${path}.${key}`;
    335     if (!MATRIX_KEY_PATTERN.test(key)) {
    336       problems.add(keyPath, 'dimension name must be a valid environment variable name');
    337       continue;
    338     }
    339     if (!Array.isArray(value)) {
    340       problems.add(keyPath, `expected a list of values, got ${typeName(value)}`);
    341       continue;
    342     }
    343     if (value.length === 0) {
    344       problems.add(keyPath, 'must contain at least one value');
    345       continue;
    346     }
    347 
    348     const values = [];
    349     value.forEach((item, i) => {
    350       if (typeof item !== 'string' && typeof item !== 'number' && typeof item !== 'boolean') {
    351         problems.add(`${keyPath}[${i}]`, `expected a scalar value, got ${typeName(item)}`);
    352         return;
    353       }
    354       const s = String(item);
    355       if (!NAME_PATTERN.test(s)) {
    356         problems.add(
    357           `${keyPath}[${i}]`,
    358           `${JSON.stringify(s)} may only contain letters, digits, underscore, dot and hyphen, ` +
    359           'because it becomes part of the task name'
    360         );
    361         return;
    362       }
    363       if (values.includes(s)) {
    364         problems.add(`${keyPath}[${i}]`, `duplicate value ${JSON.stringify(s)}`);
    365         return;
    366       }
    367       values.push(s);
    368     });
    369     out[key] = values;
    370   }
    371   return out;
    372 }
    373 
    374 function parseServices(problems, path, raw) {
    375   if (raw === undefined) return [];
    376   if (!Array.isArray(raw)) {
    377     problems.add(path, `expected a list of services, got ${typeName(raw)}`);
    378     return [];
    379   }
    380 
    381   const out = [];
    382   const aliases = new Set();
    383   raw.forEach((item, i) => {
    384     const itemPath = `${path}[${i}]`;
    385     const service = typeof item === 'string' ? { image: item } : item;
    386     if (!isPlainObject(service)) {
    387       problems.add(itemPath, `expected an image name or a mapping, got ${typeName(item)}`);
    388       return;
    389     }
    390     checkUnknown(problems, itemPath, service, SERVICE_KEYS);
    391 
    392     const image = asString(problems, `${itemPath}.image`, service.image);
    393     if (image === undefined) return;
    394 
    395     // Default alias is the image name without registry, path or tag, which
    396     // is what a task would naturally use as a hostname.
    397     const derived = image.split('/').pop().split(':')[0];
    398     const alias = service.alias === undefined
    399       ? derived
    400       : asName(problems, `${itemPath}.alias`, service.alias, { max: 63 });
    401     if (alias === undefined) return;
    402     if (aliases.has(alias)) {
    403       problems.add(`${itemPath}.alias`, `duplicate service alias ${JSON.stringify(alias)}`);
    404       return;
    405     }
    406     aliases.add(alias);
    407 
    408     out.push({
    409       image,
    410       alias,
    411       env: asEnvMap(problems, `${itemPath}.env`, service.env),
    412       entrypoint: asStringList(problems, `${itemPath}.entrypoint`, service.entrypoint),
    413       command: asStringList(problems, `${itemPath}.command`, service.command),
    414     });
    415   });
    416   return out;
    417 }
    418 
    419 function parseArtifacts(problems, path, raw) {
    420   if (raw === undefined) return null;
    421 
    422   // The common case is a bare list of paths.
    423   const spec = Array.isArray(raw) ? { paths: raw } : raw;
    424   if (!isPlainObject(spec)) {
    425     problems.add(path, `expected a list of paths or a mapping, got ${typeName(raw)}`);
    426     return null;
    427   }
    428   checkUnknown(problems, path, spec, ARTIFACT_KEYS);
    429 
    430   const paths = asStringList(problems, `${path}.paths`, spec.paths, { max: 512 });
    431   if (paths.length === 0) {
    432     problems.add(`${path}.paths`, 'must list at least one path');
    433     return null;
    434   }
    435   paths.forEach((p, i) => {
    436     if (p.startsWith('/')) problems.add(`${path}.paths[${i}]`, 'must be relative to the workspace');
    437   });
    438 
    439   const when = spec.when === undefined ? 'on_success' : asString(problems, `${path}.when`, spec.when);
    440   if (when !== undefined && !ARTIFACT_WHEN.includes(when)) {
    441     problems.add(`${path}.when`, `expected one of ${ARTIFACT_WHEN.join(', ')}, got ${JSON.stringify(when)}`);
    442   }
    443 
    444   return {
    445     paths,
    446     when: ARTIFACT_WHEN.includes(when) ? when : 'on_success',
    447     expire: spec.expire === undefined ? null : asDuration(problems, `${path}.expire`, spec.expire, { max: 365 * 24 * 3600 }),
    448   };
    449 }
    450 
    451
	conductor CI task system
	git clone git://git.finwo.net/app/conductor
	Log \| Files \| Refs \| README \| LICENSE