config.js - conductor - CI task system

config.js (15471B)
      1 // src/lib/config.js - configuration loading for all conductor services
      2 //
      3 // Resolution order, lowest precedence first:
      4 //   1. built-in defaults (see DEFAULTS)
      5 //   2. YAML file, from CONDUCTOR_CONFIG or ./conductor.yaml when present
      6 //   3. environment variables (see ENV_MAP)
      7 //
      8 // Two capabilities switch on automatically when configured, and fall back
      9 // otherwise:
     10 //   storage.s3.bucket set       -> S3 object storage, else local filesystem
     11 //   auth.oidc.discovery_url set -> OIDC, else built-in user accounts
     12 
     13 import fs from 'node:fs';
     14 import path from 'node:path';
     15 import crypto from 'node:crypto';
     16 import YAML from 'yaml';
     17 import { dialectFromUrl } from './db/dialect.js';
     18 
     19 const DEFAULTS = {
     20   server: {
     21     host: '0.0.0.0',
     22     port: 8080,
     23     // Absolute URL this conductor is reachable at. Used when handing workers
     24     // callback URLs, so it must be correct behind a reverse proxy.
     25     public_url: 'http://127.0.0.1:8080',
     26   },
     27   database: {
     28     // Unset means sqlite at database.path. Otherwise the scheme selects the
     29     // dialect: mysql:// or postgres:// (postgresql:// is accepted too).
     30     url: null,
     31     path: './data/conductor.db',
     32     // Pool sizing for the networked dialects, ignored by sqlite.
     33     connection_limit: 10,
     34   },
     35   storage: {
     36     path: './data/storage',
     37     s3: {
     38       endpoint: null,
     39       region: 'us-east-1',
     40       bucket: null,
     41       access_key_id: null,
     42       secret_access_key: null,
     43       // Garage and MinIO need path style addressing.
     44       force_path_style: true,
     45     },
     46   },
     47   auth: {
     48     // Random per boot when unset, which invalidates existing sessions.
     49     session_secret: null,
     50     access_ttl: 900,
     51     refresh_ttl: 2592000,
     52     refresh_sliding: true,
     53     local: {
     54       enabled: null,
     55       redirect_uris: null,
     56     },
     57     oidc: {
     58       // The provider's OpenID configuration document. Its issuer and every
     59       // endpoint are read from here rather than configured separately.
     60       discovery_url: null,
     61       client_id: null,
     62       client_secret: null,
     63       scopes: 'openid profile email groups',
     64       admin_role: 'conductor-admin',
     65     },
     66     // Created on first boot when the users table is empty.
     67     bootstrap_admin: {
     68       username: 'admin',
     69       password: null,
     70     },
     71   },
     72   secrets: {
     73     // 32 bytes, hex or base64, for AES-256-GCM encryption of project
     74     // variables at rest. Required only once project variables are used.
     75     encryption_key: null,
     76   },
     77   git: {
     78     mirror_path: './data/mirrors',
     79     // Seconds before a cached mirror fetch is considered stale.
     80     fetch_interval: 60,
     81     timeout: 300,
     82   },
     83   log: {
     84     spool_path: './data/logs',
     85     // Refuse log appends past this size, to bound a runaway job.
     86     max_size: 64 * 1024 * 1024,
     87   },
     88   // What happens to build output over time. A project may override any of
     89   // these; see migrations/sqlite/001_initial.sql for how the artifact
     90   // rules combine. Zero means keep forever.
     91   retention: {
     92     // The last this many runs keep their artifacts whatever their age.
     93     artifact_keep_jobs: 10,
     94     // Artifacts younger than this are kept whatever has followed them.
     95     artifact_keep_days: 30,
     96     // Logs go purely by age, and are the reason this exists at all: they
     97     // are written for every job, read for almost none, and never stop.
     98     log_keep_days: 14,
     99     // A machine that stops claiming for this long is forgotten.
    100     worker_agent_keep_hours: 24,
    101     // How often to sweep. Deleting is not urgent, and a sweep walks every
    102     // project, so this is deliberately not the scheduler's interval.
    103     sweep_interval: 3600,
    104     // How much to delete in one pass, so a first sweep over a large
    105     // backlog cannot monopolise the database or the object store.
    106     batch: 500,
    107   },
    108   scheduler: {
    109     // A claimed job whose worker stops sending heartbeats for this long is
    110     // considered lost and is requeued or failed.
    111     heartbeat_timeout: 120,
    112     reap_interval: 30,
    113     default_task_timeout: 3600,
    114     max_attempts: 1,
    115   },
    116 };
    117 
    118 // [environment variable, dotted config path, parser]
    119 const ENV_MAP = [
    120   ['CONDUCTOR_RETENTION_ARTIFACT_JOBS', 'retention.artifact_keep_jobs', toInt],
    121   ['CONDUCTOR_RETENTION_ARTIFACT_DAYS', 'retention.artifact_keep_days', toInt],
    122   ['CONDUCTOR_RETENTION_LOG_DAYS',      'retention.log_keep_days',      toInt],
    123   ['CONDUCTOR_RETENTION_WORKER_AGENT_HOURS', 'retention.worker_agent_keep_hours', toInt],
    124   ['CONDUCTOR_RETENTION_SWEEP_INTERVAL','retention.sweep_interval',     toInt],
    125   ['CONDUCTOR_HOST',                 'server.host',                 String],
    126   ['CONDUCTOR_PORT',                 'server.port',                 toInt],
    127   ['CONDUCTOR_PUBLIC_URL',           'server.public_url',           String],
    128   ['CONDUCTOR_DATABASE_URL',         'database.url',                String],
    129   ['CONDUCTOR_DATABASE_PATH',        'database.path',               String],
    130   ['CONDUCTOR_STORAGE_PATH',         'storage.path',                String],
    131   ['CONDUCTOR_S3_ENDPOINT',          'storage.s3.endpoint',         String],
    132   ['CONDUCTOR_S3_REGION',            'storage.s3.region',           String],
    133   ['CONDUCTOR_S3_BUCKET',            'storage.s3.bucket',           String],
    134   ['CONDUCTOR_S3_ACCESS_KEY_ID',     'storage.s3.access_key_id',    String],
    135   ['CONDUCTOR_S3_SECRET_ACCESS_KEY', 'storage.s3.secret_access_key', String],
    136   ['CONDUCTOR_S3_FORCE_PATH_STYLE',  'storage.s3.force_path_style', toBool],
    137   ['CONDUCTOR_SESSION_SECRET',       'auth.session_secret',         String],
    138   ['CONDUCTOR_ACCESS_TTL',           'auth.access_ttl',             toInt],
    139   ['CONDUCTOR_REFRESH_TTL',          'auth.refresh_ttl',            toInt],
    140   ['CONDUCTOR_REFRESH_SLIDING',      'auth.refresh_sliding',        toBool],
    141   ['CONDUCTOR_LOCAL_AUTH',           'auth.local.enabled',          toBool],
    142   ['CONDUCTOR_LOCAL_REDIRECT_URIS',  'auth.local.redirect_uris',    String],
    143   ['CONDUCTOR_OIDC_DISCOVERY_URL',   'auth.oidc.discovery_url',     String],
    144   ['CONDUCTOR_OIDC_CLIENT_ID',       'auth.oidc.client_id',         String],
    145   ['CONDUCTOR_OIDC_CLIENT_SECRET',   'auth.oidc.client_secret',     String],
    146   ['CONDUCTOR_OIDC_SCOPES',          'auth.oidc.scopes',            String],
    147   ['CONDUCTOR_OIDC_ADMIN_ROLE',      'auth.oidc.admin_role',        String],
    148   ['CONDUCTOR_ADMIN_USERNAME',       'auth.bootstrap_admin.username', String],
    149   ['CONDUCTOR_ADMIN_PASSWORD',       'auth.bootstrap_admin.password', String],
    150   ['CONDUCTOR_SECRET_KEY',           'secrets.encryption_key',      String],
    151   ['CONDUCTOR_MIRROR_PATH',          'git.mirror_path',             String],
    152   ['CONDUCTOR_LOG_PATH',             'log.spool_path',              String],
    153 ];
    154 
    155 // Paths resolved relative to the config file directory, or cwd when there is
    156 // no config file.
    157 const PATH_KEYS = ['database.path', 'storage.path', 'git.mirror_path', 'log.spool_path'];
    158 
    159 function toInt(v) {
    160   const n = parseInt(v, 10);
    161   if (!Number.isFinite(n)) throw new Error(`expected an integer, got ${JSON.stringify(v)}`);
    162   return n;
    163 }
    164 
    165 function toBool(v) {
    166   if (typeof v === 'boolean') return v;
    167   const s = String(v).toLowerCase();
    168   if (['1', 'true', 'yes', 'on'].includes(s)) return true;
    169   if (['0', 'false', 'no', 'off'].includes(s)) return false;
    170   throw new Error(`expected a boolean, got ${JSON.stringify(v)}`);
    171 }
    172 
    173 function isPlainObject(v) {
    174   return v !== null && typeof v === 'object' && !Array.isArray(v);
    175 }
    176 
    177 function deepMerge(base, overlay) {
    178   const out = Array.isArray(base) ? [...base] : { ...base };
    179   for (const [k, v] of Object.entries(overlay || {})) {
    180     if (v === undefined) continue;
    181     out[k] = isPlainObject(v) && isPlainObject(base?.[k]) ? deepMerge(base[k], v) : v;
    182   }
    183   return out;
    184 }
    185 
    186 function getPath(obj, dotted) {
    187   return dotted.split('.').reduce((acc, k) => (acc == null ? acc : acc[k]), obj);
    188 }
    189 
    190 function setPath(obj, dotted, value) {
    191   const keys = dotted.split('.');
    192   const last = keys.pop();
    193   let cur = obj;
    194   for (const k of keys) {
    195     if (!isPlainObject(cur[k])) cur[k] = {};
    196     cur = cur[k];
    197   }
    198   cur[last] = value;
    199 }
    200 
    201 function deepFreeze(obj) {
    202   for (const v of Object.values(obj)) {
    203     if (isPlainObject(v) || Array.isArray(v)) deepFreeze(v);
    204   }
    205   return Object.freeze(obj);
    206 }
    207 
    208 // Comma separated from the environment, a list from a config file.
    209 function redirectUris(value, publicUrl) {
    210   const given = Array.isArray(value) ? value : String(value ?? '').split(',');
    211   const uris = given.map((u) => String(u).trim().replace(/\/+$/, '')).filter(Boolean);
    212   if (uris.length === 0) return [publicUrl.replace(/\/+$/, '')];
    213   return uris.includes('*') ? ['*'] : uris;
    214 }
    215 
    216 // Reads a 32 byte key given as hex or base64. Returns a Buffer, or null.
    217 export function parseKey(value, label) {
    218   if (!value) return null;
    219   let buf = null;
    220   if (/^[0-9a-fA-F]{64}$/.test(value)) buf = Buffer.from(value, 'hex');
    221   else buf = Buffer.from(value, 'base64');
    222   if (buf.length !== 32) {
    223     throw new Error(`${label} must decode to 32 bytes, got ${buf.length}; use 64 hex chars or base64`);
    224   }
    225   return buf;
    226 }
    227 
    228 function validate(cfg) {
    229   const errors = [];
    230 
    231   // Port 0 asks the operating system for a free port, which is useful in
    232   // tests and for ephemeral instances.
    233   const port = cfg.server.port;
    234   if (!Number.isInteger(port) || port < 0 || port > 65535) {
    235     errors.push(`server.port must be between 0 and 65535, got ${port}`);
    236   }
    237   if (cfg.database.url && dialectFromUrl(cfg.database.url) === null) {
    238     errors.push(
    239       `database.url has an unsupported scheme: ${String(cfg.database.url).split(':')[0]}; ` +
    240       'use mysql://, postgres:// or leave it unset for sqlite'
    241     );
    242   }
    243 
    244   const s3 = cfg.storage.s3;
    245   const s3Given = [s3.endpoint, s3.bucket, s3.access_key_id, s3.secret_access_key].filter(Boolean);
    246   if (s3Given.length > 0 && s3Given.length < 4) {
    247     errors.push(
    248       'storage.s3 is partially configured; endpoint, bucket, access_key_id and ' +
    249       'secret_access_key are all required, or leave them all unset for local storage'
    250     );
    251   }
    252 
    253   if (cfg.auth.oidc.discovery_url) {
    254     try {
    255       new URL(cfg.auth.oidc.discovery_url);
    256     } catch {
    257       errors.push(
    258         `auth.oidc.discovery_url must be an absolute URL, got ${JSON.stringify(cfg.auth.oidc.discovery_url)}`
    259       );
    260     }
    261     if (!cfg.auth.oidc.client_id) {
    262       errors.push('auth.oidc.client_id is required when auth.oidc.discovery_url is set');
    263     }
    264   } else if (cfg.auth.local.enabled === false) {
    265     errors.push('auth.local.enabled is false and no auth.oidc.discovery_url is set, so nobody could sign in');
    266   }
    267 
    268   // A bare host:port parses with a nonsense scheme and would match nothing.
    269   for (const uri of redirectUris(cfg.auth.local.redirect_uris, cfg.server.public_url)) {
    270     if (uri === '*') continue;
    271     let parsed = null;
    272     try {
    273       parsed = new URL(uri);
    274     } catch { /* reported below */ }
    275 
    276     if (!parsed || !parsed.protocol.startsWith('http')) {
    277       errors.push(
    278         `auth.local.redirect_uris entry ${JSON.stringify(uri)} must be an absolute http or https URL, `
    279         + 'such as https://ci.example.com, or * to allow any origin'
    280       );
    281     }
    282   }
    283 
    284   try {
    285     new URL(cfg.server.public_url);
    286   } catch {
    287     errors.push(`server.public_url must be an absolute URL, got ${JSON.stringify(cfg.server.public_url)}`);
    288   }
    289 
    290   try {
    291     parseKey(cfg.secrets.encryption_key, 'secrets.encryption_key');
    292   } catch (e) {
    293     errors.push(e.message);
    294   }
    295 
    296   if (cfg.scheduler.heartbeat_timeout <= cfg.scheduler.reap_interval) {
    297     errors.push(
    298       `scheduler.heartbeat_timeout (${cfg.scheduler.heartbeat_timeout}) must be greater than ` +
    299       `scheduler.reap_interval (${cfg.scheduler.reap_interval}), or healthy jobs will be reaped`
    300     );
    301   }
    302 
    303   // Retention deletes things, so a nonsensical value here is worth
    304   // refusing at startup rather than discovering from missing artifacts.
    305   for (const key of ['artifact_keep_jobs', 'artifact_keep_days', 'log_keep_days', 'worker_agent_keep_hours']) {
    306     const value = cfg.retention[key];
    307     if (!Number.isInteger(value) || value < 0) {
    308       errors.push(`retention.${key} must be a whole number of zero or more, got ${JSON.stringify(value)}`);
    309     }
    310   }
    311 
    312   if (!Number.isInteger(cfg.retention.sweep_interval) || cfg.retention.sweep_interval < 60) {
    313     errors.push(
    314       `retention.sweep_interval must be at least 60 seconds, got ${JSON.stringify(cfg.retention.sweep_interval)}`
    315     );
    316   }
    317 
    318   if (errors.length > 0) {
    319     throw new Error(`invalid configuration:\n  - ${errors.join('\n  - ')}`);
    320   }
    321 }
    322 
    323 export function loadConfig(explicitPath) {
    324   const file = explicitPath || process.env.CONDUCTOR_CONFIG || 'conductor.yaml';
    325   let cfg = structuredClone(DEFAULTS);
    326   let baseDir = process.cwd();
    327   let source = null;
    328 
    329   if (fs.existsSync(file)) {
    330     const text = fs.readFileSync(file, 'utf8');
    331     let parsed;
    332     try {
    333       parsed = YAML.parse(text) || {};
    334     } catch (e) {
    335       throw new Error(`failed to parse config file ${file}: ${e.message}`);
    336     }
    337     if (!isPlainObject(parsed)) {
    338       throw new Error(`config file ${file} must contain a YAML mapping at the top level`);
    339     }
    340     cfg = deepMerge(cfg, parsed);
    341     baseDir = path.dirname(path.resolve(file));
    342     source = path.resolve(file);
    343   } else if (explicitPath || process.env.CONDUCTOR_CONFIG) {
    344     throw new Error(`config file not found: ${file}`);
    345   }
    346 
    347   for (const [env, dotted, parse] of ENV_MAP) {
    348     const raw = process.env[env];
    349     if (raw === undefined || raw === '') continue;
    350     try {
    351       setPath(cfg, dotted, parse(raw));
    352     } catch (e) {
    353       throw new Error(`invalid value for ${env}: ${e.message}`);
    354     }
    355   }
    356 
    357   // A sqlite: or file: url is just another way of spelling database.path.
    358   // Collapse it now so that everything downstream sees one representation.
    359   if (cfg.database.url && /^(sqlite|file):/i.test(cfg.database.url)) {
    360     const raw = cfg.database.url.replace(/^(sqlite|file):(\/\/)?/i, '');
    361     if (!raw) throw new Error(`database.url ${cfg.database.url} does not contain a file path`);
    362     cfg.database.path = raw;
    363     cfg.database.url = null;
    364   }
    365 
    366   for (const key of PATH_KEYS) {
    367     const v = getPath(cfg, key);
    368     if (typeof v === 'string' && v.length > 0) setPath(cfg, key, path.resolve(baseDir, v));
    369   }
    370 
    371   validate(cfg);
    372 
    373   // Derived flags, so callers never re-implement the fallback rules.
    374   cfg.source = source;
    375   cfg.database.dialect = dialectFromUrl(cfg.database.url);
    376   cfg.storage.driver = cfg.storage.s3.bucket ? 's3' : 'local';
    377 
    378   // The discovery document names one provider, so these two always agree.
    379   if (cfg.auth.local.enabled === null) cfg.auth.local.enabled = !cfg.auth.oidc.discovery_url;
    380   cfg.auth.mode = cfg.auth.local.enabled ? 'local' : 'oidc';
    381   cfg.auth.local.redirect_uris = redirectUris(cfg.auth.local.redirect_uris, cfg.server.public_url);
    382 
    383   if (!cfg.auth.session_secret) {
    384     cfg.auth.session_secret = crypto.randomBytes(32).toString('hex');
    385     cfg.auth.session_secret_ephemeral = true;
    386   }
    387 
    388   return deepFreeze(cfg);
    389 }
    390 
    391 // Directories that must exist before a service starts. Which ones matter
    392 // depends on the selected drivers, so this is derived rather than fixed.
    393 export function stateDirs(cfg) {
    394   const dirs = [cfg.git.mirror_path, cfg.log.spool_path];
    395   if (cfg.storage.driver === 'local') dirs.push(cfg.storage.path);
    396   if (cfg.database.dialect === 'sqlite') dirs.push(path.dirname(cfg.database.path));
    397   return [...new Set(dirs)];
    398 }
    399 
    400 export function ensureStateDirs(cfg) {
    401   for (const dir of stateDirs(cfg)) fs.mkdirSync(dir, { recursive: true });
    402 }
	conductor CI task system
	git clone git://git.finwo.net/app/conductor
	Log \| Files \| Refs \| README \| LICENSE