config.js (13539B)
1 // src/lib/config.js - configuration loading for all conductor services 2 // 3 // Resolution order, lowest precedence first: 4 // 1. built-in defaults (see DEFAULTS) 5 // 2. YAML file, from CONDUCTOR_CONFIG or ./conductor.yaml when present 6 // 3. environment variables (see ENV_MAP) 7 // 8 // Two capabilities switch on automatically when configured, and fall back 9 // otherwise: 10 // storage.s3.bucket set -> S3 object storage, else local filesystem 11 // auth.oidc.discovery_url set -> OIDC, else built-in user accounts 12 13 import fs from 'node:fs'; 14 import path from 'node:path'; 15 import crypto from 'node:crypto'; 16 import YAML from 'yaml'; 17 import { dialectFromUrl } from './db/dialect.js'; 18 19 const DEFAULTS = { 20 server: { 21 host: '0.0.0.0', 22 port: 8080, 23 // Absolute URL this conductor is reachable at. Used when handing workers 24 // callback URLs, so it must be correct behind a reverse proxy. 25 public_url: 'http://127.0.0.1:8080', 26 }, 27 database: { 28 // Unset means sqlite at database.path. Otherwise the scheme selects the 29 // dialect: mysql:// or postgres:// (postgresql:// is accepted too). 30 url: null, 31 path: './data/conductor.db', 32 // Pool sizing for the networked dialects, ignored by sqlite. 33 connection_limit: 10, 34 }, 35 storage: { 36 path: './data/storage', 37 s3: { 38 endpoint: null, 39 region: 'us-east-1', 40 bucket: null, 41 access_key_id: null, 42 secret_access_key: null, 43 // Garage and MinIO need path style addressing. 44 force_path_style: true, 45 }, 46 }, 47 auth: { 48 // Signs built-in session tokens. Random per boot when unset, which 49 // invalidates existing sessions on restart. 50 session_secret: null, 51 session_ttl: 43200, 52 oidc: { 53 // The provider's OpenID configuration document. Its issuer and every 54 // endpoint are read from here rather than configured separately. 55 discovery_url: null, 56 client_id: null, 57 client_secret: null, 58 scopes: 'openid profile email', 59 admin_role: 'conductor-admin', 60 }, 61 // Created on first boot when the users table is empty. 62 bootstrap_admin: { 63 username: 'admin', 64 password: null, 65 }, 66 }, 67 secrets: { 68 // 32 bytes, hex or base64, for AES-256-GCM encryption of project 69 // variables at rest. Required only once project variables are used. 70 encryption_key: null, 71 }, 72 git: { 73 mirror_path: './data/mirrors', 74 // Seconds before a cached mirror fetch is considered stale. 75 fetch_interval: 60, 76 timeout: 300, 77 }, 78 log: { 79 spool_path: './data/logs', 80 // Refuse log appends past this size, to bound a runaway job. 81 max_size: 64 * 1024 * 1024, 82 }, 83 // What happens to build output over time. A project may override any of 84 // these; see migrations/sqlite/001_initial.sql for how the artifact 85 // rules combine. Zero means keep forever. 86 retention: { 87 // The last this many runs keep their artifacts whatever their age. 88 artifact_keep_jobs: 10, 89 // Artifacts younger than this are kept whatever has followed them. 90 artifact_keep_days: 30, 91 // Logs go purely by age, and are the reason this exists at all: they 92 // are written for every job, read for almost none, and never stop. 93 log_keep_days: 14, 94 // How often to sweep. Deleting is not urgent, and a sweep walks every 95 // project, so this is deliberately not the scheduler's interval. 96 sweep_interval: 3600, 97 // How much to delete in one pass, so a first sweep over a large 98 // backlog cannot monopolise the database or the object store. 99 batch: 500, 100 }, 101 scheduler: { 102 // A claimed job whose worker stops sending heartbeats for this long is 103 // considered lost and is requeued or failed. 104 heartbeat_timeout: 120, 105 reap_interval: 30, 106 default_task_timeout: 3600, 107 max_attempts: 3, 108 }, 109 }; 110 111 // [environment variable, dotted config path, parser] 112 const ENV_MAP = [ 113 ['CONDUCTOR_RETENTION_ARTIFACT_JOBS', 'retention.artifact_keep_jobs', toInt], 114 ['CONDUCTOR_RETENTION_ARTIFACT_DAYS', 'retention.artifact_keep_days', toInt], 115 ['CONDUCTOR_RETENTION_LOG_DAYS', 'retention.log_keep_days', toInt], 116 ['CONDUCTOR_RETENTION_SWEEP_INTERVAL','retention.sweep_interval', toInt], 117 ['CONDUCTOR_HOST', 'server.host', String], 118 ['CONDUCTOR_PORT', 'server.port', toInt], 119 ['CONDUCTOR_PUBLIC_URL', 'server.public_url', String], 120 ['CONDUCTOR_DATABASE_URL', 'database.url', String], 121 ['CONDUCTOR_DATABASE_PATH', 'database.path', String], 122 ['CONDUCTOR_STORAGE_PATH', 'storage.path', String], 123 ['CONDUCTOR_S3_ENDPOINT', 'storage.s3.endpoint', String], 124 ['CONDUCTOR_S3_REGION', 'storage.s3.region', String], 125 ['CONDUCTOR_S3_BUCKET', 'storage.s3.bucket', String], 126 ['CONDUCTOR_S3_ACCESS_KEY_ID', 'storage.s3.access_key_id', String], 127 ['CONDUCTOR_S3_SECRET_ACCESS_KEY', 'storage.s3.secret_access_key', String], 128 ['CONDUCTOR_S3_FORCE_PATH_STYLE', 'storage.s3.force_path_style', toBool], 129 ['CONDUCTOR_SESSION_SECRET', 'auth.session_secret', String], 130 ['CONDUCTOR_OIDC_DISCOVERY_URL', 'auth.oidc.discovery_url', String], 131 ['CONDUCTOR_OIDC_CLIENT_ID', 'auth.oidc.client_id', String], 132 ['CONDUCTOR_OIDC_CLIENT_SECRET', 'auth.oidc.client_secret', String], 133 ['CONDUCTOR_OIDC_SCOPES', 'auth.oidc.scopes', String], 134 ['CONDUCTOR_OIDC_ADMIN_ROLE', 'auth.oidc.admin_role', String], 135 ['CONDUCTOR_ADMIN_USERNAME', 'auth.bootstrap_admin.username', String], 136 ['CONDUCTOR_ADMIN_PASSWORD', 'auth.bootstrap_admin.password', String], 137 ['CONDUCTOR_SECRET_KEY', 'secrets.encryption_key', String], 138 ['CONDUCTOR_MIRROR_PATH', 'git.mirror_path', String], 139 ['CONDUCTOR_LOG_PATH', 'log.spool_path', String], 140 ]; 141 142 // Paths resolved relative to the config file directory, or cwd when there is 143 // no config file. 144 const PATH_KEYS = ['database.path', 'storage.path', 'git.mirror_path', 'log.spool_path']; 145 146 function toInt(v) { 147 const n = parseInt(v, 10); 148 if (!Number.isFinite(n)) throw new Error(`expected an integer, got ${JSON.stringify(v)}`); 149 return n; 150 } 151 152 function toBool(v) { 153 if (typeof v === 'boolean') return v; 154 const s = String(v).toLowerCase(); 155 if (['1', 'true', 'yes', 'on'].includes(s)) return true; 156 if (['0', 'false', 'no', 'off'].includes(s)) return false; 157 throw new Error(`expected a boolean, got ${JSON.stringify(v)}`); 158 } 159 160 function isPlainObject(v) { 161 return v !== null && typeof v === 'object' && !Array.isArray(v); 162 } 163 164 function deepMerge(base, overlay) { 165 const out = Array.isArray(base) ? [...base] : { ...base }; 166 for (const [k, v] of Object.entries(overlay || {})) { 167 if (v === undefined) continue; 168 out[k] = isPlainObject(v) && isPlainObject(base?.[k]) ? deepMerge(base[k], v) : v; 169 } 170 return out; 171 } 172 173 function getPath(obj, dotted) { 174 return dotted.split('.').reduce((acc, k) => (acc == null ? acc : acc[k]), obj); 175 } 176 177 function setPath(obj, dotted, value) { 178 const keys = dotted.split('.'); 179 const last = keys.pop(); 180 let cur = obj; 181 for (const k of keys) { 182 if (!isPlainObject(cur[k])) cur[k] = {}; 183 cur = cur[k]; 184 } 185 cur[last] = value; 186 } 187 188 function deepFreeze(obj) { 189 for (const v of Object.values(obj)) { 190 if (isPlainObject(v) || Array.isArray(v)) deepFreeze(v); 191 } 192 return Object.freeze(obj); 193 } 194 195 // Reads a 32 byte key given as hex or base64. Returns a Buffer, or null. 196 export function parseKey(value, label) { 197 if (!value) return null; 198 let buf = null; 199 if (/^[0-9a-fA-F]{64}$/.test(value)) buf = Buffer.from(value, 'hex'); 200 else buf = Buffer.from(value, 'base64'); 201 if (buf.length !== 32) { 202 throw new Error(`${label} must decode to 32 bytes, got ${buf.length}; use 64 hex chars or base64`); 203 } 204 return buf; 205 } 206 207 function validate(cfg) { 208 const errors = []; 209 210 // Port 0 asks the operating system for a free port, which is useful in 211 // tests and for ephemeral instances. 212 const port = cfg.server.port; 213 if (!Number.isInteger(port) || port < 0 || port > 65535) { 214 errors.push(`server.port must be between 0 and 65535, got ${port}`); 215 } 216 if (cfg.database.url && dialectFromUrl(cfg.database.url) === null) { 217 errors.push( 218 `database.url has an unsupported scheme: ${String(cfg.database.url).split(':')[0]}; ` + 219 'use mysql://, postgres:// or leave it unset for sqlite' 220 ); 221 } 222 223 const s3 = cfg.storage.s3; 224 const s3Given = [s3.endpoint, s3.bucket, s3.access_key_id, s3.secret_access_key].filter(Boolean); 225 if (s3Given.length > 0 && s3Given.length < 4) { 226 errors.push( 227 'storage.s3 is partially configured; endpoint, bucket, access_key_id and ' + 228 'secret_access_key are all required, or leave them all unset for local storage' 229 ); 230 } 231 232 if (cfg.auth.oidc.discovery_url) { 233 try { 234 new URL(cfg.auth.oidc.discovery_url); 235 } catch { 236 errors.push( 237 `auth.oidc.discovery_url must be an absolute URL, got ${JSON.stringify(cfg.auth.oidc.discovery_url)}` 238 ); 239 } 240 if (!cfg.auth.oidc.client_id) { 241 errors.push('auth.oidc.client_id is required when auth.oidc.discovery_url is set'); 242 } 243 } else if (cfg.auth.oidc.client_id) { 244 errors.push('auth.oidc.client_id is set but auth.oidc.discovery_url is not'); 245 } 246 247 try { 248 new URL(cfg.server.public_url); 249 } catch { 250 errors.push(`server.public_url must be an absolute URL, got ${JSON.stringify(cfg.server.public_url)}`); 251 } 252 253 try { 254 parseKey(cfg.secrets.encryption_key, 'secrets.encryption_key'); 255 } catch (e) { 256 errors.push(e.message); 257 } 258 259 if (cfg.scheduler.heartbeat_timeout <= cfg.scheduler.reap_interval) { 260 errors.push( 261 `scheduler.heartbeat_timeout (${cfg.scheduler.heartbeat_timeout}) must be greater than ` + 262 `scheduler.reap_interval (${cfg.scheduler.reap_interval}), or healthy jobs will be reaped` 263 ); 264 } 265 266 // Retention deletes things, so a nonsensical value here is worth 267 // refusing at startup rather than discovering from missing artifacts. 268 for (const key of ['artifact_keep_jobs', 'artifact_keep_days', 'log_keep_days']) { 269 const value = cfg.retention[key]; 270 if (!Number.isInteger(value) || value < 0) { 271 errors.push(`retention.${key} must be a whole number of zero or more, got ${JSON.stringify(value)}`); 272 } 273 } 274 275 if (!Number.isInteger(cfg.retention.sweep_interval) || cfg.retention.sweep_interval < 60) { 276 errors.push( 277 `retention.sweep_interval must be at least 60 seconds, got ${JSON.stringify(cfg.retention.sweep_interval)}` 278 ); 279 } 280 281 if (errors.length > 0) { 282 throw new Error(`invalid configuration:\n - ${errors.join('\n - ')}`); 283 } 284 } 285 286 export function loadConfig(explicitPath) { 287 const file = explicitPath || process.env.CONDUCTOR_CONFIG || 'conductor.yaml'; 288 let cfg = structuredClone(DEFAULTS); 289 let baseDir = process.cwd(); 290 let source = null; 291 292 if (fs.existsSync(file)) { 293 const text = fs.readFileSync(file, 'utf8'); 294 let parsed; 295 try { 296 parsed = YAML.parse(text) || {}; 297 } catch (e) { 298 throw new Error(`failed to parse config file ${file}: ${e.message}`); 299 } 300 if (!isPlainObject(parsed)) { 301 throw new Error(`config file ${file} must contain a YAML mapping at the top level`); 302 } 303 cfg = deepMerge(cfg, parsed); 304 baseDir = path.dirname(path.resolve(file)); 305 source = path.resolve(file); 306 } else if (explicitPath || process.env.CONDUCTOR_CONFIG) { 307 throw new Error(`config file not found: ${file}`); 308 } 309 310 for (const [env, dotted, parse] of ENV_MAP) { 311 const raw = process.env[env]; 312 if (raw === undefined || raw === '') continue; 313 try { 314 setPath(cfg, dotted, parse(raw)); 315 } catch (e) { 316 throw new Error(`invalid value for ${env}: ${e.message}`); 317 } 318 } 319 320 // A sqlite: or file: url is just another way of spelling database.path. 321 // Collapse it now so that everything downstream sees one representation. 322 if (cfg.database.url && /^(sqlite|file):/i.test(cfg.database.url)) { 323 const raw = cfg.database.url.replace(/^(sqlite|file):(\/\/)?/i, ''); 324 if (!raw) throw new Error(`database.url ${cfg.database.url} does not contain a file path`); 325 cfg.database.path = raw; 326 cfg.database.url = null; 327 } 328 329 for (const key of PATH_KEYS) { 330 const v = getPath(cfg, key); 331 if (typeof v === 'string' && v.length > 0) setPath(cfg, key, path.resolve(baseDir, v)); 332 } 333 334 validate(cfg); 335 336 // Derived flags, so callers never re-implement the fallback rules. 337 cfg.source = source; 338 cfg.database.dialect = dialectFromUrl(cfg.database.url); 339 cfg.storage.driver = cfg.storage.s3.bucket ? 's3' : 'local'; 340 cfg.auth.mode = cfg.auth.oidc.discovery_url ? 'oidc' : 'local'; 341 342 if (!cfg.auth.session_secret) { 343 cfg.auth.session_secret = crypto.randomBytes(32).toString('hex'); 344 cfg.auth.session_secret_ephemeral = true; 345 } 346 347 return deepFreeze(cfg); 348 } 349 350 // Directories that must exist before a service starts. Which ones matter 351 // depends on the selected drivers, so this is derived rather than fixed. 352 export function stateDirs(cfg) { 353 const dirs = [cfg.git.mirror_path, cfg.log.spool_path]; 354 if (cfg.storage.driver === 'local') dirs.push(cfg.storage.path); 355 if (cfg.database.dialect === 'sqlite') dirs.push(path.dirname(cfg.database.path)); 356 return [...new Set(dirs)]; 357 } 358 359 export function ensureStateDirs(cfg) { 360 for (const dir of stateDirs(cfg)) fs.mkdirSync(dir, { recursive: true }); 361 }