harness.js (12150B)
1 // test/helpers/harness.js - a conductor on a temporary database 2 // 3 // Builds a real server against a real git repository and a real sqlite file, 4 // then drives it through fastify inject. No ports are bound and no 5 // background processes are started, so the tests stay deterministic. 6 7 import fs from 'node:fs/promises'; 8 import os from 'node:os'; 9 import net from 'node:net'; 10 import path from 'node:path'; 11 import crypto from 'node:crypto'; 12 import assert from 'node:assert/strict'; 13 import { execFile } from 'node:child_process'; 14 import { promisify } from 'node:util'; 15 import { loadConfig } from '../../src/lib/config.js'; 16 import { createServices, buildServer } from '../../src/conductor/app.js'; 17 18 const execFileAsync = promisify(execFile); 19 20 export const DEFAULT_PIPELINE = ` 21 version: 1 22 defaults: 23 image: debian:bookworm-slim 24 tasks: 25 lint: 26 script: [make lint] 27 build: 28 arch: [x86_64, aarch64] 29 script: ['./build.sh $ARCH'] 30 artifacts: 31 paths: [dist/**] 32 package: 33 needs: [build] 34 arch: [x86_64, aarch64] 35 matrix: 36 pkg: [musl] 37 requires: [sign-key] 38 script: ['./pkg.sh $MATRIX_PKG $ARCH'] 39 publish: 40 needs: [package, lint] 41 script: [./publish.sh] 42 `; 43 44 export async function createRepo(dir, { pipeline = DEFAULT_PIPELINE, configPath = '.conductor.yml' } = {}) { 45 await fs.mkdir(dir, { recursive: true }); 46 const git = (...args) => execFileAsync('git', ['-C', dir, ...args]); 47 48 await execFileAsync('git', ['init', '-q', '-b', 'main', dir]); 49 await git('config', 'user.email', 'test@example.invalid'); 50 await git('config', 'user.name', 'Test'); 51 52 await fs.mkdir(path.dirname(path.join(dir, configPath)), { recursive: true }); 53 if (pipeline !== null) await fs.writeFile(path.join(dir, configPath), pipeline); 54 await fs.writeFile(path.join(dir, 'README.md'), 'test repository\n'); 55 56 await git('add', '-A'); 57 await git('commit', '-q', '-m', 'initial commit'); 58 const { stdout } = await git('rev-parse', 'HEAD'); 59 return stdout.trim(); 60 } 61 62 export async function startHarness(options = {}) { 63 const root = await fs.mkdtemp(path.join(os.tmpdir(), 'conductor-it-')); 64 const repoDir = path.join(root, 'repo'); 65 // The file in the repository and the project's config_path are usually 66 // the same, but not when the point of the test is that they differ. 67 const sha = await createRepo(repoDir, { 68 ...options, 69 configPath: options.repoConfigPath ?? options.configPath ?? '.conductor.yml', 70 }); 71 72 const configFile = path.join(root, 'conductor.yaml'); 73 await fs.writeFile(configFile, [ 74 'server:', 75 // A real port and a matching public_url are needed whenever something 76 // outside the process, such as a worker, has to reach back in. 77 ` port: ${options.port ?? 0}`, 78 ` public_url: ${options.publicUrl ?? 'http://conductor.test'}`, 79 'database:', 80 ' path: ./state/conductor.db', 81 'storage:', 82 ' path: ./state/storage', 83 // Given an s3 block, artifacts and finished logs go there instead of 84 // to the local path above. 85 ...(options.s3 86 ? [ 87 ' s3:', 88 ` endpoint: ${options.s3.endpoint}`, 89 ` region: ${options.s3.region}`, 90 ` bucket: ${options.s3.bucket}`, 91 ` access_key_id: ${options.s3.access_key_id}`, 92 ` secret_access_key: ${options.s3.secret_access_key}`, 93 ` force_path_style: ${options.s3.force_path_style !== false}`, 94 ] 95 : []), 96 'git:', 97 ' mirror_path: ./state/mirrors', 98 ' fetch_interval: 0', 99 'log:', 100 ' spool_path: ./state/logs', 101 'secrets:', 102 ` encryption_key: ${crypto.randomBytes(32).toString('hex')}`, 103 'scheduler:', 104 ' heartbeat_timeout: 120', 105 ' reap_interval: 30', 106 ...(options.retention 107 ? ['retention:', ...Object.entries(options.retention).map(([k, v]) => ` ${k}: ${v}`)] 108 : []), 109 'auth:', 110 ' session_secret: test-session-secret-not-for-real-use', 111 ...(options.accessTtl ? [` access_ttl: ${options.accessTtl}`] : []), 112 ...(options.refreshTtl ? [` refresh_ttl: ${options.refreshTtl}`] : []), 113 ...(options.refreshSliding === undefined ? [] : [` refresh_sliding: ${options.refreshSliding}`]), 114 ...(options.redirectUris 115 ? [' local:', ` redirect_uris: ${JSON.stringify(options.redirectUris)}`] 116 : []), 117 ...(options.oidcDiscoveryUrl 118 ? [ 119 ' oidc:', 120 ` discovery_url: ${options.oidcDiscoveryUrl}`, 121 ` client_id: ${options.oidcClientId ?? 'conductor'}`, 122 ...(options.oidcScopes ? [` scopes: ${options.oidcScopes}`] : []), 123 ...(options.oidcAdminRole ? [` admin_role: ${options.oidcAdminRole}`] : []), 124 ] 125 : []), 126 ' bootstrap_admin:', 127 ` username: ${options.adminUsername ?? 'admin'}`, 128 ` password: ${options.adminPassword ?? 'bootstrap-password'}`, 129 '', 130 ].join('\n')); 131 132 const savedConfig = process.env.CONDUCTOR_CONFIG; 133 process.env.CONDUCTOR_CONFIG = configFile; 134 const cfg = loadConfig(); 135 if (savedConfig === undefined) delete process.env.CONDUCTOR_CONFIG; 136 else process.env.CONDUCTOR_CONFIG = savedConfig; 137 138 const services = await createServices(cfg, { 139 migrationLogger: () => {}, 140 logger: silentLogger(), 141 // scrypt is deliberately slow, so the admin account is only created for 142 // the tests that actually sign in. 143 bootstrap: options.bootstrap === true, 144 }); 145 const app = await buildServer(services, { logger: false }); 146 147 const project = await services.projects.create({ 148 id: 'demo', 149 name: 'Demo', 150 repo_url: repoDir, 151 trigger_secret: 'test-secret', 152 config_path: options.configPath ?? '.conductor.yml', 153 154 // Public by default so that tests about scheduling can read jobs back 155 // without signing in. Visibility itself is covered by its own suite, 156 // which creates private projects explicitly. 157 visibility: options.visibility ?? 'public', 158 owner_id: options.ownerId ?? null, 159 }, { allowLocalPath: true }); 160 161 const worker = await services.workerTokens.create('test-worker'); 162 163 return { 164 root, 165 repoDir, 166 sha, 167 cfg, 168 app, 169 services, 170 project, 171 worker, 172 auth: { authorization: `Bearer ${worker.token}` }, 173 174 // Commits a change and returns the new commit id. 175 async commit(files, message = 'update') { 176 for (const [name, content] of Object.entries(files)) { 177 await fs.mkdir(path.dirname(path.join(repoDir, name)), { recursive: true }); 178 await fs.writeFile(path.join(repoDir, name), content); 179 } 180 await execFileAsync('git', ['-C', repoDir, 'add', '-A']); 181 await execFileAsync('git', ['-C', repoDir, 'commit', '-q', '-m', message]); 182 const { stdout } = await execFileAsync('git', ['-C', repoDir, 'rev-parse', 'HEAD']); 183 return stdout.trim(); 184 }, 185 186 sign(body) { 187 return `sha256=${crypto.createHmac('sha256', 'test-secret').update(body).digest('hex')}`; 188 }, 189 190 async trigger(payload, { secret = 'test-secret' } = {}) { 191 const body = JSON.stringify(payload); 192 const signature = `sha256=${crypto.createHmac('sha256', secret).update(body).digest('hex')}`; 193 return app.inject({ 194 method: 'POST', 195 url: '/api/v1/projects/demo/trigger', 196 headers: { 'content-type': 'application/json', 'x-hub-signature-256': signature }, 197 payload: body, 198 }); 199 }, 200 201 // Starts a task the deliberate way, rather than by forwarding a push. 202 async createJob(payload, { secret = 'test-secret' } = {}) { 203 const body = JSON.stringify(payload); 204 const signature = `sha256=${crypto.createHmac('sha256', secret).update(body).digest('hex')}`; 205 return app.inject({ 206 method: 'POST', 207 url: '/api/v1/projects/demo/jobs', 208 headers: { ...this.auth, 'content-type': 'application/json', 'x-hub-signature-256': signature }, 209 payload: body, 210 }); 211 }, 212 213 // Waits for a job to leave the pending state (compilation complete). 214 // Reads directly from the database so visibility rules don't block us. 215 async waitForJob(jobId, { timeout = 10000 } = {}) { 216 const start = Date.now(); 217 while (Date.now() - start < timeout) { 218 const job = await services.db.get('SELECT * FROM jobs WHERE id = {id}', { id: jobId }); 219 if (job && job.state !== 'pending') return job; 220 await new Promise((r) => setTimeout(r, 100)); 221 } 222 throw new Error(`job ${jobId} did not leave pending state within ${timeout}ms`); 223 }, 224 225 async claim(body = {}) { 226 return app.inject({ 227 method: 'POST', 228 url: '/api/v1/tasks/claim', 229 headers: { ...this.auth, 'content-type': 'application/json' }, 230 payload: JSON.stringify(body), 231 }); 232 }, 233 234 // Drives the built-in provider's authorization code flow the way the 235 // application does, and returns the grant plus a ready bearer header. 236 async login(username = options.adminUsername ?? 'admin', password = options.adminPassword ?? 'bootstrap-password') { 237 const redirectUri = `${cfg.server.public_url}/callback`; 238 const verifier = crypto.randomBytes(32).toString('base64url'); 239 const query = new URLSearchParams({ 240 client_id: 'conductor', 241 redirect_uri: redirectUri, 242 response_type: 'code', 243 code_challenge: crypto.createHash('sha256').update(verifier).digest('base64url'), 244 code_challenge_method: 'S256', 245 state: 'harness', 246 }); 247 248 const authorized = await app.inject({ 249 method: 'POST', 250 url: `/auth/authorize?${query}`, 251 headers: { 'content-type': 'application/x-www-form-urlencoded' }, 252 payload: new URLSearchParams({ username, password }).toString(), 253 }); 254 if (authorized.statusCode !== 303) { 255 throw new Error(`authorize failed: ${authorized.statusCode} ${authorized.body.slice(0, 200)}`); 256 } 257 258 const code = new URL(authorized.headers.location).searchParams.get('code'); 259 const granted = await this.token({ 260 grant_type: 'authorization_code', 261 code, 262 redirect_uri: redirectUri, 263 code_verifier: verifier, 264 }); 265 if (granted.statusCode !== 200) { 266 throw new Error(`token exchange failed: ${granted.statusCode} ${granted.body.slice(0, 200)}`); 267 } 268 269 const grant = granted.json(); 270 return { ...grant, headers: { authorization: `Bearer ${grant.access_token}` } }; 271 }, 272 273 async token(body) { 274 return app.inject({ 275 method: 'POST', 276 url: '/auth/token', 277 headers: { 'content-type': 'application/x-www-form-urlencoded' }, 278 payload: new URLSearchParams(body).toString(), 279 }); 280 }, 281 282 async discovery() { 283 return app.inject({ method: 'GET', url: '/.well-known/openid-configuration' }); 284 }, 285 286 async stop() { 287 await app.close(); 288 await services.db.close(); 289 // Best effort: a container that ran as root may have left files the 290 // test process cannot remove, and that must not mask a real failure. 291 await fs.rm(root, { recursive: true, force: true }).catch(() => {}); 292 }, 293 }; 294 } 295 296 // Unwraps an API response, asserting the envelope on the way through, so 297 // every call site that reads a body also checks the shape it arrived in. 298 export function data(res) { 299 const body = res.json(); 300 assert.equal(body.ok, true, `expected ok:true, got ${res.statusCode} ${res.body.slice(0, 200)}`); 301 return body.data; 302 } 303 304 // The failure half. Returns { name, message, details } for asserting on. 305 export function apiError(res) { 306 const body = res.json(); 307 assert.equal(body.ok, false, `expected ok:false, got ${res.statusCode} ${res.body.slice(0, 200)}`); 308 return body.error; 309 } 310 311 function silentLogger() { 312 return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; 313 } 314 315 // Asks the kernel for a free port and releases it. Only needed because the 316 // public_url has to be known before the server is built. 317 export async function freePort() { 318 return new Promise((resolve, reject) => { 319 const server = net.createServer(); 320 server.unref(); 321 server.on('error', reject); 322 server.listen(0, '127.0.0.1', () => { 323 const { port } = server.address(); 324 server.close(() => resolve(port)); 325 }); 326 }); 327 }