harness.js (9898B)
1 // test/helpers/harness.js - a conductor on a temporary database 2 // 3 // Builds a real server against a real git repository and a real sqlite file, 4 // then drives it through fastify inject. No ports are bound and no 5 // background processes are started, so the tests stay deterministic. 6 7 import fs from 'node:fs/promises'; 8 import os from 'node:os'; 9 import net from 'node:net'; 10 import path from 'node:path'; 11 import crypto from 'node:crypto'; 12 import { execFile } from 'node:child_process'; 13 import { promisify } from 'node:util'; 14 import { loadConfig } from '../../src/lib/config.js'; 15 import { createServices, buildServer } from '../../src/conductor/app.js'; 16 17 const execFileAsync = promisify(execFile); 18 19 export const DEFAULT_PIPELINE = ` 20 version: 1 21 defaults: 22 image: debian:bookworm-slim 23 tasks: 24 lint: 25 script: [make lint] 26 build: 27 arch: [x86_64, aarch64] 28 script: ['./build.sh $ARCH'] 29 artifacts: 30 paths: [dist/**] 31 package: 32 needs: [build] 33 arch: [x86_64, aarch64] 34 matrix: 35 pkg: [musl] 36 requires: [sign-key] 37 script: ['./pkg.sh $MATRIX_PKG $ARCH'] 38 publish: 39 needs: [package, lint] 40 script: [./publish.sh] 41 `; 42 43 export async function createRepo(dir, { pipeline = DEFAULT_PIPELINE, configPath = '.conductor.yml' } = {}) { 44 await fs.mkdir(dir, { recursive: true }); 45 const git = (...args) => execFileAsync('git', ['-C', dir, ...args]); 46 47 await execFileAsync('git', ['init', '-q', '-b', 'main', dir]); 48 await git('config', 'user.email', 'test@example.invalid'); 49 await git('config', 'user.name', 'Test'); 50 51 await fs.mkdir(path.dirname(path.join(dir, configPath)), { recursive: true }); 52 if (pipeline !== null) await fs.writeFile(path.join(dir, configPath), pipeline); 53 await fs.writeFile(path.join(dir, 'README.md'), 'test repository\n'); 54 55 await git('add', '-A'); 56 await git('commit', '-q', '-m', 'initial commit'); 57 const { stdout } = await git('rev-parse', 'HEAD'); 58 return stdout.trim(); 59 } 60 61 export async function startHarness(options = {}) { 62 const root = await fs.mkdtemp(path.join(os.tmpdir(), 'conductor-it-')); 63 const repoDir = path.join(root, 'repo'); 64 // The file in the repository and the project's config_path are usually 65 // the same, but not when the point of the test is that they differ. 66 const sha = await createRepo(repoDir, { 67 ...options, 68 configPath: options.repoConfigPath ?? options.configPath ?? '.conductor.yml', 69 }); 70 71 const configFile = path.join(root, 'conductor.yaml'); 72 await fs.writeFile(configFile, [ 73 'server:', 74 // A real port and a matching public_url are needed whenever something 75 // outside the process, such as a worker, has to reach back in. 76 ` port: ${options.port ?? 0}`, 77 ` public_url: ${options.publicUrl ?? 'http://conductor.test'}`, 78 'database:', 79 ' path: ./state/conductor.db', 80 'storage:', 81 ' path: ./state/storage', 82 // Given an s3 block, artifacts and finished logs go there instead of 83 // to the local path above. 84 ...(options.s3 85 ? [ 86 ' s3:', 87 ` endpoint: ${options.s3.endpoint}`, 88 ` region: ${options.s3.region}`, 89 ` bucket: ${options.s3.bucket}`, 90 ` access_key_id: ${options.s3.access_key_id}`, 91 ` secret_access_key: ${options.s3.secret_access_key}`, 92 ` force_path_style: ${options.s3.force_path_style !== false}`, 93 ] 94 : []), 95 'git:', 96 ' mirror_path: ./state/mirrors', 97 ' fetch_interval: 0', 98 'log:', 99 ' spool_path: ./state/logs', 100 'secrets:', 101 ` encryption_key: ${crypto.randomBytes(32).toString('hex')}`, 102 'scheduler:', 103 ' heartbeat_timeout: 120', 104 ' reap_interval: 30', 105 ...(options.retention 106 ? ['retention:', ...Object.entries(options.retention).map(([k, v]) => ` ${k}: ${v}`)] 107 : []), 108 'auth:', 109 ' session_secret: test-session-secret-not-for-real-use', 110 ...(options.oidcDiscoveryUrl 111 ? [ 112 ' oidc:', 113 ` discovery_url: ${options.oidcDiscoveryUrl}`, 114 ` client_id: ${options.oidcClientId ?? 'conductor'}`, 115 ...(options.oidcClientSecret ? [` client_secret: ${options.oidcClientSecret}`] : []), 116 ...(options.oidcScopes ? [` scopes: ${options.oidcScopes}`] : []), 117 ...(options.oidcAdminRole ? [` admin_role: ${options.oidcAdminRole}`] : []), 118 ] 119 : []), 120 ' bootstrap_admin:', 121 ` username: ${options.adminUsername ?? 'admin'}`, 122 ` password: ${options.adminPassword ?? 'bootstrap-password'}`, 123 '', 124 ].join('\n')); 125 126 const savedConfig = process.env.CONDUCTOR_CONFIG; 127 process.env.CONDUCTOR_CONFIG = configFile; 128 const cfg = loadConfig(); 129 if (savedConfig === undefined) delete process.env.CONDUCTOR_CONFIG; 130 else process.env.CONDUCTOR_CONFIG = savedConfig; 131 132 const services = await createServices(cfg, { 133 migrationLogger: () => {}, 134 logger: silentLogger(), 135 // scrypt is deliberately slow, so the admin account is only created for 136 // the tests that actually sign in. 137 bootstrap: options.bootstrap === true, 138 }); 139 const app = await buildServer(services, { logger: false }); 140 141 const project = await services.projects.create({ 142 id: 'demo', 143 name: 'Demo', 144 repo_url: repoDir, 145 trigger_secret: 'test-secret', 146 config_path: options.configPath ?? '.conductor.yml', 147 148 // Public by default so that tests about scheduling can read jobs back 149 // without signing in. Visibility itself is covered by its own suite, 150 // which creates private projects explicitly. 151 visibility: options.visibility ?? 'public', 152 owner_id: options.ownerId ?? null, 153 }); 154 155 const worker = await services.workerTokens.create('test-worker'); 156 157 return { 158 root, 159 repoDir, 160 sha, 161 cfg, 162 app, 163 services, 164 project, 165 worker, 166 auth: { authorization: `Bearer ${worker.token}` }, 167 168 // Commits a change and returns the new commit id. 169 async commit(files, message = 'update') { 170 for (const [name, content] of Object.entries(files)) { 171 await fs.mkdir(path.dirname(path.join(repoDir, name)), { recursive: true }); 172 await fs.writeFile(path.join(repoDir, name), content); 173 } 174 await execFileAsync('git', ['-C', repoDir, 'add', '-A']); 175 await execFileAsync('git', ['-C', repoDir, 'commit', '-q', '-m', message]); 176 const { stdout } = await execFileAsync('git', ['-C', repoDir, 'rev-parse', 'HEAD']); 177 return stdout.trim(); 178 }, 179 180 sign(body) { 181 return `sha256=${crypto.createHmac('sha256', 'test-secret').update(body).digest('hex')}`; 182 }, 183 184 async trigger(payload, { secret = 'test-secret' } = {}) { 185 const body = JSON.stringify(payload); 186 const signature = `sha256=${crypto.createHmac('sha256', secret).update(body).digest('hex')}`; 187 return app.inject({ 188 method: 'POST', 189 url: '/api/v1/projects/demo/trigger', 190 headers: { 'content-type': 'application/json', 'x-hub-signature-256': signature }, 191 payload: body, 192 }); 193 }, 194 195 // Starts a task the deliberate way, rather than by forwarding a push. 196 async createJob(payload, { secret = 'test-secret' } = {}) { 197 const body = JSON.stringify(payload); 198 const signature = `sha256=${crypto.createHmac('sha256', secret).update(body).digest('hex')}`; 199 return app.inject({ 200 method: 'POST', 201 url: '/api/v1/projects/demo/jobs', 202 headers: { ...this.auth, 'content-type': 'application/json', 'x-hub-signature-256': signature }, 203 payload: body, 204 }); 205 }, 206 207 // Waits for a job to leave the pending state (compilation complete). 208 // Reads directly from the database so visibility rules don't block us. 209 async waitForJob(jobId, { timeout = 10000 } = {}) { 210 const start = Date.now(); 211 while (Date.now() - start < timeout) { 212 const job = await services.db.get('SELECT * FROM jobs WHERE id = {id}', { id: jobId }); 213 if (job && job.state !== 'pending') return job; 214 await new Promise((r) => setTimeout(r, 100)); 215 } 216 throw new Error(`job ${jobId} did not leave pending state within ${timeout}ms`); 217 }, 218 219 async claim(body = {}) { 220 return app.inject({ 221 method: 'POST', 222 url: '/api/v1/tasks/claim', 223 headers: { ...this.auth, 'content-type': 'application/json' }, 224 payload: JSON.stringify(body), 225 }); 226 }, 227 228 // Signs in through the interface, which is the only way in, and returns 229 // the session cookie. The interface's login form is htmx only. 230 async login(username = options.adminUsername ?? 'admin', password = options.adminPassword ?? 'bootstrap-password') { 231 const res = await app.inject({ 232 method: 'POST', 233 url: '/login', 234 headers: { 'hx-request': 'true', 'content-type': 'application/x-www-form-urlencoded' }, 235 payload: new URLSearchParams({ username, password }).toString(), 236 }); 237 if (res.statusCode !== 204) { 238 throw new Error(`login failed: ${res.statusCode} ${res.body}`); 239 } 240 return { cookie: res.headers['set-cookie'].split(';')[0] }; 241 }, 242 243 async stop() { 244 await app.close(); 245 await services.db.close(); 246 // Best effort: a container that ran as root may have left files the 247 // test process cannot remove, and that must not mask a real failure. 248 await fs.rm(root, { recursive: true, force: true }).catch(() => {}); 249 }, 250 }; 251 } 252 253 function silentLogger() { 254 return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; 255 } 256 257 // Asks the kernel for a free port and releases it. Only needed because the 258 // public_url has to be known before the server is built. 259 export async function freePort() { 260 return new Promise((resolve, reject) => { 261 const server = net.createServer(); 262 server.unref(); 263 server.on('error', reject); 264 server.listen(0, '127.0.0.1', () => { 265 const { port } = server.address(); 266 server.close(() => resolve(port)); 267 }); 268 }); 269 }