retention.test.js (20103B)
1 // test/retention.test.js - deleting artifacts and logs once they are old 2 // 3 // This is the one part of the conductor whose task is to destroy data, so 4 // the rules are pinned down here rather than left to read from the 5 // implementation. Jobs and tasks are seeded directly with chosen 6 // timestamps, since waiting fourteen days for a test is not practical. 7 8 import test from 'node:test'; 9 import assert from 'node:assert/strict'; 10 import { startHarness } from './helpers/harness.js'; 11 12 const DAY = 24 * 60 * 60 * 1000; 13 const NOW = 1_800_000_000_000; 14 15 async function withHarness(options, fn) { 16 const h = await startHarness(options); 17 try { 18 return await fn(h); 19 } finally { 20 await h.stop(); 21 } 22 } 23 24 // Seeds a finished job with one task and one artifact, at a chosen age. 25 // Returns the identifiers so a test can assert on what survived. 26 async function seedRun(h, { 27 number, 28 ageDays, 29 state = 'success', 30 artifactExpiresAt = null, 31 logSize = 1024, 32 finished = true, 33 }) { 34 const db = h.services.db; 35 const at = NOW - ageDays * DAY; 36 const jobId = `job-${number}`; 37 const taskId = `${jobId}:build`; 38 const artifactId = `art-${number}`; 39 const artifactKey = `artifacts/${jobId}/build/out.txt`; 40 const logKey = `logs/${jobId}/build.log`; 41 42 await db.run( 43 `INSERT INTO jobs (id, project_id, number, ref, head_sha, trigger_type, 44 state, visibility, created_at, started_at, finished_at) 45 VALUES ({id}, {project}, {number}, {ref}, {sha}, 'push', 46 {state}, 'public', {at}, {at}, {finished})`, 47 { 48 id: jobId, 49 project: h.project.id, 50 number, 51 ref: 'refs/heads/main', 52 sha: 'a'.repeat(40), 53 state, 54 at, 55 finished: finished ? at : null, 56 } 57 ); 58 59 await db.run( 60 `INSERT INTO tasks (id, job_id, name, base_name, image, requires, spec, state, 61 allow_failure, attempt, max_attempts, timeout, 62 log_key, log_size, created_at, finished_at) 63 VALUES ({id}, {job}, 'build', 'build', 'alpine:3', '[]', '{}', {state}, 64 0, 1, 1, 3600, {logKey}, {logSize}, {at}, {finished})`, 65 { 66 id: taskId, 67 job: jobId, 68 state: finished ? state : 'running', 69 logKey, 70 logSize, 71 at, 72 finished: finished ? at : null, 73 } 74 ); 75 76 await db.run( 77 `INSERT INTO artifacts (id, task_id, job_id, path, storage_key, size, sha256, 78 created_at, expires_at) 79 VALUES ({id}, {task}, {job}, 'out.txt', {key}, 512, {sha}, {at}, {expires})`, 80 { 81 id: artifactId, 82 task: taskId, 83 job: jobId, 84 key: artifactKey, 85 sha: 'b'.repeat(64), 86 at, 87 expires: artifactExpiresAt, 88 } 89 ); 90 91 // Real objects, so a sweep that forgets to delete one is visible. 92 await h.services.storage.put(artifactKey, Buffer.from('artifact body')); 93 await h.services.storage.put(logKey, Buffer.from('log body')); 94 95 return { jobId, taskId, artifactId, artifactKey, logKey }; 96 } 97 98 async function artifactIds(h) { 99 const rows = await h.services.db.all('SELECT id FROM artifacts ORDER BY id', {}); 100 return rows.map((r) => r.id); 101 } 102 103 async function objectExists(h, key) { 104 try { 105 return (await h.services.storage.head(key)) !== null; 106 } catch { 107 return false; 108 } 109 } 110 111 test('an artifact past its own deadline is deleted', async () => { 112 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 0 } }, async (h) => { 113 const fresh = await seedRun(h, { number: 1, ageDays: 0, artifactExpiresAt: NOW + DAY }); 114 const stale = await seedRun(h, { number: 2, ageDays: 0, artifactExpiresAt: NOW - 1 }); 115 116 const result = await h.services.retention.sweep({ now: NOW }); 117 118 assert.equal(result.artifacts, 1); 119 assert.deepEqual(await artifactIds(h), ['art-1']); 120 assert.equal(await objectExists(h, fresh.artifactKey), true); 121 assert.equal(await objectExists(h, stale.artifactKey), false, 'the object must go too, not just the row'); 122 }); 123 }); 124 125 test('an explicit deadline overrides the policy in both directions', async () => { 126 // Shorter than the policy would allow, on the newest job. 127 await withHarness({ retention: { artifact_keep_jobs: 10, artifact_keep_days: 30 } }, async (h) => { 128 await seedRun(h, { number: 1, ageDays: 0, artifactExpiresAt: NOW - 1 }); 129 await h.services.retention.sweep({ now: NOW }); 130 assert.deepEqual(await artifactIds(h), [], 'a task may expire its artifacts early'); 131 }); 132 133 // Longer than the policy would allow, on a job far outside it. 134 await withHarness({ retention: { artifact_keep_jobs: 1, artifact_keep_days: 1 } }, async (h) => { 135 await seedRun(h, { number: 1, ageDays: 400, artifactExpiresAt: NOW + DAY }); 136 await seedRun(h, { number: 2, ageDays: 0 }); 137 await h.services.retention.sweep({ now: NOW }); 138 assert.ok((await artifactIds(h)).includes('art-1'), 'a task may keep its artifacts longer'); 139 }); 140 }); 141 142 test('an explicit deadline also beats the last good job', async () => { 143 // The protection exists so a quiet project keeps something downloadable. 144 // A task that asked for a short life is asking on purpose, and a bulky 145 // intermediate should not become immortal by being green. 146 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 0 } }, async (h) => { 147 await seedRun(h, { number: 1, ageDays: 0, state: 'success', artifactExpiresAt: NOW - 1 }); 148 await h.services.retention.sweep({ now: NOW }); 149 assert.deepEqual(await artifactIds(h), []); 150 }); 151 }); 152 153 test('the last few jobs keep their artifacts however old they are', async () => { 154 await withHarness({ retention: { artifact_keep_jobs: 3, artifact_keep_days: 0 } }, async (h) => { 155 for (let n = 1; n <= 6; n += 1) { 156 await seedRun(h, { number: n, ageDays: 400, state: n === 6 ? 'failed' : 'failed' }); 157 } 158 159 await h.services.retention.sweep({ now: NOW }); 160 161 // The three highest numbered jobs, regardless of age. 162 assert.deepEqual(await artifactIds(h), ['art-4', 'art-5', 'art-6']); 163 }); 164 }); 165 166 test('recent artifacts are kept however many jobs have followed', async () => { 167 // The two rules are combined by whichever keeps longer, so a job outside 168 // the count window still survives while it is inside the age window. 169 await withHarness({ retention: { artifact_keep_jobs: 2, artifact_keep_days: 30 } }, async (h) => { 170 for (let n = 1; n <= 5; n += 1) { 171 await seedRun(h, { number: n, ageDays: 1, state: 'failed' }); 172 } 173 174 await h.services.retention.sweep({ now: NOW }); 175 176 assert.equal((await artifactIds(h)).length, 5, 'all are young enough, whatever their position'); 177 }); 178 }); 179 180 test('an artifact outside both rules is deleted', async () => { 181 await withHarness({ retention: { artifact_keep_jobs: 2, artifact_keep_days: 7 } }, async (h) => { 182 const old = await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 183 await seedRun(h, { number: 2, ageDays: 1, state: 'failed' }); 184 await seedRun(h, { number: 3, ageDays: 1, state: 'failed' }); 185 186 const result = await h.services.retention.sweep({ now: NOW }); 187 188 assert.equal(result.artifacts, 1); 189 assert.deepEqual(await artifactIds(h), ['art-2', 'art-3']); 190 assert.equal(await objectExists(h, old.artifactKey), false); 191 }); 192 }); 193 194 test('the most recent successful job is kept whatever the policy says', async () => { 195 await withHarness({ retention: { artifact_keep_jobs: 1, artifact_keep_days: 1 } }, async (h) => { 196 await seedRun(h, { number: 1, ageDays: 400, state: 'success' }); 197 await seedRun(h, { number: 2, ageDays: 300, state: 'failed' }); 198 await seedRun(h, { number: 3, ageDays: 0, state: 'failed' }); 199 200 await h.services.retention.sweep({ now: NOW }); 201 202 // job 1 is the last green one, job 3 is inside both windows, and the 203 // failed job between them has nothing protecting it. 204 assert.deepEqual(await artifactIds(h), ['art-1', 'art-3']); 205 }); 206 }); 207 208 test('only the latest success is protected, not every success', async () => { 209 await withHarness({ retention: { artifact_keep_jobs: 1, artifact_keep_days: 1 } }, async (h) => { 210 await seedRun(h, { number: 1, ageDays: 400, state: 'success' }); 211 await seedRun(h, { number: 2, ageDays: 300, state: 'success' }); 212 await seedRun(h, { number: 3, ageDays: 0, state: 'failed' }); 213 214 await h.services.retention.sweep({ now: NOW }); 215 216 assert.deepEqual(await artifactIds(h), ['art-2', 'art-3']); 217 }); 218 }); 219 220 test('a project may keep artifacts forever despite a server default', async () => { 221 await withHarness({ retention: { artifact_keep_jobs: 1, artifact_keep_days: 1 } }, async (h) => { 222 await h.services.db.run( 223 'UPDATE projects SET artifact_keep_jobs = 0, artifact_keep_days = 0 WHERE id = {id}', 224 { id: h.project.id } 225 ); 226 227 await seedRun(h, { number: 1, ageDays: 900, state: 'failed' }); 228 await seedRun(h, { number: 2, ageDays: 900, state: 'failed' }); 229 230 const result = await h.services.retention.sweep({ now: NOW }); 231 232 // Zero has to mean forever rather than immediately, or a project could 233 // not opt out of a default that deletes things. 234 assert.equal(result.artifacts, 0); 235 assert.equal((await artifactIds(h)).length, 2); 236 }); 237 }); 238 239 test('a project setting overrides the server default', async () => { 240 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 365 } }, async (h) => { 241 await h.services.db.run( 242 'UPDATE projects SET artifact_keep_days = 7 WHERE id = {id}', 243 { id: h.project.id } 244 ); 245 246 await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 247 248 await h.services.retention.sweep({ now: NOW }); 249 assert.deepEqual(await artifactIds(h), [], 'the stricter project setting applies'); 250 }); 251 }); 252 253 test('an unfinished job is never swept', async () => { 254 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 1 } }, async (h) => { 255 // Backdated far enough to be well outside the window, but still going. 256 await seedRun(h, { number: 1, ageDays: 400, state: 'running', finished: false }); 257 258 const result = await h.services.retention.sweep({ now: NOW }); 259 260 assert.equal(result.artifacts, 0); 261 assert.equal(result.logs, 0); 262 assert.equal((await artifactIds(h)).length, 1); 263 }); 264 }); 265 266 test('logs older than the limit are removed but the task remains', async () => { 267 await withHarness({ retention: { log_keep_days: 14 } }, async (h) => { 268 const old = await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 269 const recent = await seedRun(h, { number: 2, ageDays: 1, state: 'failed' }); 270 271 const result = await h.services.retention.sweep({ now: NOW }); 272 273 assert.equal(result.logs, 1); 274 assert.equal(await objectExists(h, old.logKey), false); 275 assert.equal(await objectExists(h, recent.logKey), true); 276 277 // The task itself is history and stays, so a job remains explainable 278 // after its output has gone. 279 const task = await h.services.db.get( 280 'SELECT id, log_key, log_expired_at, state FROM tasks WHERE id = {id}', 281 { id: old.taskId } 282 ); 283 assert.ok(task, 'the task row must survive'); 284 assert.equal(task.log_key, null); 285 assert.equal(task.state, 'failed'); 286 assert.equal(task.log_expired_at, NOW); 287 }); 288 }); 289 290 test('a swept log is distinguishable from one that never existed', async () => { 291 await withHarness({ retention: { log_keep_days: 1 } }, async (h) => { 292 const swept = await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 293 await h.services.retention.sweep({ now: NOW }); 294 295 const gone = await h.services.db.get( 296 'SELECT log_key, log_expired_at FROM tasks WHERE id = {id}', { id: swept.taskId } 297 ); 298 // Both have no log_key, so the timestamp is the only thing telling 299 // "expired" apart from "never wrote anything". 300 assert.equal(gone.log_key, null); 301 assert.ok(gone.log_expired_at > 0); 302 }); 303 }); 304 305 test('a log is swept only once', async () => { 306 await withHarness({ retention: { log_keep_days: 1 } }, async (h) => { 307 await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 308 309 const first = await h.services.retention.sweep({ now: NOW }); 310 const second = await h.services.retention.sweep({ now: NOW }); 311 312 assert.equal(first.logs, 1); 313 assert.equal(second.logs, 0, 'a swept log must not be counted again on every pass'); 314 }); 315 }); 316 317 test('zero keeps logs forever', async () => { 318 await withHarness({ retention: { log_keep_days: 0 } }, async (h) => { 319 const old = await seedRun(h, { number: 1, ageDays: 900, state: 'failed' }); 320 const result = await h.services.retention.sweep({ now: NOW }); 321 322 assert.equal(result.logs, 0); 323 assert.equal(await objectExists(h, old.logKey), true); 324 }); 325 }); 326 327 test('a storage failure leaves the row for the next pass', async () => { 328 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 1 } }, async (h) => { 329 await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 330 331 const real = h.services.storage.delete; 332 h.services.storage.delete = async () => { throw new Error('bucket unreachable'); }; 333 334 const failed = await h.services.retention.sweep({ now: NOW }); 335 assert.equal(failed.artifacts, 0); 336 assert.equal((await artifactIds(h)).length, 1, 'the row must survive so the object is not orphaned'); 337 338 h.services.storage.delete = real; 339 340 const recovered = await h.services.retention.sweep({ now: NOW }); 341 assert.equal(recovered.artifacts, 1); 342 assert.deepEqual(await artifactIds(h), []); 343 }); 344 }); 345 346 test('a sweep deletes no more than its batch', async () => { 347 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 1 } }, async (h) => { 348 for (let n = 1; n <= 5; n += 1) { 349 await seedRun(h, { number: n, ageDays: 30, state: 'failed' }); 350 } 351 352 const first = await h.services.retention.sweep({ now: NOW, batch: 2 }); 353 assert.equal(first.artifacts, 2, 'a backlog must not be swept in one go'); 354 assert.equal((await artifactIds(h)).length, 3); 355 356 await h.services.retention.sweep({ now: NOW, batch: 10 }); 357 assert.equal((await artifactIds(h)).length, 0); 358 }); 359 }); 360 361 test('one project policy does not reach into another', async () => { 362 await withHarness({ retention: { artifact_keep_jobs: 0, artifact_keep_days: 1 } }, async (h) => { 363 await h.services.projects.create({ 364 id: 'other', 365 name: 'Other', 366 repo_url: h.repoDir, 367 trigger_secret: 'other-secret', 368 visibility: 'public', 369 }); 370 await h.services.db.run( 371 'UPDATE projects SET artifact_keep_days = 0 WHERE id = {id}', { id: 'other' } 372 ); 373 374 await seedRun(h, { number: 1, ageDays: 30, state: 'failed' }); 375 376 // An old artifact under the project that keeps everything. 377 await h.services.db.run( 378 `INSERT INTO jobs (id, project_id, number, ref, head_sha, trigger_type, 379 state, visibility, created_at, started_at, finished_at) 380 VALUES ('other-job', 'other', 1, 'refs/heads/main', {sha}, 'push', 381 'failed', 'public', {at}, {at}, {at})`, 382 { sha: 'c'.repeat(40), at: NOW - 900 * DAY } 383 ); 384 await h.services.db.run( 385 `INSERT INTO tasks (id, job_id, name, base_name, image, requires, spec, state, 386 allow_failure, attempt, max_attempts, timeout, log_size, 387 created_at, finished_at) 388 VALUES ('other-task', 'other-job', 'build', 'build', 'alpine:3', '[]', '{}', 389 'failed', 0, 1, 1, 3600, 0, {at}, {at})`, 390 { at: NOW - 900 * DAY } 391 ); 392 await h.services.db.run( 393 `INSERT INTO artifacts (id, task_id, job_id, path, storage_key, size, sha256, created_at) 394 VALUES ('art-other', 'other-task', 'other-job', 'out.txt', 395 'artifacts/other/out.txt', 1, {sha}, {at})`, 396 { sha: 'd'.repeat(64), at: NOW - 900 * DAY } 397 ); 398 399 await h.services.retention.sweep({ now: NOW }); 400 401 assert.deepEqual(await artifactIds(h), ['art-other']); 402 }); 403 }); 404 405 test('artifacts.expire in a pipeline reaches the stored artifact', async () => { 406 // The key has been in the schema since the beginning and was parsed, 407 // stored in the task spec, and then ignored by everything. This is the 408 // path from pipeline text to a deadline on the row. 409 const pipeline = [ 410 'version: 1', 411 'defaults:', 412 ' image: alpine:3', 413 'tasks:', 414 ' build:', 415 " script: ['make']", 416 ' artifacts:', 417 ' paths: [out/**]', 418 ' expire: 1h', 419 ' keep:', 420 " script: ['make']", 421 ' artifacts:', 422 ' paths: [out/**]', 423 '', 424 ].join('\n'); 425 426 await withHarness({ pipeline }, async (h) => { 427 const res = await h.trigger({ sha: h.sha, ref: 'refs/heads/main' }); 428 await h.waitForJob(res.json().job_id); 429 430 const before = Date.now(); 431 for (const name of ['build', 'keep']) { 432 const res = await h.claim({}); 433 assert.equal(res.statusCode, 200, `expected to claim ${name}`); 434 const task = res.json().task; 435 436 const upload = await h.app.inject({ 437 method: 'POST', 438 url: `/api/v1/tasks/${task.id}/artifacts`, 439 headers: { 440 ...h.auth, 441 'content-type': 'application/octet-stream', 442 'x-artifact-path': 'out/result.txt', 443 }, 444 payload: Buffer.from('packaged'), 445 }); 446 assert.equal(upload.statusCode, 200); 447 } 448 449 const rows = await h.services.db.all( 450 `SELECT t.base_name AS name, a.expires_at 451 FROM artifacts a JOIN tasks t ON t.id = a.task_id 452 ORDER BY t.base_name`, 453 {} 454 ); 455 456 const byName = Object.fromEntries(rows.map((r) => [r.name, r.expires_at])); 457 458 // An hour from when it was stored, give or take the time the test took. 459 assert.ok(byName.build >= before + 3600 * 1000, 'expire should be an hour out'); 460 assert.ok(byName.build <= Date.now() + 3600 * 1000); 461 462 // The task that said nothing is left to the project policy. 463 assert.equal(byName.keep, null); 464 }); 465 }); 466 467 test('the api reports and accepts project retention', { skip: 'the JSON API was removed' }, async () => { 468 await withHarness({ 469 bootstrap: true, 470 retention: { artifact_keep_jobs: 7, artifact_keep_days: 21, log_keep_days: 9 }, 471 }, async (h) => { 472 const login = await h.app.inject({ 473 method: 'POST', 474 url: '/api/auth/login', 475 payload: { username: 'admin', password: 'bootstrap-password' }, 476 }); 477 assert.equal(login.statusCode, 200); 478 const auth = { cookie: login.headers['set-cookie'].split(';')[0] }; 479 480 // The defaults have to be discoverable, or a null on a project is 481 // meaningless to whoever is reading it. 482 const defaults = await h.app.inject({ method: 'GET', url: '/api/retention', headers: auth }); 483 assert.equal(defaults.statusCode, 200); 484 assert.deepEqual(defaults.json().defaults, { 485 artifact_keep_jobs: 7, 486 artifact_keep_days: 21, 487 log_keep_days: 9, 488 }); 489 490 const before = await h.app.inject({ 491 method: 'GET', url: `/api/projects/${h.project.id}`, headers: auth, 492 }); 493 assert.equal(before.json().project.artifact_keep_days, null, 'unset means inherited'); 494 495 const patched = await h.app.inject({ 496 method: 'PATCH', 497 url: `/api/projects/${h.project.id}`, 498 headers: auth, 499 payload: { artifact_keep_jobs: 3, artifact_keep_days: 0, log_keep_days: 30 }, 500 }); 501 assert.equal(patched.statusCode, 200); 502 assert.equal(patched.json().project.artifact_keep_jobs, 3); 503 assert.equal(patched.json().project.artifact_keep_days, 0, 'zero is a value, not an absence'); 504 assert.equal(patched.json().project.log_keep_days, 30); 505 506 // And back to following the server default. 507 const cleared = await h.app.inject({ 508 method: 'PATCH', 509 url: `/api/projects/${h.project.id}`, 510 headers: auth, 511 payload: { artifact_keep_jobs: null }, 512 }); 513 assert.equal(cleared.json().project.artifact_keep_jobs, null); 514 515 const rejected = await h.app.inject({ 516 method: 'PATCH', 517 url: `/api/projects/${h.project.id}`, 518 headers: auth, 519 payload: { log_keep_days: -5 }, 520 }); 521 assert.equal(rejected.statusCode, 400); 522 assert.match(rejected.json().error, /zero or more/); 523 }); 524 });