conductor

CI task system
git clone git://git.finwo.net/app/conductor
Log | Files | Refs | README | LICENSE

containers.js (2301B)


      1 // test/helpers/containers.js - cleaning up after jobs that crashed
      2 //
      3 // A test job that is killed never reaches its cleanup hook and leaves its
      4 // containers behind, so each helper tidies up before it starts.
      5 //
      6 // Doing that by name prefix alone is wrong, and was: node --test jobs
      7 // test files in parallel processes, two files each start their own MinIO,
      8 // and whichever starts second force-removes the container the first is
      9 // using. The first then waits out its whole timeout and fails with the
     10 // container gone from under it. That showed up as a flake under load,
     11 // which is exactly the wrong thing to go looking for.
     12 //
     13 // So age is the thing that makes a container stale, not its name. A
     14 // sibling that started seconds ago is left alone; a leftover from a job
     15 // that died half an hour ago is removed.
     16 
     17 import { execFile } from 'node:child_process';
     18 import { promisify } from 'node:util';
     19 
     20 const execFileAsync = promisify(execFile);
     21 
     22 // Comfortably longer than any test takes, comfortably shorter than the
     23 // gap between one job and the next.
     24 export const STALE_AGE = 30 * 60 * 1000;
     25 
     26 export async function reapStale(prefix, maxAgeMs = STALE_AGE) {
     27   try {
     28     const { stdout } = await execFileAsync(
     29       'docker', ['ps', '-aq', '--filter', `name=${prefix}`], { timeout: 30000 },
     30     );
     31     const ids = stdout.trim().split('\n').filter(Boolean);
     32     if (ids.length === 0) return;
     33 
     34     // One call for all of them; .Created is RFC3339 and parses directly.
     35     const { stdout: detail } = await execFileAsync(
     36       'docker', ['inspect', '--format', '{{.Id}} {{.Created}}', ...ids], { timeout: 30000 },
     37     );
     38 
     39     const cutoff = Date.now() - maxAgeMs;
     40     const stale = detail.trim().split('\n')
     41       .map((line) => line.split(' '))
     42       .filter(([, created]) => {
     43         const at = Date.parse(created);
     44         // A timestamp that will not parse is not grounds for deleting
     45         // somebody's running container.
     46         return Number.isFinite(at) && at < cutoff;
     47       })
     48       .map(([id]) => id);
     49 
     50     if (stale.length > 0) {
     51       await execFileAsync('docker', ['rm', '-f', ...stale], { timeout: 60000 });
     52     }
     53   } catch {
     54     // Best effort. A failure here must not fail the tests, and whatever
     55     // is wrong will be reported more usefully by the start that follows.
     56   }
     57 }