conductor

CI task system
git clone git://git.finwo.net/app/conductor
Log | Files | Refs | README | LICENSE

git.js (9462B)

      1 // src/lib/git.js - repository mirrors
      2 //
      3 // The conductor keeps one bare mirror per project. Everything it needs from
      4 // a push is read out of that mirror: the pipeline file at the pushed commit,
      5 // the commit subject, and the source tarball handed to workers.
      6 //
      7 // Serving the source from here rather than having workers clone means an
      8 // untrusted worker needs no repository credentials and cannot reach any ref
      9 // other than the commit it was given work for. It also means the worker host
     10 // does not need git installed.
     11 //
     12 // Every value interpolated into a git invocation is validated first. Refs
     13 // and object ids arrive from webhooks, and a value beginning with a hyphen
     14 // would otherwise be read as an option.
     15 
     16 import fs from 'node:fs/promises';
     17 import path from 'node:path';
     18 import { execFile, spawn } from 'node:child_process';
     19 import { existsSync } from 'node:fs';
     20 import { promisify } from 'node:util';
     21 
     22 const execFileAsync = promisify(execFile);
     23 
     24 export const SHA_PATTERN = /^[0-9a-f]{40}$/;
     25 export const SHORT_SHA_PATTERN = /^[0-9a-f]{7,40}$/;
     26 
     27 // Reading a repository owned by somebody else is the normal case here, not
     28 // a suspicious one: project paths belong to whoever put them on disk, and
     29 // the conductor is a different user again once it runs in a container.
     30 // Git refuses that by default with "detected dubious ownership", which is
     31 // the right default for a shell and the wrong one for this.
     32 //
     33 // Applied through the environment rather than a global git config, so it
     34 // covers both spawn and execFile below and affects nothing outside this
     35 // process. Anything already set in the environment is preserved, since
     36 // the indices have to be contiguous for git to read them.
     37 function gitEnv(env = process.env) {
     38   const count = Number.parseInt(env.GIT_CONFIG_COUNT ?? '0', 10);
     39   const base = Number.isInteger(count) && count > 0 ? count : 0;
     40   return {
     41     ...env,
     42     // Refuses credential prompts, so a misconfigured private repository
     43     // fails promptly instead of hanging until the request times out.
     44     GIT_TERMINAL_PROMPT: '0',
     45     GIT_ASKPASS: '',
     46     GCM_INTERACTIVE: 'never',
     47     // Backstop for assertRepoUrl: blocks ext:: and every other helper
     48     // transport. file stays for local mirrors, which assertRepoUrl gates.
     49     GIT_ALLOW_PROTOCOL: 'file:git:ssh:https',
     50     GIT_PROTOCOL_FROM_USER: '0',
     51     LC_ALL: 'C',
     52     GIT_CONFIG_COUNT: String(base + 1),
     53     [`GIT_CONFIG_KEY_${base}`]: 'safe.directory',
     54     [`GIT_CONFIG_VALUE_${base}`]: '*',
     55   };
     56 }
     57 
     58 const GIT_ENV = gitEnv();
     59 
     60 export { gitEnv };
     61 
     62 export class GitError extends Error {
     63   constructor(message, { stderr, code } = {}) {
     64     super(stderr ? `${message}: ${String(stderr).trim().split('\n').slice(0, 3).join('; ')}` : message);
     65     this.name = 'GitError';
     66     this.stderr = stderr;
     67     this.exitCode = code;
     68   }
     69 }
     70 
     71 export function assertSha(sha, label = 'commit') {
     72   if (typeof sha !== 'string' || !SHA_PATTERN.test(sha)) {
     73     throw new GitError(`${label} must be a full 40 character hex object id, got ${JSON.stringify(sha)}`);
     74   }
     75   return sha;
     76 }
     77 
     78 // Refs come straight from a push hook. Reject anything git itself would
     79 // consider malformed, along with leading hyphens.
     80 export function assertRef(ref) {
     81   if (typeof ref !== 'string' || ref.length === 0 || ref.length > 255) {
     82     throw new GitError(`invalid ref: ${JSON.stringify(ref)}`);
     83   }
     84   if (ref.startsWith('-') || ref.includes('..') || /[\s~^:?*[\\]/.test(ref) || ref.endsWith('.lock')) {
     85     throw new GitError(`invalid ref: ${JSON.stringify(ref)}`);
     86   }
     87   return ref;
     88 }
     89 
     90 // A path inside the repository, used for the pipeline file.
     91 export function assertRepoPath(p) {
     92   if (typeof p !== 'string' || p.length === 0 || p.length > 512) {
     93     throw new GitError(`invalid repository path: ${JSON.stringify(p)}`);
     94   }
     95   if (p.startsWith('/') || p.startsWith('-') || p.split('/').some((s) => s === '' || s === '.' || s === '..')) {
     96     throw new GitError(`invalid repository path: ${JSON.stringify(p)}`);
     97   }
     98   return p;
     99 }
    100 
    101 // Serializes work per mirror, so two pushes to one project cannot run
    102 // concurrent fetches into the same directory.
    103 const locks = new Map();
    104 
    105 async function withLock(key, fn) {
    106   const previous = locks.get(key) ?? Promise.resolve();
    107   let release;
    108   const current = new Promise((resolve) => { release = resolve; });
    109   locks.set(key, previous.then(() => current));
    110   await previous;
    111   try {
    112     return await fn();
    113   } finally {
    114     release();
    115     if (locks.get(key) === current) locks.delete(key);
    116   }
    117 }
    118 
    119 export function createGit(cfg) {
    120   const timeout = cfg.git.timeout * 1000;
    121   const fetchInterval = cfg.git.fetch_interval * 1000;
    122   const lastFetch = new Map();
    123 
    124   function mirrorPath(projectId) {
    125     return path.join(cfg.git.mirror_path, `${projectId}.git`);
    126   }
    127 
    128   // Given --git-dir explicitly, git still treats the process working
    129   // directory as a work tree. That makes the result depend on whatever
    130   // happens to sit in the conductor's own directory: asking for a file
    131   // that is absent from a commit reports "exists on disk, but not in
    132   // <sha>" rather than a plain absence, purely because a file of that
    133   // name is next to the running process. Working inside the bare mirror
    134   // takes the stray work tree out of the picture.
    135   //
    136   // Only when it is already there, since clone is told --git-dir for a
    137   // directory it is about to create.
    138   function repoCwd(args) {
    139     const i = args.indexOf('--git-dir');
    140     if (i < 0) return undefined;
    141     const dir = args[i + 1];
    142     return dir && existsSync(dir) ? dir : undefined;
    143   }
    144 
    145   async function run(args, options = {}) {
    146     try {
    147       const { stdout, stderr } = await execFileAsync('git', args, {
    148         env: GIT_ENV,
    149         cwd: options.cwd ?? repoCwd(args),
    150         timeout,
    151         maxBuffer: options.maxBuffer ?? 16 * 1024 * 1024,
    152         encoding: options.encoding ?? 'utf8',
    153       });
    154       return { stdout, stderr };
    155     } catch (e) {
    156       throw new GitError(`git ${args[0]} failed`, { stderr: e.stderr || e.message, code: e.code });
    157     }
    158   }
    159 
    160   async function exists(dir) {
    161     try {
    162       await fs.stat(path.join(dir, 'HEAD'));
    163       return true;
    164     } catch {
    165       return false;
    166     }
    167   }
    168 
    169   return {
    170     mirrorPath,
    171 
    172     // Clones on first use, then fetches at most once per fetch_interval
    173     // unless force is set. A trigger always forces, since it needs the
    174     // commit that was just pushed.
    175     async sync(project, { force = false } = {}) {
    176       const dir = mirrorPath(project.id);
    177       return withLock(dir, async () => {
    178         if (!(await exists(dir))) {
    179           await fs.mkdir(path.dirname(dir), { recursive: true });
    180           await run(['clone', '--mirror', '--quiet', '--', project.repo_url, dir]);
    181           lastFetch.set(dir, Date.now());
    182           return dir;
    183         }
    184 
    185         const since = Date.now() - (lastFetch.get(dir) ?? 0);
    186         if (!force && since < fetchInterval) return dir;
    187 
    188         await run(['--git-dir', dir, 'fetch', '--prune', '--quiet', 'origin']);
    189         lastFetch.set(dir, Date.now());
    190         return dir;
    191       });
    192     },
    193 
    194     async hasCommit(project, sha) {
    195       assertSha(sha);
    196       try {
    197         const { stdout } = await run(['--git-dir', mirrorPath(project.id), 'cat-file', '-t', sha]);
    198         return stdout.trim() === 'commit';
    199       } catch {
    200         return false;
    201       }
    202     },
    203 
    204     async resolve(project, rev) {
    205       assertRef(rev);
    206       const { stdout } = await run(['--git-dir', mirrorPath(project.id), 'rev-parse', '--verify', `${rev}^{commit}`]);
    207       return stdout.trim();
    208     },
    209 
    210     // Reads one file at a commit. Returns null when the path is absent,
    211     // which is how a repository without a pipeline is detected.
    212     async readFile(project, sha, filePath) {
    213       assertSha(sha);
    214       assertRepoPath(filePath);
    215       try {
    216         const { stdout } = await run(
    217           ['--git-dir', mirrorPath(project.id), 'cat-file', 'blob', `${sha}:${filePath}`],
    218           { maxBuffer: 4 * 1024 * 1024 }
    219         );
    220         return stdout;
    221       } catch (e) {
    222         if (/does not exist|exists on disk, but not in|not a valid object name|Not a valid object/i.test(e.stderr ?? '')) return null;
    223         throw e;
    224       }
    225     },
    226 
    227     async commitInfo(project, sha) {
    228       assertSha(sha);
    229       // A unit separator keeps the fields unambiguous when a subject
    230       // contains anything at all.
    231       const { stdout } = await run([
    232         '--git-dir', mirrorPath(project.id),
    233         'show', '--no-patch', '--format=%s%x1f%an%x1f%ae%x1f%aI', sha,
    234       ]);
    235       const [subject = '', authorName = '', authorEmail = '', authoredAt = ''] = stdout.trim().split('\x1f');
    236       return { subject, authorName, authorEmail, authoredAt };
    237     },
    238 
    239     // A gzipped tar of the tree at a commit, streamed rather than buffered.
    240     // The caller pipes this straight to the worker.
    241     archiveStream(project, sha, { prefix = '' } = {}) {
    242       assertSha(sha);
    243       const args = ['--git-dir', mirrorPath(project.id), 'archive', '--format=tar.gz'];
    244       if (prefix) args.push(`--prefix=${prefix}`);
    245       args.push(sha);
    246 
    247       const child = spawn('git', args, { env: GIT_ENV, cwd: repoCwd(args), stdio: ['ignore', 'pipe', 'pipe'] });
    248       let stderr = '';
    249       child.stderr.on('data', (chunk) => { stderr += chunk.toString().slice(0, 4096); });
    250       child.on('close', (code) => {
    251         if (code !== 0) child.stdout.destroy(new GitError('git archive failed', { stderr, code }));
    252       });
    253       return child.stdout;
    254     },
    255   };
    256 }