commit f997ab1673eacee94e95040b6a698507bd03eb6d parent 9099ead8e113aa11cd487852870f4c0cd3f96233 Author: finwo <finwo@pm.me> Date: Sun, 20 Sep 2026 03:24:29 +0200 Describe the schema as jobs and tasks, in one migration per dialect Diffstat:
21 files changed, 359 insertions(+), 783 deletions(-)
diff --git a/migrations/mysql/001_initial.sql b/migrations/mysql/001_initial.sql @@ -1,39 +1,93 @@ -- 001_initial.sql - base schema for conductor (mysql and tidb) -- --- Conventions, kept identical to the sqlite dialect: +-- Conventions, kept identical to the sqlite and postgres dialects: -- timestamps are BIGINT epoch milliseconds -- booleans are TINYINT 0 or 1 -- json documents are TEXT -- secrets are TEXT produced by src/lib/secretbox.js -- +-- Vocabulary. A trigger creates a job. Compiling the repository's pipeline +-- turns that job into one or more tasks, and a task is the standalone unit +-- a worker actually runs: one container, one script, one result. Workers +-- deal only in tasks and never learn which project or job a task came from. +-- -- Column widths are chosen so that every indexed column stays within the --- InnoDB 3072 byte key limit under utf8mb4. +-- InnoDB 3072 byte key limit under utf8mb4. Tables are declared in +-- dependency order, since a referenced table must exist already. + +CREATE TABLE users ( + id VARCHAR(64) NOT NULL PRIMARY KEY, + username VARCHAR(191) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + -- Two roles: 'admin' and 'user'. A user is not read only; they may own + -- projects and workers. Administration is the thing being gated, not + -- participation. + role VARCHAR(32) NOT NULL DEFAULT 'user', + disabled TINYINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL, + last_login_at BIGINT, + -- With OIDC the provider owns identity, but the conductor still needs a + -- local row per person: projects and worker tokens reference users(id), + -- so a user who exists only inside a token cannot own anything. The + -- account is created on first sight of a valid token, keyed by issuer + -- and subject rather than by the display name, which a provider is free + -- to change. + external_id VARCHAR(255), + UNIQUE KEY idx_users_external (external_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE projects ( id VARCHAR(64) NOT NULL PRIMARY KEY, name VARCHAR(255) NOT NULL, repo_url TEXT NOT NULL, default_branch VARCHAR(255) NOT NULL DEFAULT 'main', + -- The preferred pipeline file. When it is one of the conventional names + -- the other extension is accepted as a fallback, so a repository may + -- spell it .conductor.yml or .conductor.yaml without being configured. config_path VARCHAR(255) NOT NULL DEFAULT '.conductor.yml', - -- How workers obtain the source: 'archive' has them download a tarball - -- from the conductor, 'clone' has them git clone repo_url themselves. - source_mode VARCHAR(16) NOT NULL DEFAULT 'archive', -- Shared secret for trigger HMAC verification, encrypted at rest. trigger_secret TEXT, enabled TINYINT NOT NULL DEFAULT 1, - run_counter BIGINT NOT NULL DEFAULT 0, + job_counter BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE users ( - id VARCHAR(64) NOT NULL PRIMARY KEY, - username VARCHAR(191) NOT NULL UNIQUE, - password_hash TEXT NOT NULL, - role VARCHAR(32) NOT NULL DEFAULT 'viewer', - disabled TINYINT NOT NULL DEFAULT 0, - created_at BIGINT NOT NULL, - last_login_at BIGINT + updated_at BIGINT NOT NULL, + -- A user may register their own projects and their own workers. A worker + -- that has an owner is only ever offered tasks belonging to that owner's + -- projects, so lending someone build capacity exposes nothing else. A + -- worker with no owner is shared and can run any project's tasks. + -- + -- Deleting a user deletes what they owned, cascading on to their jobs, + -- tasks and artifacts. Leaving the rows behind unowned would be worse + -- than losing them, since an unowned worker token is shared capacity: a + -- deleted account would quietly widen a worker's reach, not remove it. + owner_id VARCHAR(64), + visibility VARCHAR(16) NOT NULL DEFAULT 'private', + -- Retention. NULL is "no opinion", and the server default applies. Zero + -- means keep forever, which has to be distinguishable from unset or a + -- project could not opt out of a server default that deletes things. + -- + -- Artifacts have two rules and an artifact survives if either wants it: + -- the last artifact_keep_jobs jobs are kept however old they are, and + -- anything younger than artifact_keep_days is kept however many jobs + -- have followed it. The artifacts of the most recent successful job are + -- kept regardless, so a project that has gone quiet still has something + -- to download. A task that set artifacts.expire in the pipeline overrides + -- all of it with an exact deadline, in either direction, since a bulky + -- intermediate is worth dropping early even from a green build. + -- + -- Logs are simpler: they go by age alone. + artifact_keep_jobs INT, + artifact_keep_days INT, + log_keep_days INT, + -- Where a task's tree is unpacked inside its container. Three answers, + -- most specific first: the workdir key in the repository's pipeline, + -- this column, and failing both the server default of /work. The + -- repository wins because the path belongs with the code: an image that + -- expects to build in /usr/src/app knows that, and whoever registered + -- the project should not have to. + workdir VARCHAR(1024), + KEY idx_projects_owner (owner_id), + CONSTRAINT fk_projects_owner FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE worker_tokens ( @@ -43,10 +97,13 @@ CREATE TABLE worker_tokens ( enabled TINYINT NOT NULL DEFAULT 1, created_at BIGINT NOT NULL, last_seen_at BIGINT, - last_ip VARCHAR(64) + last_ip VARCHAR(64), + owner_id VARCHAR(64), + KEY idx_worker_tokens_owner (owner_id), + CONSTRAINT fk_worker_tokens_owner FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE runs ( +CREATE TABLE jobs ( id VARCHAR(64) NOT NULL PRIMARY KEY, project_id VARCHAR(64) NOT NULL, number BIGINT NOT NULL, @@ -58,31 +115,42 @@ CREATE TABLE runs ( actor VARCHAR(255), title TEXT, state VARCHAR(16) NOT NULL DEFAULT 'pending', - -- Resolved pipeline as scheduled, retained so a run stays explainable + -- Resolved pipeline as scheduled, retained so a job stays explainable -- after the repository moves on. pipeline MEDIUMTEXT, error TEXT, created_at BIGINT NOT NULL, started_at BIGINT, finished_at BIGINT, - CONSTRAINT fk_runs_project FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, - UNIQUE KEY idx_runs_project_number (project_id, number), - KEY idx_runs_created (created_at), - KEY idx_runs_state (state) + -- A job is private unless it says otherwise. The value comes from the + -- project, and the pipeline at the built commit may override it, so a + -- repository decides whether its own results are public. + visibility VARCHAR(16) NOT NULL DEFAULT 'private', + CONSTRAINT fk_jobs_project FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + UNIQUE KEY idx_jobs_project_number (project_id, number), + KEY idx_jobs_created (created_at), + KEY idx_jobs_state (state), + KEY idx_jobs_visibility (visibility) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE jobs ( - id VARCHAR(96) NOT NULL PRIMARY KEY, - run_id VARCHAR(64) NOT NULL, +CREATE TABLE tasks ( + -- Opaque and time ordered. A task id deliberately carries no structure: + -- a worker holding one learns nothing about the job or project it + -- belongs to, and the id stays usable in a URL, a container name and a + -- storage key without escaping. + id VARCHAR(64) NOT NULL PRIMARY KEY, + job_id VARCHAR(64) NOT NULL, -- Expanded name, for example 'package:pkg=musl'. name VARCHAR(191) NOT NULL, -- Template name before matrix and arch expansion. base_name VARCHAR(191) NOT NULL, arch VARCHAR(32), image TEXT NOT NULL, - -- json array of worker feature names this job needs. + -- json array of worker feature names this task needs. requires TEXT NOT NULL, - -- json document: script, env, services, artifacts, cache, matrix values. + -- json document: script, env, services, artifacts, matrix values, the + -- resolved needs and depth, and the workdir settled when the job was + -- created so a retry cannot land somewhere else. spec MEDIUMTEXT NOT NULL, state VARCHAR(16) NOT NULL DEFAULT 'queued', allow_failure TINYINT NOT NULL DEFAULT 0, @@ -93,6 +161,9 @@ CREATE TABLE jobs ( error TEXT, log_key VARCHAR(255), log_size BIGINT NOT NULL DEFAULT 0, + -- Distinguishes a log that was swept from one that never existed, so the + -- interface can say which without guessing from an empty log_key. + log_expired_at BIGINT, worker_token_id VARCHAR(64), worker_name VARCHAR(255), claimed_at BIGINT, @@ -100,36 +171,40 @@ CREATE TABLE jobs ( created_at BIGINT NOT NULL, started_at BIGINT, finished_at BIGINT, - CONSTRAINT fk_jobs_run FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE, - UNIQUE KEY idx_jobs_run_name (run_id, name), - KEY idx_jobs_run (run_id), - KEY idx_jobs_state (state), - -- Supports the reaper scan for claimed jobs that stopped reporting. - KEY idx_jobs_heartbeat (state, heartbeat_at) + CONSTRAINT fk_tasks_job FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE, + UNIQUE KEY idx_tasks_job_name (job_id, name), + KEY idx_tasks_job (job_id), + KEY idx_tasks_state (state), + -- Supports the reaper scan for claimed tasks that stopped reporting. + KEY idx_tasks_heartbeat (state, heartbeat_at), + -- The sweep walks finished tasks oldest first, per project. + KEY idx_tasks_log_sweep (log_expired_at, finished_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE job_deps ( - job_id VARCHAR(96) NOT NULL, - depends_on_id VARCHAR(96) NOT NULL, - PRIMARY KEY (job_id, depends_on_id), - KEY idx_job_deps_reverse (depends_on_id), - CONSTRAINT fk_job_deps_job FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE, - CONSTRAINT fk_job_deps_target FOREIGN KEY (depends_on_id) REFERENCES jobs(id) ON DELETE CASCADE +CREATE TABLE task_deps ( + task_id VARCHAR(64) NOT NULL, + depends_on_id VARCHAR(64) NOT NULL, + PRIMARY KEY (task_id, depends_on_id), + KEY idx_task_deps_reverse (depends_on_id), + CONSTRAINT fk_task_deps_task FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE, + CONSTRAINT fk_task_deps_target FOREIGN KEY (depends_on_id) REFERENCES tasks(id) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE artifacts ( id VARCHAR(64) NOT NULL PRIMARY KEY, - job_id VARCHAR(96) NOT NULL, - run_id VARCHAR(64) NOT NULL, + task_id VARCHAR(64) NOT NULL, + -- Denormalized so the retention sweep and the download path can filter + -- by job without joining through tasks. + job_id VARCHAR(64) NOT NULL, path VARCHAR(1024) NOT NULL, storage_key VARCHAR(1024) NOT NULL, size BIGINT NOT NULL, sha256 CHAR(64) NOT NULL, created_at BIGINT NOT NULL, expires_at BIGINT, - CONSTRAINT fk_artifacts_job FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE, + CONSTRAINT fk_artifacts_task FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE, + KEY idx_artifacts_task (task_id), KEY idx_artifacts_job (job_id), - KEY idx_artifacts_run (run_id), KEY idx_artifacts_expires (expires_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/migrations/mysql/002_ownership.sql b/migrations/mysql/002_ownership.sql @@ -1,18 +0,0 @@ --- 002_ownership.sql - project ownership and run visibility (mysql and tidb) --- --- See migrations/sqlite/002_ownership.sql for the reasoning. - -ALTER TABLE projects - ADD COLUMN owner_id VARCHAR(64) NULL, - ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private', - ADD KEY idx_projects_owner (owner_id), - ADD CONSTRAINT fk_projects_owner FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE; - -ALTER TABLE worker_tokens - ADD COLUMN owner_id VARCHAR(64) NULL, - ADD KEY idx_worker_tokens_owner (owner_id), - ADD CONSTRAINT fk_worker_tokens_owner FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE; - -ALTER TABLE runs - ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private', - ADD KEY idx_runs_visibility (visibility); diff --git a/migrations/mysql/003_external_accounts.sql b/migrations/mysql/003_external_accounts.sql @@ -1,7 +0,0 @@ --- 003_external_accounts.sql - accounts provisioned by an identity provider --- --- See migrations/sqlite/003_external_accounts.sql for the reasoning. - -ALTER TABLE users - ADD COLUMN external_id VARCHAR(255) NULL, - ADD UNIQUE KEY idx_users_external (external_id); diff --git a/migrations/mysql/004_retention.sql b/migrations/mysql/004_retention.sql @@ -1,12 +0,0 @@ --- 004_retention.sql - how long artifacts and logs are kept --- --- See migrations/sqlite/004_retention.sql for the reasoning. - -ALTER TABLE projects - ADD COLUMN artifact_keep_runs INT NULL, - ADD COLUMN artifact_keep_days INT NULL, - ADD COLUMN log_keep_days INT NULL; - -ALTER TABLE jobs - ADD COLUMN log_expired_at BIGINT NULL, - ADD KEY idx_jobs_log_sweep (log_expired_at, finished_at); diff --git a/migrations/mysql/005_drop_source_mode.sql b/migrations/mysql/005_drop_source_mode.sql @@ -1,5 +0,0 @@ --- 005_drop_source_mode.sql - one way to get the source, not two --- --- See migrations/sqlite/005_drop_source_mode.sql for the reasoning. - -ALTER TABLE projects DROP COLUMN source_mode; diff --git a/migrations/mysql/006_project_workdir.sql b/migrations/mysql/006_project_workdir.sql @@ -1,5 +0,0 @@ --- 006_project_workdir.sql - where a job's tree is put in its container --- --- See migrations/sqlite/006_project_workdir.sql for the reasoning. - -ALTER TABLE projects ADD COLUMN workdir VARCHAR(1024) NULL; diff --git a/migrations/postgres/001_initial.sql b/migrations/postgres/001_initial.sql @@ -6,33 +6,90 @@ -- so that a row read back looks the same on every dialect -- json documents are TEXT, not JSONB, since nothing queries inside them -- secrets are TEXT produced by src/lib/secretbox.js +-- +-- Vocabulary. A trigger creates a job. Compiling the repository's pipeline +-- turns that job into one or more tasks, and a task is the standalone unit +-- a worker actually runs: one container, one script, one result. Workers +-- deal only in tasks and never learn which project or job a task came from. +-- +-- Tables are declared in dependency order, since a referenced table must +-- exist already. + +CREATE TABLE users ( + id TEXT NOT NULL PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + -- Two roles: 'admin' and 'user'. A user is not read only; they may own + -- projects and workers. Administration is the thing being gated, not + -- participation. + role TEXT NOT NULL DEFAULT 'user', + disabled SMALLINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL, + last_login_at BIGINT, + -- With OIDC the provider owns identity, but the conductor still needs a + -- local row per person: projects and worker tokens reference users(id), + -- so a user who exists only inside a token cannot own anything. The + -- account is created on first sight of a valid token, keyed by issuer + -- and subject rather than by the display name, which a provider is free + -- to change. + external_id TEXT +); + +CREATE UNIQUE INDEX idx_users_external ON users (external_id); CREATE TABLE projects ( id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, repo_url TEXT NOT NULL, default_branch TEXT NOT NULL DEFAULT 'main', + -- The preferred pipeline file. When it is one of the conventional names + -- the other extension is accepted as a fallback, so a repository may + -- spell it .conductor.yml or .conductor.yaml without being configured. config_path TEXT NOT NULL DEFAULT '.conductor.yml', - -- How workers obtain the source: 'archive' has them download a tarball - -- from the conductor, 'clone' has them git clone repo_url themselves. - source_mode TEXT NOT NULL DEFAULT 'archive', -- Shared secret for trigger HMAC verification, encrypted at rest. trigger_secret TEXT, enabled SMALLINT NOT NULL DEFAULT 1, - run_counter BIGINT NOT NULL DEFAULT 0, + job_counter BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL + updated_at BIGINT NOT NULL, + -- A user may register their own projects and their own workers. A worker + -- that has an owner is only ever offered tasks belonging to that owner's + -- projects, so lending someone build capacity exposes nothing else. A + -- worker with no owner is shared and can run any project's tasks. + -- + -- Deleting a user deletes what they owned, cascading on to their jobs, + -- tasks and artifacts. Leaving the rows behind unowned would be worse + -- than losing them, since an unowned worker token is shared capacity: a + -- deleted account would quietly widen a worker's reach, not remove it. + owner_id TEXT REFERENCES users(id) ON DELETE CASCADE, + visibility TEXT NOT NULL DEFAULT 'private', + -- Retention. NULL is "no opinion", and the server default applies. Zero + -- means keep forever, which has to be distinguishable from unset or a + -- project could not opt out of a server default that deletes things. + -- + -- Artifacts have two rules and an artifact survives if either wants it: + -- the last artifact_keep_jobs jobs are kept however old they are, and + -- anything younger than artifact_keep_days is kept however many jobs + -- have followed it. The artifacts of the most recent successful job are + -- kept regardless, so a project that has gone quiet still has something + -- to download. A task that set artifacts.expire in the pipeline overrides + -- all of it with an exact deadline, in either direction, since a bulky + -- intermediate is worth dropping early even from a green build. + -- + -- Logs are simpler: they go by age alone. + artifact_keep_jobs INTEGER, + artifact_keep_days INTEGER, + log_keep_days INTEGER, + -- Where a task's tree is unpacked inside its container. Three answers, + -- most specific first: the workdir key in the repository's pipeline, + -- this column, and failing both the server default of /work. The + -- repository wins because the path belongs with the code: an image that + -- expects to build in /usr/src/app knows that, and whoever registered + -- the project should not have to. + workdir TEXT ); -CREATE TABLE users ( - id TEXT NOT NULL PRIMARY KEY, - username TEXT NOT NULL UNIQUE, - password_hash TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'viewer', - disabled SMALLINT NOT NULL DEFAULT 0, - created_at BIGINT NOT NULL, - last_login_at BIGINT -); +CREATE INDEX idx_projects_owner ON projects (owner_id); CREATE TABLE worker_tokens ( id TEXT NOT NULL PRIMARY KEY, @@ -41,10 +98,13 @@ CREATE TABLE worker_tokens ( enabled SMALLINT NOT NULL DEFAULT 1, created_at BIGINT NOT NULL, last_seen_at BIGINT, - last_ip TEXT + last_ip TEXT, + owner_id TEXT REFERENCES users(id) ON DELETE CASCADE ); -CREATE TABLE runs ( +CREATE INDEX idx_worker_tokens_owner ON worker_tokens (owner_id); + +CREATE TABLE jobs ( id TEXT NOT NULL PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, number BIGINT NOT NULL, @@ -56,31 +116,42 @@ CREATE TABLE runs ( actor TEXT, title TEXT, state TEXT NOT NULL DEFAULT 'pending', - -- Resolved pipeline as scheduled, retained so a run stays explainable + -- Resolved pipeline as scheduled, retained so a job stays explainable -- after the repository moves on. pipeline TEXT, error TEXT, created_at BIGINT NOT NULL, started_at BIGINT, - finished_at BIGINT + finished_at BIGINT, + -- A job is private unless it says otherwise. The value comes from the + -- project, and the pipeline at the built commit may override it, so a + -- repository decides whether its own results are public. + visibility TEXT NOT NULL DEFAULT 'private' ); -CREATE UNIQUE INDEX idx_runs_project_number ON runs (project_id, number); -CREATE INDEX idx_runs_created ON runs (created_at); -CREATE INDEX idx_runs_state ON runs (state); +CREATE UNIQUE INDEX idx_jobs_project_number ON jobs (project_id, number); +CREATE INDEX idx_jobs_created ON jobs (created_at); +CREATE INDEX idx_jobs_state ON jobs (state); +CREATE INDEX idx_jobs_visibility ON jobs (visibility); -CREATE TABLE jobs ( +CREATE TABLE tasks ( + -- Opaque and time ordered. A task id deliberately carries no structure: + -- a worker holding one learns nothing about the job or project it + -- belongs to, and the id stays usable in a URL, a container name and a + -- storage key without escaping. id TEXT NOT NULL PRIMARY KEY, - run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, -- Expanded name, for example 'package:pkg=musl'. name TEXT NOT NULL, -- Template name before matrix and arch expansion. base_name TEXT NOT NULL, arch TEXT, image TEXT NOT NULL, - -- json array of worker feature names this job needs. + -- json array of worker feature names this task needs. requires TEXT NOT NULL, - -- json document: script, env, services, artifacts, cache, matrix values. + -- json document: script, env, services, artifacts, matrix values, the + -- resolved needs and depth, and the workdir settled when the job was + -- created so a retry cannot land somewhere else. spec TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'queued', allow_failure SMALLINT NOT NULL DEFAULT 0, @@ -91,6 +162,9 @@ CREATE TABLE jobs ( error TEXT, log_key TEXT, log_size BIGINT NOT NULL DEFAULT 0, + -- Distinguishes a log that was swept from one that never existed, so the + -- interface can say which without guessing from an empty log_key. + log_expired_at BIGINT, worker_token_id TEXT, worker_name TEXT, claimed_at BIGINT, @@ -100,34 +174,38 @@ CREATE TABLE jobs ( finished_at BIGINT ); -CREATE UNIQUE INDEX idx_jobs_run_name ON jobs (run_id, name); -CREATE INDEX idx_jobs_run ON jobs (run_id); -CREATE INDEX idx_jobs_state ON jobs (state); --- Supports the reaper scan for claimed jobs that stopped reporting. -CREATE INDEX idx_jobs_heartbeat ON jobs (state, heartbeat_at); +CREATE UNIQUE INDEX idx_tasks_job_name ON tasks (job_id, name); +CREATE INDEX idx_tasks_job ON tasks (job_id); +CREATE INDEX idx_tasks_state ON tasks (state); +-- Supports the reaper scan for claimed tasks that stopped reporting. +CREATE INDEX idx_tasks_heartbeat ON tasks (state, heartbeat_at); +-- The sweep walks finished tasks oldest first, per project. +CREATE INDEX idx_tasks_log_sweep ON tasks (log_expired_at, finished_at); -CREATE TABLE job_deps ( - job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - depends_on_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - PRIMARY KEY (job_id, depends_on_id) +CREATE TABLE task_deps ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) ); -CREATE INDEX idx_job_deps_reverse ON job_deps (depends_on_id); +CREATE INDEX idx_task_deps_reverse ON task_deps (depends_on_id); CREATE TABLE artifacts ( - id TEXT NOT NULL PRIMARY KEY, - job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - run_id TEXT NOT NULL, - path TEXT NOT NULL, - storage_key TEXT NOT NULL, - size BIGINT NOT NULL, + id TEXT NOT NULL PRIMARY KEY, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + -- Denormalized so the retention sweep and the download path can filter + -- by job without joining through tasks. + job_id TEXT NOT NULL, + path TEXT NOT NULL, + storage_key TEXT NOT NULL, + size BIGINT NOT NULL, sha256 CHAR(64) NOT NULL, - created_at BIGINT NOT NULL, + created_at BIGINT NOT NULL, expires_at BIGINT ); +CREATE INDEX idx_artifacts_task ON artifacts (task_id); CREATE INDEX idx_artifacts_job ON artifacts (job_id); -CREATE INDEX idx_artifacts_run ON artifacts (run_id); CREATE INDEX idx_artifacts_expires ON artifacts (expires_at); CREATE TABLE project_variables ( diff --git a/migrations/postgres/002_ownership.sql b/migrations/postgres/002_ownership.sql @@ -1,17 +0,0 @@ --- 002_ownership.sql - project ownership and run visibility (postgres) --- --- See migrations/sqlite/002_ownership.sql for the reasoning. - -ALTER TABLE projects - ADD COLUMN owner_id TEXT REFERENCES users(id) ON DELETE CASCADE, - ADD COLUMN visibility TEXT NOT NULL DEFAULT 'private'; - -ALTER TABLE worker_tokens - ADD COLUMN owner_id TEXT REFERENCES users(id) ON DELETE CASCADE; - -ALTER TABLE runs - ADD COLUMN visibility TEXT NOT NULL DEFAULT 'private'; - -CREATE INDEX idx_projects_owner ON projects (owner_id); -CREATE INDEX idx_worker_tokens_owner ON worker_tokens (owner_id); -CREATE INDEX idx_runs_visibility ON runs (visibility); diff --git a/migrations/postgres/003_external_accounts.sql b/migrations/postgres/003_external_accounts.sql @@ -1,7 +0,0 @@ --- 003_external_accounts.sql - accounts provisioned by an identity provider --- --- See migrations/sqlite/003_external_accounts.sql for the reasoning. - -ALTER TABLE users ADD COLUMN external_id TEXT; - -CREATE UNIQUE INDEX idx_users_external ON users (external_id); diff --git a/migrations/postgres/004_retention.sql b/migrations/postgres/004_retention.sql @@ -1,11 +0,0 @@ --- 004_retention.sql - how long artifacts and logs are kept --- --- See migrations/sqlite/004_retention.sql for the reasoning. - -ALTER TABLE projects ADD COLUMN artifact_keep_runs INTEGER; -ALTER TABLE projects ADD COLUMN artifact_keep_days INTEGER; -ALTER TABLE projects ADD COLUMN log_keep_days INTEGER; - -ALTER TABLE jobs ADD COLUMN log_expired_at BIGINT; - -CREATE INDEX idx_jobs_log_sweep ON jobs (log_expired_at, finished_at); diff --git a/migrations/postgres/005_drop_source_mode.sql b/migrations/postgres/005_drop_source_mode.sql @@ -1,5 +0,0 @@ --- 005_drop_source_mode.sql - one way to get the source, not two --- --- See migrations/sqlite/005_drop_source_mode.sql for the reasoning. - -ALTER TABLE projects DROP COLUMN source_mode; diff --git a/migrations/postgres/006_project_workdir.sql b/migrations/postgres/006_project_workdir.sql @@ -1,5 +0,0 @@ --- 006_project_workdir.sql - where a job's tree is put in its container --- --- See migrations/sqlite/006_project_workdir.sql for the reasoning. - -ALTER TABLE projects ADD COLUMN workdir TEXT; diff --git a/migrations/sqlite/001_initial.sql b/migrations/sqlite/001_initial.sql @@ -1,37 +1,94 @@ -- 001_initial.sql - base schema for conductor (sqlite) -- --- Conventions, kept identical to the mysql dialect: +-- Conventions, kept identical to the mysql and postgres dialects: -- timestamps are INTEGER epoch milliseconds -- booleans are INTEGER 0 or 1 -- json documents are TEXT -- secrets are TEXT produced by src/lib/secretbox.js +-- +-- Vocabulary. A trigger creates a job. Compiling the repository's pipeline +-- turns that job into one or more tasks, and a task is the standalone unit +-- a worker actually runs: one container, one script, one result. Workers +-- deal only in tasks and never learn which project or job a task came from. +-- +-- Tables are declared in dependency order, since mysql and postgres both +-- require a referenced table to exist already. + +CREATE TABLE users ( + id TEXT NOT NULL PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + -- Two roles: 'admin' and 'user'. A user is not read only; they may own + -- projects and workers. Administration is the thing being gated, not + -- participation. + role TEXT NOT NULL DEFAULT 'user', + disabled INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_login_at INTEGER, + -- With OIDC the provider owns identity, but the conductor still needs a + -- local row per person: projects and worker tokens reference users(id), + -- so a user who exists only inside a token cannot own anything. The + -- account is created on first sight of a valid token, keyed by issuer + -- and subject rather than by the display name, which a provider is free + -- to change. + external_id TEXT +); + +CREATE UNIQUE INDEX idx_users_external ON users (external_id); CREATE TABLE projects ( id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, repo_url TEXT NOT NULL, default_branch TEXT NOT NULL DEFAULT 'main', + -- The preferred pipeline file. When it is one of the conventional names + -- the other extension is accepted as a fallback, so a repository may + -- spell it .conductor.yml or .conductor.yaml without being configured. config_path TEXT NOT NULL DEFAULT '.conductor.yml', - -- How workers obtain the source: 'archive' has them download a tarball - -- from the conductor, 'clone' has them git clone repo_url themselves. - source_mode TEXT NOT NULL DEFAULT 'archive', -- Shared secret for trigger HMAC verification, encrypted at rest. trigger_secret TEXT, enabled INTEGER NOT NULL DEFAULT 1, - run_counter INTEGER NOT NULL DEFAULT 0, + job_counter INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + -- A user may register their own projects and their own workers. A worker + -- that has an owner is only ever offered tasks belonging to that owner's + -- projects, so lending someone build capacity exposes nothing else. A + -- worker with no owner is shared and can run any project's tasks. + -- + -- Deleting a user deletes what they owned, cascading on to their jobs, + -- tasks and artifacts. Leaving the rows behind unowned would be worse + -- than losing them, since an unowned worker token is shared capacity: a + -- deleted account would quietly widen a worker's reach, not remove it. + owner_id TEXT REFERENCES users(id) ON DELETE CASCADE, + visibility TEXT NOT NULL DEFAULT 'private', + -- Retention. NULL is "no opinion", and the server default applies. Zero + -- means keep forever, which has to be distinguishable from unset or a + -- project could not opt out of a server default that deletes things. + -- + -- Artifacts have two rules and an artifact survives if either wants it: + -- the last artifact_keep_jobs jobs are kept however old they are, and + -- anything younger than artifact_keep_days is kept however many jobs + -- have followed it. The artifacts of the most recent successful job are + -- kept regardless, so a project that has gone quiet still has something + -- to download. A task that set artifacts.expire in the pipeline overrides + -- all of it with an exact deadline, in either direction, since a bulky + -- intermediate is worth dropping early even from a green build. + -- + -- Logs are simpler: they go by age alone. + artifact_keep_jobs INTEGER, + artifact_keep_days INTEGER, + log_keep_days INTEGER, + -- Where a task's tree is unpacked inside its container. Three answers, + -- most specific first: the workdir key in the repository's pipeline, + -- this column, and failing both the server default of /work. The + -- repository wins because the path belongs with the code: an image that + -- expects to build in /usr/src/app knows that, and whoever registered + -- the project should not have to. + workdir TEXT ); -CREATE TABLE users ( - id TEXT NOT NULL PRIMARY KEY, - username TEXT NOT NULL UNIQUE, - password_hash TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'viewer', - disabled INTEGER NOT NULL DEFAULT 0, - created_at INTEGER NOT NULL, - last_login_at INTEGER -); +CREATE INDEX idx_projects_owner ON projects (owner_id); CREATE TABLE worker_tokens ( id TEXT NOT NULL PRIMARY KEY, @@ -40,10 +97,13 @@ CREATE TABLE worker_tokens ( enabled INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, last_seen_at INTEGER, - last_ip TEXT + last_ip TEXT, + owner_id TEXT REFERENCES users(id) ON DELETE CASCADE ); -CREATE TABLE runs ( +CREATE INDEX idx_worker_tokens_owner ON worker_tokens (owner_id); + +CREATE TABLE jobs ( id TEXT NOT NULL PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, number INTEGER NOT NULL, @@ -55,31 +115,42 @@ CREATE TABLE runs ( actor TEXT, title TEXT, state TEXT NOT NULL DEFAULT 'pending', - -- Resolved pipeline as scheduled, retained so a run stays explainable + -- Resolved pipeline as scheduled, retained so a job stays explainable -- after the repository moves on. pipeline TEXT, error TEXT, created_at INTEGER NOT NULL, started_at INTEGER, - finished_at INTEGER + finished_at INTEGER, + -- A job is private unless it says otherwise. The value comes from the + -- project, and the pipeline at the built commit may override it, so a + -- repository decides whether its own results are public. + visibility TEXT NOT NULL DEFAULT 'private' ); -CREATE UNIQUE INDEX idx_runs_project_number ON runs (project_id, number); -CREATE INDEX idx_runs_created ON runs (created_at); -CREATE INDEX idx_runs_state ON runs (state); +CREATE UNIQUE INDEX idx_jobs_project_number ON jobs (project_id, number); +CREATE INDEX idx_jobs_created ON jobs (created_at); +CREATE INDEX idx_jobs_state ON jobs (state); +CREATE INDEX idx_jobs_visibility ON jobs (visibility); -CREATE TABLE jobs ( +CREATE TABLE tasks ( + -- Opaque and time ordered. A task id deliberately carries no structure: + -- a worker holding one learns nothing about the job or project it + -- belongs to, and the id stays usable in a URL, a container name and a + -- storage key without escaping. id TEXT NOT NULL PRIMARY KEY, - run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, -- Expanded name, for example 'package:pkg=musl'. name TEXT NOT NULL, -- Template name before matrix and arch expansion. base_name TEXT NOT NULL, arch TEXT, image TEXT NOT NULL, - -- json array of worker feature names this job needs. + -- json array of worker feature names this task needs. requires TEXT NOT NULL, - -- json document: script, env, services, artifacts, cache, matrix values. + -- json document: script, env, services, artifacts, matrix values, the + -- resolved needs and depth, and the workdir settled when the job was + -- created so a retry cannot land somewhere else. spec TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'queued', allow_failure INTEGER NOT NULL DEFAULT 0, @@ -90,6 +161,9 @@ CREATE TABLE jobs ( error TEXT, log_key TEXT, log_size INTEGER NOT NULL DEFAULT 0, + -- Distinguishes a log that was swept from one that never existed, so the + -- interface can say which without guessing from an empty log_key. + log_expired_at INTEGER, worker_token_id TEXT, worker_name TEXT, claimed_at INTEGER, @@ -99,24 +173,28 @@ CREATE TABLE jobs ( finished_at INTEGER ); -CREATE UNIQUE INDEX idx_jobs_run_name ON jobs (run_id, name); -CREATE INDEX idx_jobs_run ON jobs (run_id); -CREATE INDEX idx_jobs_state ON jobs (state); --- Supports the reaper scan for claimed jobs that stopped reporting. -CREATE INDEX idx_jobs_heartbeat ON jobs (state, heartbeat_at); +CREATE UNIQUE INDEX idx_tasks_job_name ON tasks (job_id, name); +CREATE INDEX idx_tasks_job ON tasks (job_id); +CREATE INDEX idx_tasks_state ON tasks (state); +-- Supports the reaper scan for claimed tasks that stopped reporting. +CREATE INDEX idx_tasks_heartbeat ON tasks (state, heartbeat_at); +-- The sweep walks finished tasks oldest first, per project. +CREATE INDEX idx_tasks_log_sweep ON tasks (log_expired_at, finished_at); -CREATE TABLE job_deps ( - job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - depends_on_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - PRIMARY KEY (job_id, depends_on_id) +CREATE TABLE task_deps ( + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + depends_on_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + PRIMARY KEY (task_id, depends_on_id) ); -CREATE INDEX idx_job_deps_reverse ON job_deps (depends_on_id); +CREATE INDEX idx_task_deps_reverse ON task_deps (depends_on_id); CREATE TABLE artifacts ( id TEXT NOT NULL PRIMARY KEY, - job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, - run_id TEXT NOT NULL, + task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE, + -- Denormalized so the retention sweep and the download path can filter + -- by job without joining through tasks. + job_id TEXT NOT NULL, path TEXT NOT NULL, storage_key TEXT NOT NULL, size INTEGER NOT NULL, @@ -125,8 +203,8 @@ CREATE TABLE artifacts ( expires_at INTEGER ); +CREATE INDEX idx_artifacts_task ON artifacts (task_id); CREATE INDEX idx_artifacts_job ON artifacts (job_id); -CREATE INDEX idx_artifacts_run ON artifacts (run_id); CREATE INDEX idx_artifacts_expires ON artifacts (expires_at); CREATE TABLE project_variables ( diff --git a/migrations/sqlite/002_ownership.sql b/migrations/sqlite/002_ownership.sql @@ -1,32 +0,0 @@ --- 002_ownership.sql - project ownership and run visibility (sqlite) --- --- Two changes that go together: --- --- Ownership. A user may register their own projects and their own --- workers. A worker that has an owner will only ever be offered jobs --- belonging to that owner's projects, so lending someone build capacity --- does not expose anything else. A worker with no owner is shared and --- can run any project, which is how an administrator provides general --- capacity. --- --- Visibility. A run is private unless it says otherwise. The value comes --- from the project, and the pipeline at the built commit may override it, --- so a repository decides whether its own results are public. --- --- Deleting a user deletes what they owned, cascading on to their runs, --- jobs and artifacts. Leaving the rows behind unowned would be worse than --- losing them: an unowned project is administered centrally, and an --- unowned worker token is shared capacity that accepts every project's --- jobs, so a deleted account would quietly widen a worker's reach rather --- than removing it. - -ALTER TABLE projects ADD COLUMN owner_id TEXT REFERENCES users(id) ON DELETE CASCADE; -ALTER TABLE projects ADD COLUMN visibility TEXT NOT NULL DEFAULT 'private'; - -ALTER TABLE worker_tokens ADD COLUMN owner_id TEXT REFERENCES users(id) ON DELETE CASCADE; - -ALTER TABLE runs ADD COLUMN visibility TEXT NOT NULL DEFAULT 'private'; - -CREATE INDEX idx_projects_owner ON projects (owner_id); -CREATE INDEX idx_worker_tokens_owner ON worker_tokens (owner_id); -CREATE INDEX idx_runs_visibility ON runs (visibility); diff --git a/migrations/sqlite/003_external_accounts.sql b/migrations/sqlite/003_external_accounts.sql @@ -1,15 +0,0 @@ --- 003_external_accounts.sql - accounts provisioned by an identity provider --- --- With OIDC the provider owns identity, but the conductor still needs a --- local row for each person: projects and worker tokens reference users(id), --- so a user who exists only inside a token cannot own anything. --- --- An account is therefore created the first time someone presents a valid --- token, keyed by issuer and subject rather than by the display name, which --- a provider is free to change. The local row carries the role most --- recently seen in a token, so revoking a role at the provider takes effect --- on the next request rather than requiring a change here as well. - -ALTER TABLE users ADD COLUMN external_id TEXT; - -CREATE UNIQUE INDEX idx_users_external ON users (external_id); diff --git a/migrations/sqlite/004_retention.sql b/migrations/sqlite/004_retention.sql @@ -1,32 +0,0 @@ --- 004_retention.sql - how long artifacts and logs are kept --- --- Without this everything a run produces is kept forever, and a conductor --- that has been busy for a year is mostly old build output nobody will --- read again. --- --- Each setting is NULL when the project has no opinion, in which case the --- server default applies. Zero means keep forever, which has to be --- distinguishable from "unset" or a project could not opt out of a server --- default that deletes things. --- --- Artifacts have two rules and an artifact survives if either wants it: --- the last artifact_keep_runs runs are kept however old they are, and --- anything younger than artifact_keep_days is kept however many runs have --- followed it. The artifacts of the most recent successful run are kept --- regardless, so a project that has gone quiet still has something to --- download. A job that set artifacts.expire in the pipeline overrides all --- of that with an exact deadline, in either direction, since a bulky --- intermediate is worth dropping early even from a green build. --- --- Logs are simpler: they go by age alone. - -ALTER TABLE projects ADD COLUMN artifact_keep_runs INTEGER; -ALTER TABLE projects ADD COLUMN artifact_keep_days INTEGER; -ALTER TABLE projects ADD COLUMN log_keep_days INTEGER; - --- Distinguishes a log that was swept from one that never existed, so the --- interface can say which without guessing from an empty log_key. -ALTER TABLE jobs ADD COLUMN log_expired_at INTEGER; - --- The sweep walks finished jobs oldest first, per project. -CREATE INDEX idx_jobs_log_sweep ON jobs (log_expired_at, finished_at); diff --git a/migrations/sqlite/005_drop_source_mode.sql b/migrations/sqlite/005_drop_source_mode.sql @@ -1,18 +0,0 @@ --- 005_drop_source_mode.sql - one way to get the source, not two --- --- A project could choose between having the worker download a tarball of --- the commit from the conductor, or clone the repository itself. Clone --- mode is gone. --- --- It existed to save bandwidth on large repositories, and cost a great --- deal for it: the worker needed git, needed credentials for the --- repository, and could reach any ref rather than only the commit it was --- given work for. The tarball has none of those properties, and is now --- streamed straight into the job container, so the worker needs no --- working directory of its own either. --- --- The column is dropped rather than left in place, since a setting that --- is still offered and no longer does anything is worse than one that is --- gone. - -ALTER TABLE projects DROP COLUMN source_mode; diff --git a/migrations/sqlite/006_project_workdir.sql b/migrations/sqlite/006_project_workdir.sql @@ -1,16 +0,0 @@ --- 006_project_workdir.sql - where a job's tree is put in its container --- --- The tree used to be bind mounted from the worker at a path the worker --- chose. It is now unpacked inside the container instead, which means the --- path is a property of the build rather than of the machine running it, --- and is worth being able to choose. --- --- Three answers, most specific first: the workdir key in the repository's --- pipeline, this column, and failing both the server default of /work. --- Null means the project has no opinion. --- --- A repository may override its project because the path belongs with the --- code: an image that expects to build in /usr/src/app knows that, and --- whoever registered the project should not have to. - -ALTER TABLE projects ADD COLUMN workdir TEXT; diff --git a/src/conductor/routes/trigger.js b/src/conductor/routes/trigger.js @@ -1,147 +0,0 @@ -// src/conductor/routes/trigger.js - push notifications -// -// Accepts the generic payload produced by hooks/post-receive, and the push -// payloads of GitHub, Gitea and GitLab, since they all carry the same three -// facts. Nothing in the payload is trusted beyond which commit to look at: -// the pipeline is always read from the repository at that commit. -// -// A project with a trigger secret requires a valid signature. A project -// without one accepts unsigned triggers, which is convenient on a private -// network and a bad idea anywhere else, so it is logged. - -import crypto from 'node:crypto'; -import { SHA_PATTERN } from '../../lib/git.js'; -import { PipelineError } from '../../lib/pipeline/index.js'; - -const ZERO_SHA = '0'.repeat(40); - -export default async function triggerRoutes(fastify, { projects, scheduler, logger }) { - // HMAC is computed over the exact bytes received, so the raw body has to - // survive JSON parsing. - fastify.addContentTypeParser('application/json', { parseAs: 'buffer' }, (req, body, done) => { - req.rawBody = body; - if (body.length === 0) return done(null, {}); - try { - done(null, JSON.parse(body.toString('utf8'))); - } catch (e) { - e.statusCode = 400; - done(e); - } - }); - - fastify.post('/projects/:project/trigger', async (req, reply) => { - const project = await projects.get(req.params.project); - if (!project) return reply.code(404).send({ error: 'unknown project' }); - if (project.enabled !== 1) return reply.code(409).send({ error: 'project is disabled' }); - - const secret = projects.triggerSecret(project); - if (secret) { - if (!verifySignature(req, secret)) { - return reply.code(401).send({ error: 'invalid or missing signature' }); - } - } else { - logger.warn?.(`project ${project.id} accepted an unsigned trigger; set a trigger secret`); - } - - const push = normalizePush(req.body); - if (!push) { - return reply.code(400).send({ - error: 'could not read a commit from the payload; expected sha, or a GitHub, Gitea or GitLab push event', - }); - } - if (push.sha === ZERO_SHA) { - return reply.send({ status: 'ignored', reason: 'branch deletion' }); - } - if (!SHA_PATTERN.test(push.sha)) { - return reply.code(400).send({ error: `not a full commit id: ${JSON.stringify(push.sha)}` }); - } - - try { - const { runId, jobCount } = await scheduler.createRun(project, { - ref: push.ref, - baseSha: push.base, - headSha: push.sha, - trigger: 'push', - actor: push.actor, - }); - return reply.send({ status: 'created', run_id: runId, jobs: jobCount }); - } catch (e) { - if (e instanceof PipelineError) { - // A broken pipeline is the pusher's problem, not a server fault. - return reply.code(422).send({ error: 'invalid pipeline', detail: e.message, problems: e.errors }); - } - req.log.error({ err: e }, `trigger failed for ${project.id}`); - return reply.code(500).send({ error: 'could not create run', detail: String(e.message ?? e) }); - } - }); -} - -function verifySignature(req, secret) { - const raw = req.rawBody ?? Buffer.alloc(0); - - // GitHub and Gitea: sha256=<hex> over the body. - const hubSignature = req.headers['x-hub-signature-256']; - if (typeof hubSignature === 'string' && hubSignature.length > 0) { - const expected = `sha256=${crypto.createHmac('sha256', secret).update(raw).digest('hex')}`; - return timingSafeEqual(expected, hubSignature); - } - - // GitLab: the secret itself, compared rather than signed. - const gitlabToken = req.headers['x-gitlab-token']; - if (typeof gitlabToken === 'string' && gitlabToken.length > 0) { - return timingSafeEqual(secret, gitlabToken); - } - - return false; -} - -function timingSafeEqual(a, b) { - const left = Buffer.from(String(a), 'utf8'); - const right = Buffer.from(String(b), 'utf8'); - if (left.length !== right.length) return false; - return crypto.timingSafeEqual(left, right); -} - -// Reduces the supported payload shapes to { sha, base, ref, actor }. -function normalizePush(body) { - if (!body || typeof body !== 'object') return null; - - // hooks/post-receive, and anything else driving the API directly. - if (typeof body.sha === 'string') { - return { - sha: body.sha, - base: usableSha(body.base), - ref: typeof body.ref === 'string' ? body.ref : null, - actor: typeof body.actor === 'string' ? body.actor : null, - }; - } - - // GitLab sends both checkout_sha and after; checkout_sha is the one that - // refers to the tip of the pushed branch. - if (typeof body.checkout_sha === 'string') { - return { - sha: body.checkout_sha, - base: usableSha(body.before), - ref: typeof body.ref === 'string' ? body.ref : null, - actor: body.user_username ?? body.user_name ?? null, - }; - } - - // GitHub and Gitea. - if (typeof body.after === 'string') { - return { - sha: body.after, - base: usableSha(body.before), - ref: typeof body.ref === 'string' ? body.ref : null, - actor: body.pusher?.name ?? body.pusher?.login ?? body.sender?.login ?? null, - }; - } - - return null; -} - -// A new branch reports an all zero parent, which is not a commit. -function usableSha(value) { - if (typeof value !== 'string' || value === ZERO_SHA || !SHA_PATTERN.test(value)) return null; - return value; -} diff --git a/src/conductor/routes/workers.js b/src/conductor/routes/workers.js @@ -1,303 +0,0 @@ -// src/conductor/routes/workers.js - the API workers poll and report to -// -// Every route here requires a worker token. A worker may only touch a job it -// currently holds, which is checked against worker_token_id rather than -// trusting the job id it sends. - -import { pipeline as streamPipeline } from 'node:stream/promises'; -import { LogOffsetError } from '../../lib/log.js'; -import { hashingTransform } from '../../lib/stream.js'; -import { sanitizeRelativePath, keys as storageKeys } from '../../lib/storage/index.js'; -import { maskBuffer } from '../../lib/variables.js'; -import { newArtifactId } from '../../lib/ids.js'; - -// Log chunks are small and frequent; artifacts are streamed, so this only -// bounds a single log append. -const LOG_CHUNK_LIMIT = 1024 * 1024; - -export default async function workerRoutes(fastify, { cfg, db, git, logs, storage, projects, scheduler, workerTokens, variables }) { - // Raw bodies: log chunks and artifacts arrive as octet streams and must - // not be parsed. - fastify.addContentTypeParser('application/octet-stream', (req, payload, done) => done(null, payload)); - - async function requireWorker(req, reply) { - const header = req.headers.authorization || ''; - const presented = header.startsWith('Bearer ') ? header.slice(7).trim() : ''; - const token = await workerTokens.verify(presented, { ip: req.ip }); - if (!token) { - reply.code(401).send({ error: 'invalid or missing worker token' }); - return; - } - req.worker = token; - } - - // Confirms the job exists and is held by the calling worker. - async function heldJob(req, reply) { - const job = await db.get( - `SELECT j.id, j.run_id, j.name, j.state, j.worker_token_id, j.log_size, j.spec, - r.project_id, r.head_sha - FROM jobs j JOIN runs r ON r.id = j.run_id - WHERE j.id = {id}`, - { id: req.params.id } - ); - if (!job) { - reply.code(404).send({ error: 'unknown job' }); - return null; - } - if (job.worker_token_id !== req.worker.id) { - reply.code(403).send({ error: 'job is not held by this worker' }); - return null; - } - return job; - } - - fastify.addHook('preHandler', requireWorker); - - // Ask for work. The worker describes what it can run; the scheduler - // decides. Returns 204 when there is nothing to do. - fastify.post('/workers/jobs', async (req, reply) => { - const body = req.body ?? {}; - const arches = splitList(body.arches); - const features = splitList(body.features); - const name = typeof body.name === 'string' ? body.name.slice(0, 255) : req.worker.name; - - const job = await scheduler.claim({ - arches, - features, - tokenId: req.worker.id, - workerName: name, - // A worker registered by a user only ever sees that user's projects. - ownerId: req.worker.owner_id ?? null, - }); - if (!job) return reply.code(204).send(); - - const project = await projects.get(job.project_id); - const base = cfg.server.public_url.replace(/\/+$/, ''); - - return reply.send({ - job: { - id: job.id, - run_id: job.run_id, - project_id: job.project_id, - name: job.name, - arch: job.arch, - image: job.image, - // The worker maps these onto its own local mounts, environment and - // privileges. Without them no feature is applied at all. - requires: job.requires, - script: job.script, - env: job.env, - services: job.services, - artifacts: job.artifacts, - // Where the tree is unpacked and the script runs. Resolved when - // the run was created, so it does not shift under a retry. - workdir: job.workdir, - timeout: job.timeout, - attempt: job.attempt, - sha: job.sha, - ref: job.ref, - // Values the worker must redact from the log stream. Populated - // once project variables exist; masking also happens on ingest. - masked: job.masked ?? [], - // Derived from the server's own timeout so a worker never has to - // guess how often it is expected to check in. - heartbeat_interval: Math.max(5, Math.floor(cfg.scheduler.heartbeat_timeout / 4)), - endpoints: { - // The tree comes from the conductor, so a worker never needs - // credentials for the repository and cannot reach any commit - // other than the one it was given work for. - source: `${base}/api/v1/workers/jobs/${encodeURIComponent(job.id)}/source.tar.gz`, - log: `${base}/api/v1/workers/jobs/${encodeURIComponent(job.id)}/log`, - artifact: `${base}/api/v1/workers/jobs/${encodeURIComponent(job.id)}/artifacts`, - heartbeat: `${base}/api/v1/workers/jobs/${encodeURIComponent(job.id)}/heartbeat`, - done: `${base}/api/v1/workers/jobs/${encodeURIComponent(job.id)}/complete`, - }, - }, - }); - }); - - // The tree at the job's commit, as a gzipped tar. - fastify.get('/workers/jobs/:id/source.tar.gz', async (req, reply) => { - const job = await heldJob(req, reply); - if (!job) return reply; - - const project = await projects.get(job.project_id); - if (!project) return reply.code(404).send({ error: 'unknown project' }); - - // The tree rooted at the archive, with no wrapping directory: a - // worker extracts it at whatever path the job is to run in, and - // docker creates that path on the way. - reply.header('content-type', 'application/gzip'); - reply.header('content-disposition', `attachment; filename="${job.head_sha.slice(0, 12)}.tar.gz"`); - return reply.send(git.archiveStream(project, job.head_sha)); - }); - - // Append to the live log. X-Log-Offset makes a retry after a dropped - // connection safe. - fastify.post('/workers/jobs/:id/log', { bodyLimit: LOG_CHUNK_LIMIT }, async (req, reply) => { - const job = await heldJob(req, reply); - if (!job) return reply; - - const chunks = []; - let total = 0; - for await (const chunk of req.body) { - total += chunk.length; - if (total > LOG_CHUNK_LIMIT) return reply.code(413).send({ error: 'log chunk too large' }); - chunks.push(chunk); - } - - const rawOffset = req.headers['x-log-offset']; - const offset = rawOffset === undefined ? undefined : Number(rawOffset); - if (offset !== undefined && !Number.isInteger(offset)) { - return reply.code(400).send({ error: 'x-log-offset must be an integer' }); - } - - // The worker masks before sending; this repeats it on ingest so a - // worker that does not still cannot write a secret to disk. It works - // within a chunk only, which is why the worker holds back a boundary. - let payload = Buffer.concat(chunks); - if (variables) { - try { - payload = maskBuffer(payload, await variables.maskedValues(job.project_id)); - } catch (e) { - req.log.warn({ err: e }, `could not mask log output for ${job.id}`); - } - } - - try { - const result = await logs.append(job.run_id, job.id, payload, offset); - await db.run('UPDATE jobs SET log_size = {size}, heartbeat_at = {now} WHERE id = {id}', - { id: job.id, size: result.size, now: Date.now() }); - return reply.send(result); - } catch (e) { - if (e instanceof LogOffsetError) { - // Tell the worker where to resume from. - return reply.code(409).send({ error: e.message, expected_offset: e.expected }); - } - throw e; - } - }); - - // artifacts.expire in the pipeline, as an absolute time. Returns null - // when the job said nothing, leaving the artifact to the project policy. - function artifactExpiry(specJson, now) { - if (!specJson) return null; - try { - const spec = typeof specJson === 'string' ? JSON.parse(specJson) : specJson; - const seconds = spec?.artifacts?.expire; - if (!Number.isFinite(seconds) || seconds <= 0) return null; - return now + Math.round(seconds * 1000); - } catch { - // A spec that will not parse is a problem, but not this request's - // problem: the artifact is already stored. - return null; - } - } - - // Upload one artifact. The path is worker supplied, so it is sanitized - // before it can influence a storage key. - fastify.post('/workers/jobs/:id/artifacts', async (req, reply) => { - const job = await heldJob(req, reply); - if (!job) return reply; - - const relPath = sanitizeRelativePath(req.headers['x-artifact-path']); - if (!relPath) return reply.code(400).send({ error: 'x-artifact-path is missing or unusable' }); - - const declared = Number(req.headers['content-length']); - if (!Number.isInteger(declared) || declared < 0) { - return reply.code(411).send({ error: 'content-length is required for artifact uploads' }); - } - - // Hash on the way past, so the digest costs nothing extra even when the - // backend cannot report one. - const hasher = hashingTransform(); - const key = storageKeys.artifact(job.run_id, job.id, relPath); - - let stored; - try { - [, stored] = await Promise.all([ - streamPipeline(req.body, hasher), - storage.put(key, hasher, { size: declared }), - ]); - } catch (e) { - // A body shorter or longer than content-length is the worker's - // mistake, not a server fault. - if (/size mismatch/.test(e.message)) { - return reply.code(400).send({ error: e.message }); - } - throw e; - } - - const digest = hasher.digest(); - const now = Date.now(); - - await db.run( - `INSERT INTO artifacts (id, job_id, run_id, path, storage_key, size, sha256, created_at, expires_at) - VALUES ({id}, {job}, {run}, {path}, {key}, {size}, {sha}, {now}, {expires})`, - { - id: newArtifactId(), - job: job.id, - run: job.run_id, - path: relPath, - key, - size: stored.size ?? declared, - sha: digest, - now, - // An explicit deadline from the pipeline. Absent, the project's - // retention policy decides, which is the usual case. - expires: artifactExpiry(job.spec, now), - } - ); - - return reply.send({ path: relPath, size: stored.size ?? declared, sha256: digest }); - }); - - // Keeps the job alive, and is how a worker learns it should stop. - fastify.post('/workers/jobs/:id/heartbeat', async (req, reply) => { - const result = await scheduler.heartbeat(req.params.id, req.worker.id); - if (!result.known) return reply.code(404).send({ error: 'unknown job for this worker' }); - return reply.send({ cancelled: result.cancelled, state: result.state }); - }); - - // Report the outcome. The log is copied to storage here, whatever the - // result, so a failed job keeps its output. - fastify.post('/workers/jobs/:id/complete', async (req, reply) => { - const job = await heldJob(req, reply); - if (!job) return reply; - - const body = req.body ?? {}; - const success = body.success === true; - const exitCode = Number.isInteger(body.exit_code) ? body.exit_code : null; - const error = typeof body.error === 'string' ? body.error.slice(0, 4096) : null; - - const outcome = await scheduler.complete(job.id, { - success, - exitCode, - error, - tokenId: req.worker.id, - }); - if (!outcome.ok) return reply.code(409).send({ error: outcome.reason }); - - // A retry keeps writing to the same log, so only archive it once the - // job has actually stopped. - if (outcome.state !== 'queued') { - try { - await scheduler.finalizeLog(job.run_id, job.id); - } catch (e) { - req.log.error({ err: e }, `failed to archive log for ${job.id}`); - } - } - - return reply.send({ - state: outcome.state, - retry: outcome.retry === true, - skipped: outcome.skipped ?? [], - run_state: outcome.runState ?? null, - }); - }); -} - -function splitList(value) { - const items = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []; - return items.map((s) => String(s).trim()).filter(Boolean).slice(0, 64); -} diff --git a/src/worker/job.js b/src/worker/task.js