commit 37e0eda27aeef02be479455ad7cf22502775edfb
parent 5b9c4aa529abd828cc3da30ced2190bce5c4141a
Author: finwo <finwo@pm.me>
Date: Mon, 21 Sep 2026 17:30:17 +0200
Paginate the tables that grow without bound
Diffstat:
10 files changed, 534 insertions(+), 67 deletions(-)
diff --git a/assets/style.css b/assets/style.css
@@ -181,6 +181,11 @@ a.button:hover { text-decoration: none; background: #2b3036; }
td.row-actions { white-space: nowrap; text-align: right; }
td.row-actions button.link + button.link { margin-left: .9rem; }
+/* Page links sit under their table. The count is centred between the two
+ steps so it does not shift as the labels change. */
+.pager { display: flex; gap: 1rem; align-items: center; margin-top: .75rem; font-size: .8rem; }
+.pager > span { flex: 1; text-align: center; }
+
dialog {
border: 1px solid var(--border);
border-radius: 6px;
diff --git a/src/conductor/ui/layout.js b/src/conductor/ui/layout.js
@@ -106,3 +106,32 @@ export function duration(from, to) {
export const ACTIVE_JOB_STATES = ['pending', 'running'];
export const ACTIVE_TASK_STATES = ['queued', 'running'];
+
+// A link to another page of the same table, keeping whatever else was
+// being asked for, so paging does not quietly drop a filter.
+export function pageHref(base, query = {}, page = 1) {
+ const params = new URLSearchParams();
+ for (const [key, value] of Object.entries(query)) {
+ if (key === 'page' || typeof value !== 'string' || value === '') continue;
+ params.set(key, value);
+ }
+ if (page > 1) params.set('page', String(page));
+ const search = params.toString();
+ return search ? `${base}?${search}` : base;
+}
+
+// Page links for a table, rendered only when there is more than one page
+// so short tables stay quiet.
+export function pager(window, href) {
+ if (window.pages <= 1) return '';
+
+ const step = (to, label) => (to >= 1 && to <= window.pages
+ ? html`<a href="${href(to)}">${label}</a>`
+ : html`<span class="muted">${label}</span>`);
+
+ return html`<nav class="pager">
+ ${step(window.page - 1, 'previous')}
+ <span class="muted">${window.first} to ${window.last} of ${window.total}</span>
+ ${step(window.page + 1, 'next')}
+ </nav>`;
+}
diff --git a/src/conductor/ui/pages.js b/src/conductor/ui/pages.js
@@ -6,31 +6,36 @@
// polling stops by itself rather than needing to be cancelled.
import { html, raw } from './html.js';
-import { badge, notice, oneTimeSecret, shortSha, ago, duration, ACTIVE_JOB_STATES, ACTIVE_TASK_STATES } from './layout.js';
+import {
+ badge, notice, oneTimeSecret, shortSha, ago, duration, pager, pageHref,
+ ACTIVE_JOB_STATES, ACTIVE_TASK_STATES,
+} from './layout.js';
const poll = (url, seconds = 3) => raw(`hx-get="${url}" hx-trigger="every ${seconds}s" hx-swap="outerHTML"`);
// --- jobs ---
-export function jobsPage({ jobs, user, anonymous }) {
+export function jobsPage({ jobs, user, anonymous, query = {} }) {
return html`
<h2>Jobs</h2>
${anonymous ? html`<p class="muted">Showing public jobs. <a href="/login">Sign in</a> to see your own.</p>` : ''}
- ${jobsTable(jobs)}
+ ${jobsTable(jobs, query)}
`;
}
-export function jobsTable(jobs) {
- const live = jobs.some((r) => ACTIVE_JOB_STATES.includes(r.state));
- return html`<div id="jobs" ${live ? poll('/partials/jobs', 4) : ''}>
+export function jobsTable(jobs, query = {}) {
+ const live = jobs.rows.some((r) => ACTIVE_JOB_STATES.includes(r.state));
+ // The fragment asks for the page that is on screen, filter and all, or a
+ // refresh would silently throw the reader back to the first page.
+ return html`<div id="jobs" ${live ? poll(pageHref('/partials/jobs', query, jobs.page), 4) : ''}>
<table>
<thead><tr>
<th>job</th><th>project</th><th>ref</th><th>commit</th><th>state</th><th>started</th>
</tr></thead>
<tbody>
- ${jobs.length === 0
+ ${jobs.rows.length === 0
? html`<tr><td colspan="6" class="muted">No jobs to show.</td></tr>`
- : jobs.map((job) => html`<tr>
+ : jobs.rows.map((job) => html`<tr>
<td><a href="/jobs/${job.id}">#${job.number}</a></td>
<td>${job.project_id}</td>
<td class="muted">${(job.ref ?? '').replace('refs/heads/', '')}</td>
@@ -40,6 +45,7 @@ export function jobsTable(jobs) {
</tr>`)}
</tbody>
</table>
+ ${pager(jobs, (to) => pageHref('/jobs', query, to))}
</div>`;
}
@@ -106,7 +112,7 @@ export function tasksTable(job, tasks) {
// --- tasks ---
-export function taskPage({ task, artifacts, log }) {
+export function taskPage({ task, artifacts, log, query = {} }) {
return html`
<div class="panel">
<h2>${task.name}</h2>
@@ -125,15 +131,16 @@ export function taskPage({ task, artifacts, log }) {
<h3>log</h3>
<div class="logpane">${taskLog(task, log)}</div>
- ${artifacts.length === 0 ? '' : html`
+ ${artifacts.rows.length === 0 ? '' : html`
<h3>artifacts</h3>
<table><tbody>
- ${artifacts.map((a) => html`<tr>
+ ${artifacts.rows.map((a) => html`<tr>
<td><a href="/api/v1/projects/${task.project_id}/jobs/${task.job_id}/tasks/${task.id}/artifacts/${a.id}">${a.path}</a></td>
<td class="muted">${a.size} bytes</td>
<td class="mono muted">${a.sha256.slice(0, 12)}</td>
</tr>`)}
- </tbody></table>`}
+ </tbody></table>
+ ${pager(artifacts, (to) => pageHref(`/tasks/${task.id}`, query, to))}`}
`;
}
@@ -162,7 +169,7 @@ export function loginPage({ error }) {
// --- projects ---
-export function projectsPage({ projects, user }) {
+export function projectsPage({ projects, user, query = {} }) {
return html`
<h2>Projects</h2>
@@ -184,17 +191,18 @@ export function projectsPage({ projects, user }) {
</form>
</div>
- ${projectsTable(projects, user)}
+ ${projectsTable(projects, user, query)}
`;
}
-export function projectsTable(projects, user) {
- return html`<table id="projects">
+export function projectsTable(projects, user, query = {}) {
+ return html`<div id="projects">
+ <table>
<thead><tr><th>project</th><th>repository</th><th>visibility</th><th>owner</th><th>jobs</th><th></th></tr></thead>
<tbody>
- ${projects.length === 0
+ ${projects.rows.length === 0
? html`<tr><td colspan="6" class="muted">No projects yet.</td></tr>`
- : projects.map((p) => html`<tr>
+ : projects.rows.map((p) => html`<tr>
<td><a href="/projects/${p.id}">${p.id}</a></td>
<td class="muted mono">${p.repo_url}</td>
<td>${badge(p.visibility === 'public' ? 'success' : 'skipped')}<span class="muted"> ${p.visibility}</span></td>
@@ -203,7 +211,9 @@ export function projectsTable(projects, user) {
<td><a href="/projects/${p.id}">manage</a></td>
</tr>`)}
</tbody>
- </table>`;
+ </table>
+ ${pager(projects, (to) => pageHref('/projects', query, to))}
+ </div>`;
}
export function projectPage({ project, variables, triggerUrl, secret, user, retention }) {
@@ -355,7 +365,7 @@ export function variablesTable(project, variables) {
// --- workers ---
-export function workersPage({ tokens, user, created }) {
+export function workersPage({ tokens, user, created, query = {} }) {
return html`
<h2>Workers</h2>
<p class="muted">
@@ -381,7 +391,7 @@ export function workersPage({ tokens, user, created }) {
</form>
</div>
- ${workersTable(tokens, user)}
+ ${workersTable(tokens, user, query)}
<dialog id="disableDialog">
<p>Disable worker "<span id="disableWorkerName"></span>"?</p>
@@ -420,13 +430,17 @@ export function workersPage({ tokens, user, created }) {
`;
}
-export function workersTable(tokens, user) {
- return html`<table id="workers">
+export function workersTable(tokens, user, query = {}) {
+ // The id sits on the wrapper so a swap replaces the pager along with the
+ // rows; on the table alone it would be left behind, describing a page
+ // that is no longer there.
+ return html`<div id="workers">
+ <table>
<thead><tr><th>name</th><th>scope</th><th>state</th><th>last seen</th><th>address</th><th></th></tr></thead>
<tbody>
- ${tokens.length === 0
+ ${tokens.rows.length === 0
? html`<tr><td colspan="6" class="muted">No workers registered.</td></tr>`
- : tokens.map((t) => html`<tr>
+ : tokens.rows.map((t) => html`<tr>
<td>${t.name}</td>
<td class="muted">${t.owner_id === null ? 'shared' : (t.owner_id === user.id ? 'yours' : t.owner_id)}</td>
<td>${t.enabled === 1 ? badge('success') : badge('skipped')}</td>
@@ -440,12 +454,14 @@ export function workersTable(tokens, user) {
</td>
</tr>`)}
</tbody>
- </table>`;
+ </table>
+ ${pager(tokens, (to) => pageHref('/workers', query, to))}
+ </div>`;
}
// --- users ---
-export function usersPage({ users, localLogin, error }) {
+export function usersPage({ users, localLogin, error, query = {} }) {
return html`
<h2>Users</h2>
${localLogin ? '' : html`<p class="muted">
@@ -467,7 +483,7 @@ export function usersPage({ users, localLogin, error }) {
</form>
</div>
- ${usersTable(users)}
+ ${usersTable(users, query)}
<dialog id="removeUserDialog">
<p id="removeUserMessage"></p>
@@ -490,13 +506,14 @@ export function usersPage({ users, localLogin, error }) {
`;
}
-export function usersTable(users) {
- return html`<table id="users">
+export function usersTable(users, query = {}) {
+ return html`<div id="users">
+ <table>
<thead><tr>
<th>username</th><th>role</th><th>state</th><th>owns</th><th>last login</th><th></th>
</tr></thead>
<tbody>
- ${users.map((u) => html`<tr>
+ ${users.rows.map((u) => html`<tr>
<td>${u.username}</td>
<td>${u.role}</td>
<td>${u.disabled === 1 ? badge('skipped') : badge('success')}</td>
@@ -509,7 +526,9 @@ export function usersTable(users) {
</td>
</tr>`)}
</tbody>
- </table>`;
+ </table>
+ ${pager(users, (to) => pageHref('/users', query, to))}
+ </div>`;
}
// Removing an account takes everything it owns with it, so the
diff --git a/src/conductor/ui/routes.js b/src/conductor/ui/routes.js
@@ -18,6 +18,7 @@ import {
} from './pages.js';
import { canManageProject, canViewProject, VISIBILITIES } from '../../lib/projects.js';
import { canManageWorker } from '../../lib/workers.js';
+import { paginate, pageOf, windowOfAll } from '../../lib/page.js';
import { issueToken, sessionCookie, clearedCookie } from '../../lib/auth/index.js';
import { createState, verifyState } from '../../lib/auth/state.js';
import { PipelineError, DEFAULT_WORKDIR } from '../../lib/pipeline/index.js';
@@ -124,43 +125,50 @@ export default async function uiRoutes(fastify, services) {
// --- jobs ---
- async function visibleJobs(user, projectId = null) {
+ async function visibleJobs(user, { projectId = null, page: pageNo = 1 } = {}) {
const scope = user && user.role === 'admin'
? { sql: '1 = 1', params: {} }
: user
? { sql: "(j.visibility = 'public' OR p.owner_id = {viewerId})", params: { viewerId: user.id } }
: { sql: "j.visibility = 'public'", params: {} };
- return db.all(
- `SELECT j.id, j.project_id, j.number, j.ref, j.head_sha, j.state, j.visibility,
- j.title, j.created_at, j.started_at, j.finished_at
- FROM jobs j JOIN projects p ON p.id = j.project_id
- WHERE ${scope.sql} ${projectId ? 'AND j.project_id = {project}' : ''}
- ORDER BY j.created_at DESC LIMIT 50`,
- { ...scope.params, ...(projectId ? { project: projectId } : {}) }
- );
+ return paginate(db, {
+ select: `j.id, j.project_id, j.number, j.ref, j.head_sha, j.state, j.visibility,
+ j.title, j.created_at, j.started_at, j.finished_at`,
+ from: 'jobs j JOIN projects p ON p.id = j.project_id',
+ where: `${scope.sql}${projectId ? ' AND j.project_id = {project}' : ''}`,
+ order: 'j.created_at DESC',
+ params: { ...scope.params, ...(projectId ? { project: projectId } : {}) },
+ page: pageNo,
+ });
+ }
+
+ // The project filter and the page both live in the query string, so the
+ // polling fragment can be asked for exactly what is on screen.
+ function jobsQuery(req) {
+ return {
+ projectId: typeof req.query.project === 'string' ? req.query.project : null,
+ page: pageOf(req.query),
+ };
}
- fastify.get('/', async (req, reply) => {
+ async function jobsView(req, reply) {
const user = await currentUser(req);
- const jobs = await visibleJobs(user, typeof req.query.project === 'string' ? req.query.project : null);
+ const jobs = await visibleJobs(user, jobsQuery(req));
return page(reply, {
title: 'jobs',
user,
active: 'jobs',
- body: jobsPage({ jobs, user, anonymous: !user }),
+ body: jobsPage({ jobs, user, anonymous: !user, query: req.query }),
});
- });
+ }
- fastify.get('/jobs', async (req, reply) => {
- const user = await currentUser(req);
- const jobs = await visibleJobs(user, typeof req.query.project === 'string' ? req.query.project : null);
- return page(reply, { title: 'jobs', user, active: 'jobs', body: jobsPage({ jobs, user, anonymous: !user }) });
- });
+ fastify.get('/', jobsView);
+ fastify.get('/jobs', jobsView);
fastify.get('/partials/jobs', async (req, reply) => {
const user = await currentUser(req);
- return send(reply, jobsTable(await visibleJobs(user)));
+ return send(reply, jobsTable(await visibleJobs(user, jobsQuery(req)), req.query));
});
// Loads a job the caller may see, along with whether they may act on it.
@@ -264,16 +272,20 @@ export default async function uiRoutes(fastify, services) {
return page(reply, { title: 'not found', user, body: html`<p class="error">No such task.</p>` });
}
- const artifacts = await db.all(
- 'SELECT id, path, size, sha256 FROM artifacts WHERE task_id = {task} ORDER BY path',
- { task: task.id }
- );
+ const artifacts = await paginate(db, {
+ select: 'id, path, size, sha256',
+ from: 'artifacts',
+ where: 'task_id = {task}',
+ order: 'path',
+ params: { task: task.id },
+ page: pageOf(req.query),
+ });
return page(reply, {
title: task.name,
user,
active: 'jobs',
- body: taskPage({ task, artifacts, log: await tailOf(task) }),
+ body: taskPage({ task, artifacts, log: await tailOf(task), query: req.query }),
});
});
@@ -428,8 +440,13 @@ export default async function uiRoutes(fastify, services) {
fastify.get('/projects', async (req, reply) => {
const user = await required(req, reply);
if (!user) return reply;
- const rows = user.role === 'admin' ? await projects.list() : await projects.listOwnedBy(user.id);
- return page(reply, { title: 'projects', user, active: 'projects', body: projectsPage({ projects: rows, user }) });
+ const rows = await projects.pageManagedBy(user, pageOf(req.query));
+ return page(reply, {
+ title: 'projects',
+ user,
+ active: 'projects',
+ body: projectsPage({ projects: rows, user, query: req.query }),
+ });
});
fastify.post('/projects', async (req, reply) => {
@@ -605,7 +622,7 @@ export default async function uiRoutes(fastify, services) {
title: 'workers',
user,
active: 'workers',
- body: workersPage({ tokens: await workerTokens.listVisible(user), user }),
+ body: workersPage({ tokens: await workerTokens.pageVisible(user, pageOf(req.query)), user, query: req.query }),
});
});
@@ -626,8 +643,9 @@ export default async function uiRoutes(fastify, services) {
user,
active: 'workers',
body: workersPage({
- tokens: await workerTokens.listVisible(user),
+ tokens: await workerTokens.pageVisible(user, pageOf(req.query)),
user,
+ query: req.query,
created: { name: created.name, token: created.token },
}),
});
@@ -642,7 +660,7 @@ export default async function uiRoutes(fastify, services) {
if (!token || !canManageWorker(user, token)) return fail(reply, 'No such worker.', 404);
await workerTokens.setEnabled(token.id, req.body?.enabled === true || req.body?.enabled === 'true');
- return send(reply, workersTable(await workerTokens.listVisible(user), user));
+ return send(reply, workersTable(await workerTokens.pageVisible(user, pageOf(req.query)), user, req.query));
});
fastify.delete('/workers/:id', async (req, reply) => {
@@ -654,7 +672,7 @@ export default async function uiRoutes(fastify, services) {
if (!token || !canManageWorker(user, token)) return fail(reply, 'No such worker.', 404);
await workerTokens.remove(token.id);
- return send(reply, workersTable(await workerTokens.listVisible(user), user));
+ return send(reply, workersTable(await workerTokens.pageVisible(user, pageOf(req.query)), user, req.query));
});
// --- users ---
@@ -679,7 +697,7 @@ export default async function uiRoutes(fastify, services) {
title: 'users',
user,
active: 'users',
- body: usersPage({ users: await users.listWithHoldings(), localLogin: auth.localLogin }),
+ body: usersPage({ users: await users.pageWithHoldings(pageOf(req.query)), localLogin: auth.localLogin, query: req.query }),
});
});
@@ -698,7 +716,12 @@ export default async function uiRoutes(fastify, services) {
title: 'users',
user: await currentUser(req),
active: 'users',
- body: usersPage({ users: await users.listWithHoldings(), localLogin: auth.localLogin, error: e.message }),
+ body: usersPage({
+ users: await users.pageWithHoldings(pageOf(req.query)),
+ localLogin: auth.localLogin,
+ query: req.query,
+ error: e.message,
+ }),
});
}
return refresh(reply);
@@ -716,7 +739,7 @@ export default async function uiRoutes(fastify, services) {
return fail(reply, 'This is the only administrator.');
}
await users.setDisabled(target.id, disabled);
- return send(reply, usersTable(await users.listWithHoldings()));
+ return send(reply, usersTable(await users.pageWithHoldings(pageOf(req.query)), req.query));
});
fastify.delete('/users/:id', async (req, reply) => {
@@ -730,7 +753,7 @@ export default async function uiRoutes(fastify, services) {
}
await users.remove(target.id);
- return send(reply, usersTable(await users.listWithHoldings()));
+ return send(reply, usersTable(await users.pageWithHoldings(pageOf(req.query)), req.query));
});
async function lastAdmin(exceptId) {
diff --git a/src/lib/page.js b/src/lib/page.js
@@ -0,0 +1,64 @@
+// src/lib/page.js - windows over the tables that grow without bound
+//
+// A window is one page of rows plus the total behind them. The total is
+// counted under the same conditions as the rows, from the same FROM and
+// WHERE, so a visibility rule cannot promise a page that turns out empty.
+//
+// LIMIT and OFFSET are bound rather than pasted in. All three dialects
+// accept that, and retention already relies on it.
+
+export const PER_PAGE = 25;
+
+// A page number out of a query string. Anything unusable reads as the
+// first page: a mistyped url should still answer with a table.
+export function pageOf(query, perPage = PER_PAGE) {
+ const raw = query?.page;
+ const n = Number.parseInt(typeof raw === 'string' ? raw : '', 10);
+ if (!Number.isFinite(n) || n < 1) return 1;
+ // Far enough out that the offset stays a sane integer.
+ return Math.min(n, Math.ceil(Number.MAX_SAFE_INTEGER / perPage));
+}
+
+export function offsetOf(page, perPage = PER_PAGE) {
+ return (page - 1) * perPage;
+}
+
+// Describes a page of rows for whoever renders it.
+export function windowOf({ rows, total, page, perPage = PER_PAGE }) {
+ const from = offsetOf(page, perPage);
+ return {
+ rows,
+ total,
+ page,
+ perPage,
+ pages: Math.max(1, Math.ceil(total / perPage)),
+ // 1 based and inclusive, for "11 to 20 of 57".
+ first: rows.length === 0 ? 0 : from + 1,
+ last: from + rows.length,
+ };
+}
+
+// Runs the rows and their count as one window. Sharing from and where
+// between the two is the whole point; they cannot drift apart.
+export async function paginate(db, {
+ select, from, where = '1 = 1', order, params = {}, page, perPage = PER_PAGE,
+}) {
+ const rows = await db.all(
+ `SELECT ${select} FROM ${from} WHERE ${where} ORDER BY ${order}
+ LIMIT {pageSize} OFFSET {pageFrom}`,
+ { ...params, pageSize: perPage, pageFrom: offsetOf(page, perPage) }
+ );
+
+ const counted = await db.get(
+ `SELECT COUNT(*) AS total FROM ${from} WHERE ${where}`,
+ params
+ );
+
+ return windowOf({ rows, total: Number(counted?.total ?? 0), page, perPage });
+}
+
+// For a list already in memory, where the source is not a single query.
+export function windowOfAll(all, page, perPage = PER_PAGE) {
+ const from = offsetOf(page, perPage);
+ return windowOf({ rows: all.slice(from, from + perPage), total: all.length, page, perPage });
+}
diff --git a/src/lib/projects.js b/src/lib/projects.js
@@ -13,6 +13,7 @@
// original value, so it is sealed with the secret box.
import { newProjectId, slugify } from './ids.js';
+import { paginate } from './page.js';
import { parseWorkdir } from './pipeline/parse.js';
import { Problems } from './pipeline/schema.js';
@@ -69,6 +70,20 @@ export function createProjects({ db, secrets }) {
return db.all(`SELECT ${COLUMNS} FROM projects WHERE owner_id = {owner} ORDER BY name`, { owner: ownerId });
},
+ // One page of what this user manages: everything for an administrator,
+ // their own otherwise. Same rule as the two lists above, counted.
+ async pageManagedBy(user, page) {
+ const admin = user && user.role === 'admin';
+ return paginate(db, {
+ select: COLUMNS,
+ from: 'projects',
+ where: admin ? '1 = 1' : 'owner_id = {owner}',
+ order: 'name',
+ params: admin ? {} : { owner: user.id },
+ page,
+ });
+ },
+
async create(input) {
const id = input.id ? slugify(input.id) : (slugify(input.name) || newProjectId());
const now = Date.now();
diff --git a/src/lib/users.js b/src/lib/users.js
@@ -4,6 +4,7 @@
// and never recoverable; a forgotten password is reset, not read.
import { newUserId } from './ids.js';
+import { paginate } from './page.js';
import { hashPassword, verifyPassword, generatePassword, validatePassword } from './auth/password.js';
export const ROLES = ['admin', 'user'];
@@ -102,6 +103,17 @@ export function createUsers({ db, logger = console }) {
);
},
+ async pageWithHoldings(page) {
+ return paginate(db, {
+ select: `u.id, u.username, u.role, u.disabled, u.external_id, u.created_at, u.last_login_at,
+ (SELECT COUNT(*) FROM projects p WHERE p.owner_id = u.id) AS project_count,
+ (SELECT COUNT(*) FROM worker_tokens w WHERE w.owner_id = u.id) AS worker_count`,
+ from: 'users u',
+ order: 'u.username',
+ page,
+ });
+ },
+
async count() {
return (await db.get('SELECT COUNT(*) AS c FROM users')).c;
},
diff --git a/src/lib/workers.js b/src/lib/workers.js
@@ -10,6 +10,7 @@
// distinction is enforced in the scheduler, not here.
import { newToken, hashToken, newWorkerTokenId } from './ids.js';
+import { paginate, windowOf } from './page.js';
export function createWorkerTokens({ db }) {
return {
@@ -68,6 +69,21 @@ export function createWorkerTokens({ db }) {
);
},
+ // One page of listVisible. An anonymous caller sees nothing, so the
+ // window is empty rather than a query nobody needs.
+ async pageVisible(user, page) {
+ if (!user) return windowOf({ rows: [], total: 0, page });
+ const admin = user.role === 'admin';
+ return paginate(db, {
+ select: 'id, name, enabled, owner_id, created_at, last_seen_at, last_ip',
+ from: 'worker_tokens',
+ where: admin ? '1 = 1' : 'owner_id = {owner}',
+ order: 'created_at DESC',
+ params: admin ? {} : { owner: user.id },
+ page,
+ });
+ },
+
async setEnabled(id, enabled) {
const res = await db.run(
'UPDATE worker_tokens SET enabled = {enabled} WHERE id = {id}',
diff --git a/test/dialects.test.js b/test/dialects.test.js
@@ -4,7 +4,8 @@
// exercised by reading them. This applies every migration against a real
// server and then jobs the retention sweep, which is the query that most
// depends on the dialect: it joins, filters on nulls, orders, and binds a
-// LIMIT, and the drivers do not all accept those the same way.
+// LIMIT, and the drivers do not all accept those the same way. Paging is
+// checked here for the same reason, since it binds an OFFSET as well.
//
// Skipped when docker is unavailable, or when the optional drivers are
// not installed.
@@ -17,6 +18,7 @@ import path from 'node:path';
import { startDatabase, dialectAvailable } from './helpers/database.js';
import { openDatabase, runMigrations, dialectFromUrl } from '../src/lib/db/index.js';
+import { paginate } from '../src/lib/page.js';
import { createStorage } from '../src/lib/storage/index.js';
import { createLogStore } from '../src/lib/log.js';
import { createRetention } from '../src/conductor/retention.js';
@@ -97,6 +99,24 @@ for (const dialect of ['postgres', 'mysql']) {
const logs = createLogStore(cfg);
await seed(db, storage);
+ // Paging binds both LIMIT and OFFSET, and reads a COUNT back. The
+ // count has to arrive as a number: postgres returns int8, which the
+ // driver would otherwise hand over as a string and turn the page
+ // arithmetic into concatenation. Checked before the sweep, while
+ // all five jobs are still there.
+ const firstPage = await paginate(db, {
+ select: 'id', from: 'jobs', order: 'number', page: 1, perPage: 2,
+ });
+ assert.equal(firstPage.rows.length, 2);
+ assert.equal(typeof firstPage.total, 'number', 'COUNT came back as a string');
+ assert.equal(firstPage.total, 5);
+ assert.equal(firstPage.pages, 3);
+
+ const lastPage = await paginate(db, {
+ select: 'id', from: 'jobs', order: 'number', page: 3, perPage: 2,
+ });
+ assert.deepEqual(lastPage.rows.map((r) => r.id), ['r5'], 'the offset landed wrong');
+
const retention = createRetention({ cfg, db, storage, logs, logger: {} });
const first = await retention.sweep({ now: NOW });
diff --git a/test/pagination.test.js b/test/pagination.test.js
@@ -0,0 +1,264 @@
+// test/pagination.test.js - windows over the long tables
+//
+// The pure helpers are checked directly, and the pages through a real
+// server, because the parts that go wrong are the ones that join up: a
+// count taken under different conditions than the rows, a filter dropped
+// from a page link, or a polling fragment that asks for page one forever.
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { startHarness } from './helpers/harness.js';
+import { pageOf, offsetOf, windowOf, windowOfAll, paginate, PER_PAGE } from '../src/lib/page.js';
+import { pageHref, pager } from '../src/conductor/ui/layout.js';
+import { toHtml } from '../src/conductor/ui/html.js';
+
+async function withUi(options, fn) {
+ const h = await startHarness({ bootstrap: true, ...options });
+ try {
+ return await fn(h);
+ } finally {
+ await h.stop();
+ }
+}
+
+const get = (h, url, headers = {}) => h.app.inject({ method: 'GET', url, headers });
+
+// Jobs are seeded straight into the database. Going through the scheduler
+// would mean a git sync and a pipeline compile per row, which says nothing
+// about paging and takes far longer.
+async function seedJobs(h, count, { visibility = 'public' } = {}) {
+ const now = Date.now();
+ // Job numbers are unique per project, so a second call in the same test
+ // has to carry on where the first stopped.
+ const base = h.seeded ?? 0;
+ for (let i = 0; i < count; i += 1) {
+ const n = base + i;
+ await h.services.db.run(
+ `INSERT INTO jobs (id, project_id, number, ref, head_sha, trigger_type, actor,
+ title, state, pipeline, visibility, created_at, started_at, finished_at)
+ VALUES ({id}, {project}, {number}, 'refs/heads/main', {sha}, 'push', null,
+ null, 'success', null, {visibility}, {created}, null, null)`,
+ {
+ id: `seed${String(n).padStart(4, '0')}`,
+ project: h.project.id,
+ number: 1000 + n,
+ sha: String(n).padStart(40, '0'),
+ visibility,
+ // Descending created_at, so row order is predictable.
+ created: now - n * 1000,
+ }
+ );
+ }
+ h.seeded = base + count;
+}
+
+// --- the helpers ---
+
+test('a page number is read from the query, and nonsense reads as the first', () => {
+ assert.equal(pageOf({ page: '3' }), 3);
+ assert.equal(pageOf({ page: '1' }), 1);
+ assert.equal(pageOf({}), 1);
+ assert.equal(pageOf({ page: '' }), 1);
+ assert.equal(pageOf({ page: 'banana' }), 1);
+ assert.equal(pageOf({ page: '0' }), 1);
+ assert.equal(pageOf({ page: '-4' }), 1);
+ // Arrays arrive when a parameter is repeated in the query string.
+ assert.equal(pageOf({ page: ['2', '5'] }), 1);
+});
+
+test('an absurd page number cannot produce an unusable offset', () => {
+ const offset = offsetOf(pageOf({ page: '999999999999999999999' }));
+ assert.ok(Number.isSafeInteger(offset), `offset ${offset} is not a safe integer`);
+});
+
+test('a window describes the slice it holds', () => {
+ const win = windowOf({ rows: new Array(25).fill(0), total: 57, page: 1 });
+ assert.equal(win.pages, 3);
+ assert.equal(win.first, 1);
+ assert.equal(win.last, 25);
+
+ const middle = windowOf({ rows: new Array(25).fill(0), total: 57, page: 2 });
+ assert.equal(middle.first, 26);
+ assert.equal(middle.last, 50);
+
+ const last = windowOf({ rows: new Array(7).fill(0), total: 57, page: 3 });
+ assert.equal(last.first, 51);
+ assert.equal(last.last, 57);
+});
+
+test('an empty table is one page, and counts from zero', () => {
+ const win = windowOf({ rows: [], total: 0, page: 1 });
+ assert.equal(win.pages, 1);
+ assert.equal(win.first, 0);
+ assert.equal(win.last, 0);
+});
+
+test('windowOfAll slices a list that is already in memory', () => {
+ const all = Array.from({ length: 30 }, (_, i) => i);
+ const second = windowOfAll(all, 2);
+ assert.equal(second.rows.length, 5);
+ assert.equal(second.rows[0], 25);
+ assert.equal(second.total, 30);
+});
+
+// --- the links ---
+
+test('a page link keeps the other query parameters', () => {
+ assert.equal(pageHref('/jobs', { project: 'demo' }, 2), '/jobs?project=demo&page=2');
+ // Page one is the bare url rather than an explicit page=1.
+ assert.equal(pageHref('/jobs', { project: 'demo' }, 1), '/jobs?project=demo');
+ assert.equal(pageHref('/jobs', {}, 1), '/jobs');
+ // The incoming page is replaced, not repeated.
+ assert.equal(pageHref('/jobs', { page: '4' }, 2), '/jobs?page=2');
+});
+
+test('a pager is not rendered when everything fits on one page', () => {
+ const win = windowOf({ rows: new Array(3).fill(0), total: 3, page: 1 });
+ assert.equal(toHtml(pager(win, (p) => `/x?page=${p}`)), '');
+});
+
+test('a pager offers only the steps that exist', () => {
+ const first = toHtml(pager(windowOf({ rows: new Array(25).fill(0), total: 57, page: 1 }), (p) => `/x?page=${p}`));
+ assert.match(first, /1 to 25 of 57/);
+ assert.match(first, /href="\/x\?page=2"/);
+ assert.doesNotMatch(first, /href="\/x\?page=0"/);
+
+ const last = toHtml(pager(windowOf({ rows: new Array(7).fill(0), total: 57, page: 3 }), (p) => `/x?page=${p}`));
+ assert.match(last, /href="\/x\?page=2"/);
+ assert.doesNotMatch(last, /href="\/x\?page=4"/);
+});
+
+// --- against a real database ---
+
+test('paginate counts under the same conditions as it selects', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, 10, { visibility: 'public' });
+ await seedJobs(h, 4, { visibility: 'private' });
+
+ const win = await paginate(h.services.db, {
+ select: 'id',
+ from: 'jobs',
+ where: "visibility = 'public'",
+ order: 'created_at DESC',
+ page: 1,
+ perPage: 5,
+ });
+
+ assert.equal(win.rows.length, 5);
+ // The private rows must not be counted, or the reader is offered a
+ // page that turns out to be empty.
+ assert.equal(win.total, 10);
+ assert.equal(win.pages, 2);
+ });
+});
+
+test('successive pages do not repeat or skip a row', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, 12);
+
+ const seen = [];
+ for (let p = 1; p <= 3; p += 1) {
+ const win = await paginate(h.services.db, {
+ select: 'id',
+ from: 'jobs',
+ order: 'created_at DESC',
+ page: p,
+ perPage: 5,
+ });
+ seen.push(...win.rows.map((r) => r.id));
+ }
+
+ assert.equal(seen.length, 12);
+ assert.equal(new Set(seen).size, 12, 'a row appeared on more than one page');
+ });
+});
+
+// --- the pages ---
+
+test('the job list is capped at a page and offers the next one', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, PER_PAGE + 5);
+
+ const first = await get(h, '/jobs');
+ assert.equal(first.statusCode, 200);
+ assert.equal((first.body.match(/href="\/jobs\/seed/g) ?? []).length, PER_PAGE);
+ assert.match(first.body, /href="\/jobs\?page=2"/);
+
+ const second = await get(h, '/jobs?page=2');
+ assert.equal((second.body.match(/href="\/jobs\/seed/g) ?? []).length, 5);
+ // Nothing beyond the last page is offered.
+ assert.doesNotMatch(second.body, /href="\/jobs\?page=3"/);
+ });
+});
+
+test('paging past the end is an empty table rather than an error', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, 3);
+ const res = await get(h, '/jobs?page=99');
+ assert.equal(res.statusCode, 200);
+ assert.match(res.body, /No jobs to show/);
+ });
+});
+
+test('a page link keeps the project filter, and so does the live fragment', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, PER_PAGE + 2);
+
+ const res = await get(h, `/jobs?project=${h.project.id}`);
+ assert.match(res.body, new RegExp(`href="/jobs\\?project=${h.project.id}&page=2"`));
+ });
+});
+
+test('the polling fragment asks for the page being viewed, not the first', async () => {
+ await withUi({}, async (h) => {
+ // A running job is what makes the fragment poll at all.
+ await seedJobs(h, PER_PAGE + 2);
+ await h.services.db.run(
+ "UPDATE jobs SET state = 'running' WHERE id = {id}",
+ { id: `seed${String(PER_PAGE).padStart(4, '0')}` }
+ );
+
+ const second = await get(h, '/jobs?page=2');
+ assert.match(second.body, /hx-get="\/partials\/jobs\?page=2"/);
+
+ // And the fragment itself honours the page it is asked for.
+ const partial = await get(h, '/partials/jobs?page=2');
+ assert.equal(partial.statusCode, 200);
+ assert.equal((partial.body.match(/href="\/jobs\/seed/g) ?? []).length, 2);
+ });
+});
+
+test('the count behind the pager respects who is asking', async () => {
+ await withUi({}, async (h) => {
+ await seedJobs(h, 4, { visibility: 'public' });
+ await seedJobs(h, 40, { visibility: 'private' });
+
+ // Anonymous sees only the public rows, so there is nothing to page.
+ const anon = await get(h, '/jobs');
+ assert.match(anon.body, /No jobs to show|seed/);
+ assert.doesNotMatch(anon.body, /href="\/jobs\?page=2"/);
+
+ // The administrator sees everything, so there is.
+ const { cookie } = await h.login();
+ const admin = await get(h, '/jobs', { cookie });
+ assert.match(admin.body, /href="\/jobs\?page=2"/);
+ });
+});
+
+test('the users and workers tables keep their swap target on the wrapper', async () => {
+ await withUi({}, async (h) => {
+ const { cookie } = await h.login();
+
+ const users = await get(h, '/users', { cookie });
+ assert.equal(users.statusCode, 200);
+ // The id has to survive on the element htmx replaces, or a later swap
+ // has nothing to aim at.
+ assert.match(users.body, /<div id="users">/);
+
+ const workers = await get(h, '/workers', { cookie });
+ assert.match(workers.body, /<div id="workers">/);
+
+ const projects = await get(h, '/projects', { cookie });
+ assert.match(projects.body, /<div id="projects">/);
+ });
+});
