smoke.sh (15802B)
1 #!/bin/sh 2 # deploy/smoke.sh - build the images and run a pipeline through them 3 # 4 # Checks the thing people actually deploy, rather than the code the tests 5 # import: both images build, the conductor starts on an empty volume, 6 # a worker registers, and a real pipeline runs to completion with its log 7 # and artifact readable afterwards. 8 # 9 # Usage: deploy/smoke.sh [--keep] 10 # 11 # --keep leave the stack running afterwards for poking at 12 # 13 # Every run names its own compose project, images and port, so several can 14 # run against one docker host at once. SMOKE_PORT pins the port if needed. 15 # 16 # Needs docker and a few hundred megabytes of disk. Takes about a minute. 17 18 set -eu 19 20 KEEP=0 21 [ "${1:-}" = "--keep" ] && KEEP=1 22 23 ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) 24 WORK=$(mktemp -d) 25 SECRET=smoke-trigger-secret 26 27 # Everything that a second run on the same host could collide on carries 28 # this: the compose project owns the network, volume and container names, 29 # and the image tags would otherwise be overwritten mid-run by a build of 30 # a different commit. 31 RUN=$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n') 32 PROJECT="conductor-smoke-${RUN}" 33 IMAGE="conductor:smoke-${RUN}" 34 WORKER_IMAGE="conductor-worker:smoke-${RUN}" 35 36 # 0 lets docker pick a free port; the real one is read back once the 37 # container is up. Picking one here would race with whoever takes it next. 38 PORT="${SMOKE_PORT:-0}" 39 40 # Published ports live on the docker host, which is the gateway when this 41 # runs in a container, as it does in CI. 42 if [ -z "${SMOKE_HOST:-}" ] && [ -f /.dockerenv ]; then 43 SMOKE_HOST=$(ip route 2>/dev/null | awk '/^default/{print $3; exit}') 44 fi 45 SMOKE_HOST="${SMOKE_HOST:-127.0.0.1}" 46 47 48 49 log() { printf '\n== %s\n' "$*"; } 50 fail() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } 51 52 cleanup() { 53 status=$? 54 if [ "${KEEP}" -eq 1 ] && [ "${status}" -eq 0 ]; then 55 printf '\nleaving the stack up; tear it down with:\n' 56 printf ' docker compose -p %s -f %s down -v\n' "${PROJECT}" "${WORK}/compose.yml" 57 return 58 fi 59 printf '\n== cleaning up\n' 60 SMOKE_TOKEN=unused docker compose -p "${PROJECT}" -f "${WORK}/compose.yml" down -v >/dev/null 2>&1 || true 61 docker rmi "${IMAGE}" "${WORKER_IMAGE}" >/dev/null 2>&1 || true 62 rm -rf "${WORK}" 63 } 64 trap cleanup EXIT 65 66 log "building images" 67 docker build -q -f "${ROOT}/deploy/Dockerfile" -t "${IMAGE}" "${ROOT}" 68 docker build -q -f "${ROOT}/deploy/Dockerfile.worker" -t "${WORKER_IMAGE}" "${ROOT}" 69 70 log "preparing a repository" 71 mkdir -p "${WORK}/repo" 72 cat > "${WORK}/repo/.conductor.yml" <<'PIPELINE' 73 version: 1 74 defaults: 75 image: alpine:3 76 tasks: 77 build: 78 script: 79 - echo "run $CONDUCTOR_JOB_NUMBER of $CONDUCTOR_PROJECT" 80 - mkdir -p out && echo "packaged" > out/result.txt 81 artifacts: 82 paths: [out/**] 83 publish: 84 needs: [build] 85 script: ['echo published'] 86 release: 87 needs: [build] 88 only: 89 refs: [refs/heads/main] 90 script: ['echo released'] 91 PIPELINE 92 echo 'smoke test repository' > "${WORK}/repo/README.md" 93 git -C "${WORK}/repo" init -q -b main 94 git -C "${WORK}/repo" config user.email smoke@example.invalid 95 git -C "${WORK}/repo" config user.name Smoke 96 git -C "${WORK}/repo" add -A 97 git -C "${WORK}/repo" commit -q -m 'smoke test' 98 SHA=$(git -C "${WORK}/repo" rev-parse HEAD) 99 100 cat > "${WORK}/compose.yml" <<COMPOSE 101 name: ${PROJECT} 102 services: 103 conductor: 104 image: ${IMAGE} 105 ports: ["${PORT}:9080"] 106 environment: 107 CONDUCTOR_PORT: "9080" 108 CONDUCTOR_PUBLIC_URL: http://conductor:9080 109 CONDUCTOR_SESSION_SECRET: smoke-session-secret 110 CONDUCTOR_ADMIN_PASSWORD: smoke-admin-password 111 volumes: 112 - smoke-data:/data 113 worker: 114 image: ${WORKER_IMAGE} 115 depends_on: [conductor] 116 environment: 117 CONDUCTOR_URL: http://conductor:9080 118 CONDUCTOR_WORKER_NAME: smoke-worker 119 CONDUCTOR_WORKER_TOKEN: "\${SMOKE_TOKEN}" 120 CONDUCTOR_WORKER_CONCURRENCY: "2" 121 volumes: 122 - /var/run/docker.sock:/var/run/docker.sock 123 volumes: 124 smoke-data: 125 COMPOSE 126 127 compose() { SMOKE_TOKEN="${SMOKE_TOKEN:-unused}" docker compose -p "${PROJECT}" -f "${WORK}/compose.yml" "$@"; } 128 129 log "starting the conductor" 130 compose up -d conductor 131 132 # Ask docker which port it settled on rather than assuming, since PORT is 133 # normally 0. 134 PUBLISHED=$(compose port conductor 9080 | sed 's/.*://') 135 [ -n "${PUBLISHED}" ] || fail "the conductor published no port" 136 BASE_URL="http://${SMOKE_HOST}:${PUBLISHED}" 137 printf 'using %s\n' "${BASE_URL}" 138 139 i=0 140 while [ "${i}" -lt 60 ]; do 141 curl -fsS -m 2 "${BASE_URL}/health" >/dev/null 2>&1 && break 142 i=$((i + 1)) 143 sleep 1 144 done 145 if [ "${i}" -ge 60 ]; then 146 compose logs conductor | tail -40 147 fail "the conductor did not become healthy" 148 fi 149 curl -fsS "${BASE_URL}/health" 150 printf '\n' 151 152 # Copied rather than bind mounted: the daemon resolves a mount source on the 153 # host, where a path made in this container does not exist. 154 log "copying the repository into the conductor" 155 CONDUCTOR_CID=$(compose ps -q conductor) 156 [ -n "${CONDUCTOR_CID}" ] || fail "the conductor container is not running" 157 docker cp "${WORK}/repo" "${CONDUCTOR_CID}:/repo" 158 # docker cp writes as root; git refuses a repository owned by somebody else. 159 docker exec -u 0 "${CONDUCTOR_CID}" chown -R conductor:conductor /repo 160 compose exec -T conductor git -C /repo rev-parse HEAD >/dev/null \ 161 || fail "the copied repository is not readable inside the conductor" 162 163 log "registering the project and a worker" 164 compose exec -T conductor node src/admin-cli.js project:add demo /repo --secret "${SECRET}" --visibility public >/dev/null 165 SMOKE_TOKEN=$(compose exec -T conductor node src/admin-cli.js token:add smoke-worker | awk '/token:/{print $2}') 166 [ -n "${SMOKE_TOKEN}" ] || fail "no worker token was issued" 167 export SMOKE_TOKEN 168 169 log "starting the worker" 170 compose up -d worker 171 172 log "triggering a job" 173 BODY=$(printf '{"sha":"%s","ref":"refs/heads/main"}' "${SHA}") 174 SIG=$(printf '%s' "${BODY}" | openssl dgst -sha256 -hmac "${SECRET}" | sed 's/^.*[= ]//') 175 RESPONSE=$(curl -fsS -X POST "${BASE_URL}/api/v1/projects/demo/trigger" \ 176 -H 'Content-Type: application/json' \ 177 -H "X-Hub-Signature-256: sha256=${SIG}" \ 178 -d "${BODY}") 179 printf '%s\n' "${RESPONSE}" 180 181 JOB=$(printf '%s' "${RESPONSE}" | sed 's/.*"job_id":"\([^"]*\)".*/\1/') 182 [ -n "${JOB}" ] || fail "no job was created" 183 184 # A job still being compiled is not readable yet, so a refusal is just 185 # another turn. 186 job_state() { 187 curl -fsS "${BASE_URL}/api/v1/jobs/$1" 2>/dev/null \ 188 | sed -n 's/.*"state":"\([a-z]*\)".*/\1/p' | head -1 189 } 190 191 # One object per line, so a task can be matched on its name and read for 192 # its id without depending on which order the keys came out in. 193 objects() { printf '%s' "$1" | tr '{' '\n'; } 194 195 task_id() { 196 objects "$1" | grep "\"name\":\"$2\"" | sed 's/.*"id":"\([^"]*\)".*/\1/' | head -1 197 } 198 199 # A pipeline that fails to compile is recorded on the job row and nowhere 200 # else, and leaves the job private, which reads here as a 404. 201 job_row() { 202 compose exec -T conductor node --input-type=module -e " 203 import { DatabaseSync } from 'node:sqlite'; 204 const db = new DatabaseSync(process.env.CONDUCTOR_DATABASE_PATH); 205 const row = db.prepare('SELECT state, visibility, error FROM jobs WHERE id = ?').get('$1'); 206 console.log(JSON.stringify(row)); 207 " 2>/dev/null || true 208 } 209 210 log "waiting for the job to finish" 211 STATE_NOW= 212 i=0 213 while [ "${i}" -lt 90 ]; do 214 STATE_NOW=$(job_state "${JOB}") 215 case "${STATE_NOW}" in 216 success|failed|cancelled) break ;; 217 esac 218 # An unreadable job page does not become readable by waiting it out. 219 if [ -z "${STATE_NOW}" ] && [ "${i}" -ge 10 ]; then 220 printf 'the job page stayed unreadable\njob row: %s\n' "$(job_row "${JOB}")" >&2 221 compose logs conductor | tail -20 >&2 222 fail "the job never became readable" 223 fi 224 i=$((i + 1)) 225 sleep 1 226 done 227 228 printf 'job %s finished as %s after %ss\n' "${JOB}" "${STATE_NOW}" "${i}" 229 [ "${STATE_NOW}" = success ] || { 230 printf 'job row: %s\n' "$(job_row "${JOB}")" 231 compose logs worker | tail -30 232 fail "the job ended as ${STATE_NOW}" 233 } 234 235 log "checking the log and the artifact" 236 # Task ids carry no structure, so the build task is found by name in the 237 # job's task list rather than by assembling an id. 238 TASKS=$(curl -fsS "${BASE_URL}/api/v1/jobs/${JOB}/tasks") 239 TASK=$(task_id "${TASKS}" build) 240 [ -n "${TASK}" ] || fail "no build task was listed for the job" 241 242 # The edges are what a stage graph is drawn from, and nothing else 243 # publishes them. 244 printf '%s' "${TASKS}" | grep -q '"depends_on_id"' \ 245 || fail "the task list carried no dependency edges" 246 247 LOG=$(curl -fsS "${BASE_URL}/api/v1/tasks/${TASK}/log") 248 printf '%s' "${LOG}" | grep -q 'run 1 of demo' \ 249 || fail "the task environment did not reach the script" 250 printf '%s' "${LOG}" | grep -q '"expired":false' \ 251 || fail "the log did not report whether it had expired" 252 253 ARTIFACTS=$(curl -fsS "${BASE_URL}/api/v1/tasks/${TASK}/artifacts") 254 printf '%s' "${ARTIFACTS}" | grep -q '"path":"out/result.txt"' \ 255 || fail "the artifact was not listed" 256 257 ARTIFACT_URL=$(printf '%s' "${ARTIFACTS}" | sed -n 's#.*"url":"\([^"]*\)".*#\1#p' | head -1) 258 [ -n "${ARTIFACT_URL}" ] || fail "no artifact download url was offered" 259 260 # The url is built from public_url, which is the conductor's own name on 261 # the compose network rather than the published port this runs against. 262 ARTIFACT_PATH=$(printf '%s' "${ARTIFACT_URL}" | sed 's#^https\{0,1\}://[^/]*##') 263 CONTENT=$(curl -fsSL "${BASE_URL}${ARTIFACT_PATH}") 264 printf 'artifact contents: %s\n' "${CONTENT}" 265 [ "${CONTENT}" = packaged ] || fail "the artifact did not round trip" 266 267 log "reading the task back through the api" 268 # Both paths describe a public task to anyone, with no credential at all. 269 # The absence of the environment is the point of the endpoint, so it is 270 # checked here against the running image rather than trusted from the unit 271 # tests: the pipeline above puts CONDUCTOR_PROJECT in the script, so a 272 # response carrying it would mean the environment had escaped. 273 for URL in "/api/v1/tasks/${TASK}" "/api/v1/projects/demo/tasks/${TASK}"; do 274 BODY=$(curl -fsS "${BASE_URL}${URL}") || fail "${URL} was not readable" 275 276 printf '%s' "${BODY}" | grep -q '"state":"success"' \ 277 || fail "${URL} did not report the task state" 278 279 if printf '%s' "${BODY}" | grep -q 'CONDUCTOR_PROJECT'; then 280 fail "${URL} leaked the task environment" 281 fi 282 if printf '%s' "${BODY}" | grep -q '"script"'; then 283 fail "${URL} leaked the task script" 284 fi 285 done 286 287 # Naming a project that does not own the task must not widen anything. 288 curl -fsS -o /dev/null "${BASE_URL}/api/v1/projects/absent/tasks/${TASK}" 2>/dev/null \ 289 && fail "a task was readable through a project that does not own it" 290 291 log "signing in" 292 # The whole flow against the shipped image: discovery, the login page, a 293 # code, a token, and a call that needs it. 294 DISCOVERY=$(curl -fsS "${BASE_URL}/.well-known/openid-configuration") 295 printf '%s' "${DISCOVERY}" | grep -q '"client_id":"conductor"' \ 296 || fail "discovery did not name the client" 297 298 REDIRECT="http://conductor:9080/" 299 VERIFIER=$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n') 300 CHALLENGE=$(printf '%s' "${VERIFIER}" \ 301 | openssl dgst -sha256 -binary | openssl base64 | tr '+/' '-_' | tr -d '=\n') 302 303 AUTHORIZE="${BASE_URL}/auth/authorize?response_type=code&client_id=conductor" 304 AUTHORIZE="${AUTHORIZE}&redirect_uri=http%3A%2F%2Fconductor%3A9080%2F" 305 AUTHORIZE="${AUTHORIZE}&code_challenge=${CHALLENGE}&code_challenge_method=S256&state=smoke" 306 307 curl -fsS "${AUTHORIZE}" | grep -q 'name="password"' \ 308 || fail "the login page was not served" 309 310 LOCATION=$(curl -sS -o /dev/null -w '%{redirect_url}' -X POST "${AUTHORIZE}" \ 311 -d 'username=admin&password=smoke-admin-password') 312 CODE=$(printf '%s' "${LOCATION}" | sed -n 's/.*[?&]code=\([^&]*\).*/\1/p') 313 [ -n "${CODE}" ] || fail "signing in produced no authorization code" 314 315 GRANT=$(curl -fsS -X POST "${BASE_URL}/auth/token" \ 316 -d "grant_type=authorization_code&code=${CODE}&code_verifier=${VERIFIER}&redirect_uri=${REDIRECT}") 317 ACCESS=$(printf '%s' "${GRANT}" | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p') 318 REFRESH=$(printf '%s' "${GRANT}" | sed -n 's/.*"refresh_token":"\([^"]*\)".*/\1/p') 319 [ -n "${ACCESS}" ] || fail "no access token was issued" 320 321 curl -fsS "${BASE_URL}/api/v1/users/me" -H "Authorization: Bearer ${ACCESS}" \ 322 | grep -q '"username":"admin"' || fail "the access token did not identify the administrator" 323 324 # The session is listed with what it was last used from, which is the 325 # whole point of the revocation screen. 326 curl -fsS "${BASE_URL}/api/v1/sessions" -H "Authorization: Bearer ${ACCESS}" \ 327 | grep -q '"user_agent":"curl' || fail "the session was not listed" 328 329 # Redeeming a code twice must not work. 330 curl -fsS -o /dev/null -X POST "${BASE_URL}/auth/token" \ 331 -d "grant_type=authorization_code&code=${CODE}&code_verifier=${VERIFIER}&redirect_uri=${REDIRECT}" 2>/dev/null \ 332 && fail "an authorization code was redeemed twice" 333 334 # An unregistered origin is refused rather than redirected to. 335 curl -fsS -o /dev/null "${BASE_URL}/auth/authorize?response_type=code&client_id=conductor&redirect_uri=https%3A%2F%2Fevil.example%2F&code_challenge=${CHALLENGE}&code_challenge_method=S256" 2>/dev/null \ 336 && fail "an unregistered redirect uri was accepted" 337 338 # Refreshing keeps the same token and issues a new access token. 339 SECOND=$(curl -fsS -X POST "${BASE_URL}/auth/token" \ 340 -d "grant_type=refresh_token&refresh_token=${REFRESH}") 341 printf '%s' "${SECOND}" | grep -q "\"refresh_token\":\"${REFRESH}\"" \ 342 || fail "refreshing did not keep the refresh token" 343 344 log "checking the application" 345 INDEX=$(curl -fsS "${BASE_URL}/") 346 printf '%s' "${INDEX}" | grep -q '<title>conductor</title>' \ 347 || fail "the application index was not served" 348 BUNDLE=$(printf '%s' "${INDEX}" | sed -n 's/.*<script src="\([^"]*\)".*/\1/p' | head -1) 349 [ -n "${BUNDLE}" ] || fail "the application index referenced no bundle" 350 curl -fsS -o /dev/null "${BASE_URL}/${BUNDLE}" || fail "the bundle ${BUNDLE} was not served" 351 352 # The policy is what keeps a refresh token in local storage out of reach of 353 # an injected script, so its absence is a finding rather than a detail. 354 curl -fsS -D - -o /dev/null "${BASE_URL}/" | grep -qi 'content-security-policy' \ 355 || fail "the application was served without a content security policy" 356 357 # The release task is restricted to main. A push to anything else must not 358 # produce it at all, rather than produce it and skip it, since a skipped 359 # task fails the job. 360 log "checking that a branch push leaves the restricted task out" 361 BRANCH_BODY=$(printf '{"sha":"%s","ref":"refs/heads/feature"}' "${SHA}") 362 BRANCH_SIG=$(printf '%s' "${BRANCH_BODY}" | openssl dgst -sha256 -hmac "${SECRET}" | sed 's/^.*[= ]//') 363 BRANCH=$(curl -fsS -X POST "${BASE_URL}/api/v1/projects/demo/trigger" \ 364 -H 'Content-Type: application/json' \ 365 -H "X-Hub-Signature-256: sha256=${BRANCH_SIG}" \ 366 -d "${BRANCH_BODY}") 367 printf '%s\n' "${BRANCH}" 368 369 BRANCH_JOB=$(printf '%s' "${BRANCH}" | sed 's/.*"job_id":"\([^"]*\)".*/\1/') 370 [ -n "${BRANCH_JOB}" ] || fail "no branch job was created" 371 372 i=0 373 while [ "${i}" -lt 90 ]; do 374 BRANCH_STATE=$(job_state "${BRANCH_JOB}") 375 case "${BRANCH_STATE}" in 376 success|failed|cancelled) break ;; 377 esac 378 i=$((i + 1)) 379 sleep 1 380 done 381 382 # The trigger only records the job, so the tasks are counted off the 383 # finished job rather than the response. 384 BRANCH_LIST=$(curl -fsS "${BASE_URL}/api/v1/jobs/${BRANCH_JOB}/tasks") 385 BRANCH_TASKS=$(objects "${BRANCH_LIST}" | grep -c '"name":"') 386 [ "${BRANCH_TASKS}" -eq 2 ] \ 387 || fail "expected 2 tasks on a branch, got ${BRANCH_TASKS}" 388 389 printf '%s' "${BRANCH_LIST}" | grep -q '"name":"release"' \ 390 && fail "the release task should not exist on a branch job" 391 392 printf 'branch job finished as %s with %s tasks and no release task\n' \ 393 "${BRANCH_STATE}" "${BRANCH_TASKS}" 394 [ "${BRANCH_STATE}" = success ] || fail "the branch job ended as ${BRANCH_STATE}" 395 396 printf '\nPASSED\n'