#!/usr/bin/env bash # run-pipeline.sh -- start a pipeline, record the job pods while they exist, and print what # each job reported. # # Usage: # ./scripts/run-pipeline.sh # # What it does, in order: # 1. checks the context and that the namespace is this sample's; # 2. starts a pipeline on the project's default branch, the same thing "Run pipeline" does # in GitLab's own UI; # 3. watches the namespace for job pods. A job pod exists only while its job runs and is # deleted when the job ends, so each one's name, UID, runtime class and node are # recorded as it appears; # 4. waits for the pipeline to finish and prints each job's status and log; # 5. runs the same three readings in an ordinary pod on the node a job pod used, and # removes that pod by its exact name. set -euo pipefail # shellcheck source=common.sh . "$(cd "$(dirname "$0")" && pwd)/common.sh" ORDINARY_POD=gitlab-ci-ordinary JOB_POD_SELECTOR="app=gitlab-ci-job" # --------------------------------------------------------------------------- say "1/5 context and namespace" # --------------------------------------------------------------------------- need_tools kubectl base64 jq check_cluster require_namespace echo " context '$KUBE_CONTEXT', namespace '$NAMESPACE' is this sample's." b64s() { printf '%s' "$1" | { base64 -w0 2>/dev/null || base64 | tr -d '\n'; }; } work="$(mktemp -d "${TMPDIR:-/tmp}/gitlab-ci-run.XXXXXX")" trap 'rm -rf "$work"' EXIT INT TERM chmod 700 "$work" # --------------------------------------------------------------------------- say "2/5 starting the pipeline" # --------------------------------------------------------------------------- cat > "$work/start.rb" <> "$work/start.rb" <<'RUBY' user = User.find_by_username('root') raise 'no root user on this GitLab' if user.nil? project = Project.find_by_full_path("#{user.username}/#{PROJECT_PATH}") raise "no project #{user.username}/#{PROJECT_PATH}. Run scripts/bootstrap-runner.sh first." if project.nil? branch = project.default_branch || 'main' # The same call GitLab's own "Run pipeline" button makes, options included. # ignore_skip_ci matters: bootstrap-runner.sh committed the pipeline with [ci skip] so the # commit would start nothing, and without this the explicit run is skipped too. Measured: # the pipeline is created with status "skipped" and no jobs at all. response = ::Ci::CreatePipelineService .new(project, user, ref: branch) .execute(:web, ignore_skip_ci: true, save_on_errors: false) pipeline = response.payload if pipeline.nil? || !pipeline.persisted? raise "could not create a pipeline: #{response.message || pipeline&.errors&.full_messages&.join(', ')}" end puts "PIPELINE_ID=#{pipeline.id}" puts "PIPELINE_SHA=#{pipeline.sha}" pipeline.builds.order(:id).each { |b| puts "JOB=#{b.id} #{b.name}" } RUBY start_out="$(gitlab_exec < "$work/start.rb")" \ || die "GitLab could not start a pipeline. Its output is above." PIPELINE_ID="$(sed -n 's/^PIPELINE_ID=//p' <<<"$start_out")" PIPELINE_SHA="$(sed -n 's/^PIPELINE_SHA=//p' <<<"$start_out")" [[ -n "$PIPELINE_ID" ]] \ || die "GitLab reported no pipeline id. Its output was: $start_out" echo " pipeline $PIPELINE_ID on ${PIPELINE_SHA:0:8}, with these jobs:" sed -n 's/^JOB=/ /p' <<<"$start_out" # --------------------------------------------------------------------------- say "3/5 and 4/5 the job pods, and the pipeline" # --------------------------------------------------------------------------- # A job pod lives only while its job runs, so the watch and the wait are one loop: the pods # are recorded as they appear, and the loop ends when the pipeline reaches a final status. A # record taken after the pipeline finished would find nothing. cat > "$work/status.rb" <> "$work/status.rb" <<'RUBY' pipeline = Ci::Pipeline.find(PIPELINE_ID) puts "PIPELINE_STATUS=#{pipeline.status}" pipeline.builds.order(:id).each { |b| puts "JOB_STATUS=#{b.id} #{b.name} #{b.status}" } RUBY declare -A pod_uid pod_rclass pod_node pod_vols pod_volsecrets pod_priv pod_order=() pipeline_status="" echo " watching for job pods. Each one is created when its job starts and deleted when it" echo " ends, so this is the only moment they can be read." printf '\n %-50s %-38s %-18s %s\n' POD UID "RUNTIME CLASS" PHASE # Up to 40 minutes: a sandboxed job pod has a node to be provisioned, a VM to boot and its # images to pull before the script runs. deadline=$(( $(date +%s) + 2400 )) last_status_check=0 while :; do # Record any job pod that is there now. kubectl goes straight into jq, and only the fields # below are kept. # # jq writes "-" for a field it has no value for, rather than an empty string. Tab is an IFS # whitespace character, so bash collapses a run of tabs into one delimiter: an empty field # would vanish and every field after it would shift left. Measured here before the # placeholder went in: a pod first seen while Pending, with no nodeName yet, had its PHASE # read as its node, and the table showed "Pending" in the NODE column. pods_json="$("${KCTL[@]}" get pod -n "$NAMESPACE" -l "$JOB_POD_SELECTOR" -o json 2>/dev/null)" \ || die "could not list the job pods in '$NAMESPACE'. That is a failed read, not a report that there are none." while IFS=$'\t' read -r name uid rclass node phase vols volsecrets priv; do [[ -n "$name" ]] || continue if [[ -z "${pod_uid[$name]:-}" ]]; then pod_order+=("$name") printf ' %-50s %-38s %-18s %s\n' "$name" "$uid" "$rclass" "$phase" fi pod_uid[$name]="$uid" pod_rclass[$name]="$rclass" pod_vols[$name]="$vols" pod_volsecrets[$name]="$volsecrets" pod_priv[$name]="$priv" # A pod first seen while Pending has no node yet, so the node is taken from whichever # poll first reports one and is never overwritten with the placeholder again. Written as # an if, not `[[ ... ]] && ...`: under `set -e` a standalone && list whose test fails # returns non-zero and ends the script. if [[ "$node" != "-" ]]; then pod_node[$name]="$node" else pod_node[$name]="${pod_node[$name]:-$node}" fi # What the pod MOUNTS, by name, never by value: this is how the page shows that the # runner authentication token is not in the job pod. Read while the pod exists, because # it is deleted when its job ends. # # Both ways a Secret can be mounted are collected: a volume with a `secret` of its own, # and a `projected` volume with a `secret` among its sources. Only the first was read at # first, so a pod mounting the runner's Secret through a projected volume would have # passed the check below. done < <(jq -r '(.items // [])[] | [ .metadata.name, .metadata.uid, (.spec.runtimeClassName // "-"), (.spec.nodeName // "-"), .status.phase, ([(.spec.volumes // [])[].name] | join(",") | if . == "" then "-" else . end), ([(.spec.volumes // [])[] | (.secret.secretName // empty), ((.projected.sources // [])[] | .secret.name // empty)] | unique | join(",") | if . == "" then "-" else . end), ([((.spec.initContainers // []) + (.spec.containers // []))[] | "\(.name)=\(.securityContext.privileged // false)"] | join(" ") | if . == "" then "-" else . end) ] | @tsv' <<<"$pods_json") now=$(date +%s) (( now < deadline )) || die "the pipeline had not finished after 40 minutes. Its jobs and the runner manager's log say where it is: kubectl --context $KUBE_CONTEXT -n $NAMESPACE logs deploy/gitlab-runner --tail=80 kubectl --context $KUBE_CONTEXT -n $NAMESPACE get pods -l $JOB_POD_SELECTOR" # GitLab's Rails console takes about half a minute to start, so the pipeline's status is # read every 30 seconds while the pods are polled every 2. if (( now - last_status_check >= 30 )); then last_status_check=$now status_out="$(gitlab_exec < "$work/status.rb" 2>/dev/null || true)" pipeline_status="$(sed -n 's/^PIPELINE_STATUS=//p' <<<"$status_out")" case "$pipeline_status" in success|failed|canceled|skipped) break ;; "") ;; # a read that did not come back; try again rather than treat it as finished *) echo " pipeline $PIPELINE_ID: $pipeline_status" ;; esac fi sleep 2 done echo echo " pipeline $PIPELINE_ID finished: $pipeline_status" sed -n 's/^JOB_STATUS=/ job /p' <<<"$status_out" pod_count="${#pod_order[@]}" job_count="$(grep -c '^JOB_STATUS=' <<<"$status_out" || true)" echo echo " jobs GitLab reported: $job_count" echo " job pods seen: $pod_count" # One pod per job, no fewer. A positive count on its own is not enough: with it, one pod seen # out of two jobs would pass, and so would a run in which the label this script discovers pods # by had been changed in config.toml, since the pods it could not see would simply be absent # from the count. Comparing against the number of jobs GitLab itself reported makes a pod this # script could not see a failure rather than a silence. (( job_count > 0 )) \ || die "GitLab reported no jobs for pipeline $PIPELINE_ID, so there is nothing to show that a job was isolated. A pipeline that was created with no jobs at all does this; its status is above." (( pod_count == job_count )) \ || die "pipeline $PIPELINE_ID had $job_count job(s) but only $pod_count job pod(s) were seen while it ran. Every job must have had a pod, and every pod must have been visible to this script. The usual cause is that the pod_labels block in runner/config.toml.template no longer carries '$JOB_POD_SELECTOR', so a pod existed and could not be found; the other is a job that never started a pod at all." # Every job pod must have been a sandbox. One with no runtime class would mean the runner # ignored runtime_class_name, which is the whole of what this sample configures, and that job # ran as an ordinary pod on the node. A runner release that does not know the setting does # exactly that, silently, which is why this is checked rather than assumed. unsandboxed="" for name in "${pod_order[@]}"; do rc="${pod_rclass[$name]}" if [[ "$rc" != "isolated-sandbox" ]]; then if [[ "$rc" == "-" ]]; then rc=""; fi unsandboxed+=" $name (runtime class: $rc)"$'\n' fi done [[ -z "$unsandboxed" ]] \ || die "a job pod did not carry runtimeClassName isolated-sandbox, so that job did not run in a sandbox: $unsandboxed Check that runner/config.toml.template's runtime_class_name reached the Secret, that the runner manager was restarted after it changed, and that the runner release supports runtime_class_name; the run on the page used v19.5.0." uid_count="$(printf '%s\n' "${pod_uid[@]}" | sort -u | grep -c .)" echo " distinct pod UIDs: $uid_count (one per job, so no job reused another's pod)" # A pod UID is unique in the cluster, so a repeated UID among the pods seen would mean two jobs # shared one pod, and a checkout or a build cache could carry from one to the next. # # What these two counts establish, and what they do not. They are a lower bound: the pods are # read every two seconds, so a pod that was created and deleted between two polls is invisible, # and a job that ran twice could therefore look like one pod. Comparing against the jobs GitLab # reported catches the case that matters here, a job whose pod was never seen at all, which is # what a runner ignoring runtime_class_name or a changed pod label would produce. The page says # the same, rather than claiming an exact census. (( uid_count == pod_count )) \ || die "$pod_count job pod(s) were seen but only $uid_count distinct pod UID(s), so two jobs shared a pod and nothing separated their checkouts." for name in "${pod_order[@]}"; do printf ' %s\n uid %s runtime class %s on %s\n containers, privileged flag: %s\n volumes: %s\n volumes from a Secret: %s\n' \ "$name" "${pod_uid[$name]}" "${pod_rclass[$name]}" "${pod_node[$name]}" \ "${pod_priv[$name]}" "${pod_vols[$name]}" "${pod_volsecrets[$name]}" done # The runner authentication token is in the Secret the MANAGER mounts. A job pod that mounted # it would put the credential inside the thing running pipeline code, which is the one place # this sample must keep it out of. Checked rather than asserted on the page. for name in "${pod_order[@]}"; do case ",${pod_volsecrets[$name]}," in *",$RUNNER_SECRET,"*) die "job pod $name mounts Secret '$RUNNER_SECRET', which holds the runner authentication token. A job pod must not have it: pipeline code runs in there." ;; esac done echo " no job pod mounts Secret '$RUNNER_SECRET', which is where the runner authentication token is." [[ "$pipeline_status" == success ]] \ || die "the pipeline finished as '$pipeline_status'. The job logs below in GitLab, and the runner manager's log, say why: kubectl --context $KUBE_CONTEXT -n $NAMESPACE logs deploy/gitlab-runner --tail=80" # --------------------------------------------------------------------------- say "the job logs" # --------------------------------------------------------------------------- cat > "$work/trace.rb" <> "$work/trace.rb" <<'RUBY' pipeline = Ci::Pipeline.find(PIPELINE_ID) pipeline.builds.order(:id).each do |build| puts "===== job #{build.id} #{build.name} (#{build.status}) =====" puts(build.trace.raw || '(no log)') end RUBY # GitLab stores a job log with a per-line timestamp and stream marker, its own # section_start and section_end markers, and the runner's terminal colour codes. None of # that is readable here, so it is taken out: the lines, their order and their text are # exactly as the job printed them. gitlab_exec < "$work/trace.rb" \ | sed -E -e 's/\x1b\[[0-9;]*[A-Za-z]//g' \ -e 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z [0-9]{2}[A-Za-z][+]? ?//' \ -e '/^section_(start|end):[0-9]+:/d' \ || die "could not read the job logs from GitLab." # --------------------------------------------------------------------------- say "5/5 what an ordinary pod on the same node reports" # --------------------------------------------------------------------------- # The comparison only means something on the node a job pod actually ran on: a reading from # another node compares two unrelated machines. NODE_NAME="" for name in "${pod_order[@]}"; do if [[ "${pod_node[$name]}" != "-" ]]; then NODE_NAME="${pod_node[$name]}" break fi done [[ -n "$NODE_NAME" ]] \ || die "none of the job pods recorded a node, so there is no node to compare against." echo " the job pods ran on $NODE_NAME." # A pod this script made for itself, at a name of its own, removed by that exact name and # never by a selector: a selector meant for one helper pod can match another demo's. "${KCTL[@]}" delete pod "$ORDINARY_POD" -n "$NAMESPACE" --ignore-not-found --wait >/dev/null sed -e "s||$NAMESPACE|g" -e "s||$NODE_NAME|g" \ "$SAMPLE_DIR/deploy/ordinary-pod.yaml" | "${KCTL[@]}" apply -f - >/dev/null \ || die "could not create the ordinary pod." cleanup_ordinary() { "${KCTL[@]}" delete pod "$ORDINARY_POD" -n "$NAMESPACE" --ignore-not-found --wait=false >/dev/null 2>&1 || true } trap 'cleanup_ordinary; rm -rf "$work"' EXIT INT TERM "${KCTL[@]}" wait --for=condition=Ready "pod/$ORDINARY_POD" -n "$NAMESPACE" --timeout=180s >/dev/null \ || die "the ordinary pod did not become Ready on $NODE_NAME." echo echo " --- its kernel (the node's, shared with every ordinary pod on it) ---" "${KCTL[@]}" exec -n "$NAMESPACE" "$ORDINARY_POD" -- uname -rv echo " --- its uptime (the node's, not this pod's age) ---" "${KCTL[@]}" exec -n "$NAMESPACE" "$ORDINARY_POD" -- sh -c "cut -d' ' -f1 /proc/uptime" echo " --- its block devices (the instance's NVMe disks) ---" "${KCTL[@]}" exec -n "$NAMESPACE" "$ORDINARY_POD" -- ls /sys/block echo " --- its product name (the EC2 instance type) ---" "${KCTL[@]}" exec -n "$NAMESPACE" "$ORDINARY_POD" -- cat /sys/class/dmi/id/product_name "${KCTL[@]}" delete pod "$ORDINARY_POD" -n "$NAMESPACE" --ignore-not-found --wait >/dev/null trap 'rm -rf "$work"' EXIT INT TERM echo echo " removed pod/$ORDINARY_POD." rm -rf "$work" trap - EXIT INT TERM say "the pipeline is done" cat <