First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+233
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* RolloutRail — verify a deployment per target, by what is actually serving.
|
||||
*
|
||||
* All three traps below are common in practice, and they share one
|
||||
* shape: a signal that is easy to collect standing in for the one that
|
||||
* matters.
|
||||
*
|
||||
* 1. **A green pipeline is not a deployed artifact.** An unpinned linter
|
||||
* started failing, the deploy job's dependency was unsatisfied, and the
|
||||
* job was SKIPPED. A skipped job does not fail a pipeline, so the
|
||||
* pipeline stayed green and nothing shipped.
|
||||
*
|
||||
* 2. **A 200 is not your build.** The old container answering healthily
|
||||
* returns exactly the same 200 as the new one. Worse, a loose smoke test
|
||||
* can pass on HTTP 500, so even the status check may not be checking
|
||||
* what it appears to. Only the served ARTIFACT — a version, a
|
||||
* commit SHA, a build id — distinguishes them.
|
||||
*
|
||||
* 3. **An untouched target reports its previous success.** During a staged
|
||||
* rollout, querying pipeline status for a repo that has not been merged
|
||||
* yet returns the PRE-rollout run, which is green. A monitor that polls
|
||||
* status therefore reports "all green" while half the fleet has not
|
||||
* started.
|
||||
*
|
||||
* So the only pass here is an observed artifact matching the intended one,
|
||||
* obtained independently of the deploy. Everything else is NOT ASSESSED, and
|
||||
* says which weaker signal it settled for.
|
||||
*
|
||||
* Pure: takes observations, returns findings. It deploys nothing.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CapabilityReport,
|
||||
type Finding,
|
||||
type FormatOptions,
|
||||
evidence,
|
||||
formatReport as renderReport,
|
||||
} from '@extant2000/evidence-record'
|
||||
|
||||
/** How a target's served artifact was established. */
|
||||
export type Probe =
|
||||
/** The target was asked what it is serving, and answered. The only pass. */
|
||||
| 'artifact-probe'
|
||||
/** CI said the pipeline was green. Says nothing about what is serving. */
|
||||
| 'pipeline-status'
|
||||
/** Something answered a request. Says nothing about which build. */
|
||||
| 'http-status'
|
||||
/** Nothing was checked. */
|
||||
| 'none'
|
||||
|
||||
export type PipelineStatus = 'success' | 'failed' | 'skipped' | 'running' | 'created' | 'canceled'
|
||||
|
||||
export interface Target {
|
||||
name: string
|
||||
/** The artifact intended for this rollout — a version, SHA or build id. */
|
||||
expected: string
|
||||
/** What the target reports serving. `null` when unknown. */
|
||||
observed: string | null
|
||||
via: Probe
|
||||
/** Where the observation came from — a URL, a command, a dashboard. */
|
||||
where: string
|
||||
httpStatus?: number
|
||||
pipelineStatus?: PipelineStatus
|
||||
/**
|
||||
* Jobs that actually RAN. A pipeline can be green with the deploy job
|
||||
* skipped, and this is the only field that shows it.
|
||||
*/
|
||||
jobsRun?: string[]
|
||||
/** Whether this target has been started at all in a staged rollout. */
|
||||
started?: boolean
|
||||
}
|
||||
|
||||
export interface RolloutOptions {
|
||||
/** Name of the job that performs the deploy. Defaults to "deploy". */
|
||||
deployJob?: string
|
||||
}
|
||||
|
||||
function targetEvidence(t: Target) {
|
||||
return evidence.file(t.where, undefined,
|
||||
t.observed === null ? 'no artifact observed' : `serving ${t.observed}`,
|
||||
`probe: ${t.via}${t.pipelineStatus ? `, pipeline ${t.pipelineStatus}` : ''}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* A green pipeline whose deploy job never ran.
|
||||
*
|
||||
* The highest-value check in the library, because the job list is the only
|
||||
* place this is visible. A skipped job does not fail a pipeline.
|
||||
*/
|
||||
function skippedDeploy(t: Target, deployJob: string): Finding | null {
|
||||
if (t.pipelineStatus !== 'success' || !t.jobsRun) return null
|
||||
if (t.jobsRun.includes(deployJob)) return null
|
||||
|
||||
return {
|
||||
id: `${t.name}/skipped-deploy`,
|
||||
summary: `${t.name}: pipeline is green and "${deployJob}" never ran`,
|
||||
determination: 'fail',
|
||||
severity: 'critical',
|
||||
detail: `A skipped job does not fail a pipeline. Jobs that ran: ${t.jobsRun.join(', ') || 'none'}. Nothing shipped, and the only place that is visible is the job list.`,
|
||||
evidence: [evidence.command(`pipeline jobs for ${t.name}`, 0, t.jobsRun.join(', ') || '(none)', t.where)],
|
||||
}
|
||||
}
|
||||
|
||||
function findingFor(t: Target): Finding {
|
||||
const id = t.name
|
||||
|
||||
if (t.started === false) {
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name} has not started`,
|
||||
determination: 'not-assessed',
|
||||
severity: 'medium',
|
||||
// Trap 3. A status query here returns the PRE-rollout run, which is
|
||||
// green, and reads as success for work that has not begun.
|
||||
detail: 'Any status read from this target right now describes the previous rollout, not this one. It is green because nothing has happened yet.',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
if (t.via === 'none' || t.observed === null) {
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name}: nothing was observed serving`,
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
if (t.via === 'pipeline-status') {
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name}: pipeline reported ${t.pipelineStatus ?? 'success'}, artifact not checked`,
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: 'A green pipeline is not a deployed artifact. Ask the target what it is serving.',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
if (t.via === 'http-status') {
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name}: answered HTTP ${t.httpStatus ?? '2xx'}, artifact not checked`,
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: 'The previous build returns the same status as the new one. A response code identifies neither the version nor, with a loose smoke test, even the health: a check that accepts any response passes on HTTP 500.',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
if (t.observed !== t.expected) {
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name} is serving ${t.observed}, expected ${t.expected}`,
|
||||
determination: 'fail',
|
||||
severity: 'critical',
|
||||
detail: 'The target is up, healthy, and running the previous build. This is the failure a status check cannot see.',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
summary: `${t.name} is serving ${t.observed}`,
|
||||
determination: 'pass',
|
||||
severity: 'info',
|
||||
evidence: [targetEvidence(t)],
|
||||
}
|
||||
}
|
||||
|
||||
/** Verify a rollout target by target. */
|
||||
export function verifyRollout(targets: Target[], opts: RolloutOptions = {}): CapabilityReport {
|
||||
const list = targets ?? []
|
||||
const deployJob = opts.deployJob ?? 'deploy'
|
||||
|
||||
if (list.length === 0) {
|
||||
return {
|
||||
capability: 'RolloutRail',
|
||||
scope: 'no targets',
|
||||
examined: 0,
|
||||
findings: [],
|
||||
notes: ['No targets were supplied, so no rollout was verified.'],
|
||||
}
|
||||
}
|
||||
|
||||
const skipped = list.map(t => skippedDeploy(t, deployJob)).filter((f): f is Finding => f !== null)
|
||||
const confirmed = list.filter(t => t.via === 'artifact-probe' && t.observed === t.expected && t.started !== false)
|
||||
|
||||
return {
|
||||
capability: 'RolloutRail',
|
||||
scope: `${list.length} target(s)`,
|
||||
examined: list.length,
|
||||
findings: [...skipped, ...list.map(findingFor)],
|
||||
notes: [
|
||||
`${confirmed.length} of ${list.length} target(s) confirmed serving the intended artifact by independent probe`,
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether it is safe to continue a staged rollout.
|
||||
*
|
||||
* Deliberately strict: NOT ASSESSED blocks exactly as a failure does. The
|
||||
* point of a staged rollout is that the next wave depends on the previous
|
||||
* one having worked, and "we did not check" is not evidence that it did.
|
||||
*/
|
||||
export function safeToProceed(
|
||||
targets: Target[],
|
||||
opts: RolloutOptions = {},
|
||||
): { ok: boolean, reason: string } {
|
||||
const report = verifyRollout(targets, opts)
|
||||
if (report.examined === 0) return { ok: false, reason: 'No targets were verified.' }
|
||||
|
||||
const bad = report.findings.filter(f => f.determination === 'fail')
|
||||
const unknown = report.findings.filter(f => f.determination === 'not-assessed')
|
||||
|
||||
if (bad.length > 0) {
|
||||
return { ok: false, reason: `${bad.length} target(s) failed: ${bad.map(f => f.id).join(', ')}` }
|
||||
}
|
||||
if (unknown.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `${unknown.length} target(s) unverified: ${unknown.map(f => f.id).join(', ')}. The next wave depends on this one having worked, and "not checked" is not evidence that it did.`,
|
||||
}
|
||||
}
|
||||
return { ok: true, reason: `All ${report.examined} target(s) confirmed serving the intended artifact.` }
|
||||
}
|
||||
|
||||
/** Render a rollout for a terminal, CI log or evidence package. */
|
||||
export function formatRollout(targets: Target[], opts: RolloutOptions & FormatOptions = {}): string {
|
||||
return renderReport(verifyRollout(targets, opts), opts)
|
||||
}
|
||||
Reference in New Issue
Block a user