First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* The four things a check can conclude — and the three of them that are not
|
||||
* a pass.
|
||||
*
|
||||
* A boolean cannot carry this. `passed: true` collapses "we checked and it
|
||||
* held" together with "there was nothing to check", and those are the two
|
||||
* cases a reviewer most needs kept apart. An earlier version of these
|
||||
* libraries said VACUOUS for the second, which is precise and unreadable; this
|
||||
* replaces the jargon without softening the meaning.
|
||||
*/
|
||||
export type Determination =
|
||||
/** Something was examined against the criterion, and it held. */
|
||||
| 'pass'
|
||||
/** Something was examined against the criterion, and it did not hold. */
|
||||
| 'fail'
|
||||
/** The criterion was examined and genuinely does not apply here. */
|
||||
| 'not-applicable'
|
||||
/** Nothing was examined. Never a pass. */
|
||||
| 'not-assessed'
|
||||
|
||||
/**
|
||||
* One-word label for a column or badge.
|
||||
*
|
||||
* `not-assessed` deliberately does NOT get a neutral word like "skipped" —
|
||||
* "skipped" reads as a decision someone made, and this is usually an absence
|
||||
* nobody noticed.
|
||||
*/
|
||||
export function label(d: Determination): string {
|
||||
switch (d) {
|
||||
case 'pass': return 'MET'
|
||||
case 'fail': return 'NOT MET'
|
||||
case 'not-applicable': return 'N/A'
|
||||
case 'not-assessed': return 'NOT ASSESSED'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A full sentence a reviewer can read without a glossary.
|
||||
*
|
||||
* `subject` names what was being checked, so the sentence stands alone when
|
||||
* it is lifted out of the report and pasted into an email — which is how
|
||||
* these findings actually travel.
|
||||
*/
|
||||
export function describe(d: Determination, subject = 'This item'): string {
|
||||
switch (d) {
|
||||
case 'pass':
|
||||
return `${subject} was checked and met the criterion.`
|
||||
case 'fail':
|
||||
return `${subject} was checked and did not meet the criterion.`
|
||||
case 'not-applicable':
|
||||
return `${subject} was examined and the criterion does not apply to it.`
|
||||
case 'not-assessed':
|
||||
return `${subject} was not assessed. This is not a pass — an unassessed control is not an implemented control.`
|
||||
}
|
||||
}
|
||||
|
||||
/** True only for the one determination that is genuinely a clean result. */
|
||||
export function isPass(d: Determination): boolean {
|
||||
return d === 'pass'
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll several determinations into one.
|
||||
*
|
||||
* Order matters and is deliberate:
|
||||
* any fail → fail
|
||||
* nothing examined → not-assessed (NOT a pass; this is the vacuous case)
|
||||
* everything N/A → not-applicable
|
||||
* otherwise → pass
|
||||
*
|
||||
* The empty list returns `not-assessed` rather than `pass`. `[].every(...)`
|
||||
* being true is the exact trap this type exists to close.
|
||||
*/
|
||||
export function rollUp(ds: Determination[]): Determination {
|
||||
if (ds.some(d => d === 'fail')) return 'fail'
|
||||
if (ds.length === 0) return 'not-assessed'
|
||||
if (ds.some(d => d === 'not-assessed')) return 'not-assessed'
|
||||
if (ds.every(d => d === 'not-applicable')) return 'not-applicable'
|
||||
return 'pass'
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Where a finding came from.
|
||||
*
|
||||
* The rule this type exists to enforce: a finding names the artefact that
|
||||
* produced it. SideDoor already did this — it prints the service, the port
|
||||
* binding and the compose file — and it is noticeably the most useful of these
|
||||
* outputs to read. A result that is asserted without a source leaves the
|
||||
* reader to take it on trust, and an unverified claim should count as absent.
|
||||
*/
|
||||
|
||||
/** A location in source or configuration. */
|
||||
export interface FileEvidence {
|
||||
kind: 'file'
|
||||
path: string
|
||||
/** 1-indexed. Omitted when the finding is about the file as a whole. */
|
||||
line?: number
|
||||
/** The matched text. Keep it short — one line, trimmed. */
|
||||
excerpt?: string
|
||||
}
|
||||
|
||||
/** A row in a database. */
|
||||
export interface RecordEvidence {
|
||||
kind: 'record'
|
||||
table: string
|
||||
id: string
|
||||
/** The column that carried the value the finding turns on. */
|
||||
field?: string
|
||||
value?: string
|
||||
}
|
||||
|
||||
/** Output of a command that was actually run. */
|
||||
export interface CommandEvidence {
|
||||
kind: 'command'
|
||||
command: string
|
||||
exitCode?: number
|
||||
excerpt?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A number that was measured, together with how much data stood behind it.
|
||||
*
|
||||
* `samples` is not decoration. An availability of 100% over zero samples and
|
||||
* an availability of 100% over 8,640 samples are different claims, and only
|
||||
* one of them is evidence.
|
||||
*/
|
||||
export interface MeasurementEvidence {
|
||||
kind: 'measurement'
|
||||
metric: string
|
||||
/** Null when it could not be computed. Never substitute a flattering default. */
|
||||
value: number | null
|
||||
unit?: string
|
||||
samples: number
|
||||
/** e.g. "2026-07-01 → 2026-07-31". */
|
||||
window?: string
|
||||
}
|
||||
|
||||
/** A written source: a policy, a standard, a control catalogue. */
|
||||
export interface DocumentEvidence {
|
||||
kind: 'document'
|
||||
title: string
|
||||
section?: string
|
||||
url?: string
|
||||
}
|
||||
|
||||
export type EvidenceSource =
|
||||
| FileEvidence
|
||||
| RecordEvidence
|
||||
| CommandEvidence
|
||||
| MeasurementEvidence
|
||||
| DocumentEvidence
|
||||
|
||||
export interface Evidence {
|
||||
source: EvidenceSource
|
||||
/** ISO 8601. When the observation was made, if the caller tracks it. */
|
||||
observedAt?: string
|
||||
/** One line on why this artefact supports the finding. */
|
||||
note?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A short, copy-pasteable citation — the thing a reader scans for.
|
||||
*
|
||||
* Formats follow the conventions each kind is already read in: `path:line`
|
||||
* for source (clickable in most terminals), `table#id` for rows.
|
||||
*/
|
||||
export function cite(e: Evidence): string {
|
||||
const s = e.source
|
||||
switch (s.kind) {
|
||||
case 'file':
|
||||
return s.line ? `${s.path}:${s.line}` : s.path
|
||||
case 'record':
|
||||
return s.field ? `${s.table}#${s.id} (${s.field})` : `${s.table}#${s.id}`
|
||||
case 'command':
|
||||
return s.exitCode === undefined ? `$ ${s.command}` : `$ ${s.command} → exit ${s.exitCode}`
|
||||
case 'measurement': {
|
||||
const v = s.value === null ? 'not computable' : `${s.value}${s.unit ?? ''}`
|
||||
const n = `${s.samples} sample${s.samples === 1 ? '' : 's'}`
|
||||
return s.window ? `${s.metric} = ${v} (${n}, ${s.window})` : `${s.metric} = ${v} (${n})`
|
||||
}
|
||||
case 'document':
|
||||
return s.section ? `${s.title} § ${s.section}` : s.title
|
||||
}
|
||||
}
|
||||
|
||||
/** Longer form: the citation plus whatever context was attached to it. */
|
||||
export function citeVerbose(e: Evidence): string[] {
|
||||
const out = [cite(e)]
|
||||
const s = e.source
|
||||
if ((s.kind === 'file' || s.kind === 'command') && s.excerpt) out.push(` ${s.excerpt.trim()}`)
|
||||
if (s.kind === 'record' && s.value !== undefined) out.push(` value: ${s.value}`)
|
||||
if (e.note) out.push(` ${e.note}`)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Convenience constructors, so call sites stay one line. */
|
||||
export const evidence = {
|
||||
file(path: string, line?: number, excerpt?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'file', path, line, excerpt }, note }
|
||||
},
|
||||
record(table: string, id: string, field?: string, value?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'record', table, id, field, value }, note }
|
||||
},
|
||||
command(command: string, exitCode?: number, excerpt?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'command', command, exitCode, excerpt }, note }
|
||||
},
|
||||
measurement(
|
||||
metric: string,
|
||||
value: number | null,
|
||||
samples: number,
|
||||
opts: { unit?: string, window?: string, note?: string } = {},
|
||||
): Evidence {
|
||||
return {
|
||||
source: { kind: 'measurement', metric, value, samples, unit: opts.unit, window: opts.window },
|
||||
note: opts.note,
|
||||
}
|
||||
},
|
||||
document(title: string, section?: string, url?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'document', title, section, url }, note }
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a measurement out loud, including the case where there is nothing to
|
||||
* read.
|
||||
*
|
||||
* `uptime null, met=false` was correct and unreadable. A reader should not
|
||||
* have to know that zero samples is not 100% in order to understand the line.
|
||||
*/
|
||||
export function describeMeasurement(s: MeasurementEvidence): string {
|
||||
if (s.samples === 0) {
|
||||
return `No samples for ${s.metric}${s.window ? ` in ${s.window}` : ''}; it cannot be computed.`
|
||||
}
|
||||
if (s.value === null) {
|
||||
return `${s.metric} could not be computed from the ${s.samples} available sample${s.samples === 1 ? '' : 's'}.`
|
||||
}
|
||||
return `${s.metric} is ${s.value}${s.unit ?? ''} over ${s.samples} sample${s.samples === 1 ? '' : 's'}${s.window ? ` (${s.window})` : ''}.`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export type { Determination } from './determination.js'
|
||||
export { label, describe, isPass, rollUp } from './determination.js'
|
||||
|
||||
export type {
|
||||
Evidence,
|
||||
EvidenceSource,
|
||||
FileEvidence,
|
||||
RecordEvidence,
|
||||
CommandEvidence,
|
||||
MeasurementEvidence,
|
||||
DocumentEvidence,
|
||||
} from './evidence.js'
|
||||
export { cite, citeVerbose, evidence, describeMeasurement } from './evidence.js'
|
||||
|
||||
export type {
|
||||
Severity,
|
||||
Finding,
|
||||
CapabilityReport,
|
||||
ReportProblem,
|
||||
FormatOptions,
|
||||
} from './report.js'
|
||||
export { overall, validateReport, formatReport, formatRollUp } from './report.js'
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
import { type Determination, label, describe, rollUp } from './determination.js'
|
||||
import { type Evidence, cite, citeVerbose } from './evidence.js'
|
||||
|
||||
export type Severity = 'critical' | 'high' | 'medium' | 'info'
|
||||
|
||||
const RANK: Record<Severity, number> = { critical: 0, high: 1, medium: 2, info: 3 }
|
||||
|
||||
export interface Finding {
|
||||
/**
|
||||
* Stable within one report only. These identify a row in THIS run, not a
|
||||
* durable record — do not persist them as entity keys.
|
||||
*/
|
||||
id: string
|
||||
/** One line, in the reader's language, not the checker's. */
|
||||
summary: string
|
||||
determination: Determination
|
||||
severity: Severity
|
||||
/**
|
||||
* What produced this finding. Required, and allowed to be empty only for
|
||||
* `not-assessed` — where the whole point is that nothing was observed.
|
||||
*/
|
||||
evidence: Evidence[]
|
||||
/** Optional paragraph: why it matters, what to do. */
|
||||
detail?: string
|
||||
}
|
||||
|
||||
export interface CapabilityReport {
|
||||
/** The capability that produced this, e.g. "SideDoor". */
|
||||
capability: string
|
||||
/** What was examined, in a phrase: "49 compose services on the build host". */
|
||||
scope: string
|
||||
/** How many items were actually examined. Zero forces `not-assessed`. */
|
||||
examined: number
|
||||
findings: Finding[]
|
||||
/** Free-form lines printed under the summary — caveats, exclusions. */
|
||||
notes?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The report's overall determination, derived rather than asserted.
|
||||
*
|
||||
* Deriving it closes the gap where a capability reports zero findings and a
|
||||
* reader infers a pass. Zero findings over zero examined items is
|
||||
* `not-assessed`, and it says so.
|
||||
*/
|
||||
export function overall(r: CapabilityReport): Determination {
|
||||
if (r.examined === 0) return 'not-assessed'
|
||||
if (r.findings.length === 0) return 'pass'
|
||||
return rollUp(r.findings.map(f => f.determination))
|
||||
}
|
||||
|
||||
export interface ReportProblem {
|
||||
findingId: string
|
||||
problem: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the report against the standard before it is shown to anyone.
|
||||
*
|
||||
* This is the enforcement half of "evidence attached, not asserted". A
|
||||
* capability that concludes something without naming what it looked at fails
|
||||
* here, in its own test suite, rather than in front of a reviewer.
|
||||
*/
|
||||
export function validateReport(r: CapabilityReport): ReportProblem[] {
|
||||
const problems: ReportProblem[] = []
|
||||
const seen = new Set<string>()
|
||||
|
||||
for (const f of r.findings) {
|
||||
if (seen.has(f.id)) problems.push({ findingId: f.id, problem: 'duplicate finding id' })
|
||||
seen.add(f.id)
|
||||
|
||||
if (f.determination !== 'not-assessed' && f.evidence.length === 0) {
|
||||
problems.push({
|
||||
findingId: f.id,
|
||||
problem: `determination "${f.determination}" carries no evidence — an asserted result is not a verified one`,
|
||||
})
|
||||
}
|
||||
if (!f.summary.trim()) {
|
||||
problems.push({ findingId: f.id, problem: 'empty summary' })
|
||||
}
|
||||
}
|
||||
|
||||
if (r.examined > 0 && r.findings.length === 0 && r.notes?.length === 0) {
|
||||
// Not a problem, but worth surfacing: a clean run should still say what
|
||||
// it looked at, and `scope` is the only place that lives.
|
||||
if (!r.scope.trim()) problems.push({ findingId: '(report)', problem: 'clean run with no scope described' })
|
||||
}
|
||||
|
||||
return problems
|
||||
}
|
||||
|
||||
export interface FormatOptions {
|
||||
/**
|
||||
* `inline` — one citation per finding (default; fits a CI log)
|
||||
* `full` — every citation, with excerpts and notes
|
||||
* `none` — findings only, for a summary email
|
||||
*/
|
||||
evidence?: 'inline' | 'full' | 'none'
|
||||
/** Drop findings below this severity. Defaults to showing everything. */
|
||||
minSeverity?: Severity
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a report for a terminal, a CI log, or an email.
|
||||
*
|
||||
* Worst first, so a critical never hides under a page of info lines, and the
|
||||
* unflattering cases stay unflattering: a not-assessed run leads with the
|
||||
* fact that nothing was checked instead of a green tick.
|
||||
*/
|
||||
export function formatReport(r: CapabilityReport, opts: FormatOptions = {}): string {
|
||||
const mode = opts.evidence ?? 'inline'
|
||||
const floor = opts.minSeverity ? RANK[opts.minSeverity] : RANK.info
|
||||
const verdict = overall(r)
|
||||
const out: string[] = []
|
||||
|
||||
out.push(`${r.capability} — ${r.scope}`)
|
||||
out.push(` ${label(verdict)} · ${r.examined} item${r.examined === 1 ? '' : 's'} examined · ${r.findings.length} finding${r.findings.length === 1 ? '' : 's'}`)
|
||||
|
||||
if (verdict === 'not-assessed' && r.examined === 0) {
|
||||
out.push('')
|
||||
out.push(` ${describe('not-assessed', `${r.capability}'s scope`)}`)
|
||||
for (const n of r.notes ?? []) out.push(` ${n}`)
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
for (const n of r.notes ?? []) out.push(` ${n}`)
|
||||
|
||||
const shown = r.findings
|
||||
.filter(f => RANK[f.severity] <= floor)
|
||||
.sort((a, b) => RANK[a.severity] - RANK[b.severity] || a.id.localeCompare(b.id))
|
||||
|
||||
if (shown.length > 0) out.push('')
|
||||
|
||||
for (const f of shown) {
|
||||
const tag = f.severity === 'critical' ? '✗ CRITICAL'
|
||||
: f.severity === 'high' ? '✗ HIGH '
|
||||
: f.severity === 'medium' ? '· MEDIUM '
|
||||
: '◦ info '
|
||||
out.push(`${tag} ${f.summary} [${label(f.determination)}]`)
|
||||
if (f.detail) out.push(` ${f.detail}`)
|
||||
|
||||
if (mode === 'none') continue
|
||||
|
||||
if (f.evidence.length === 0) {
|
||||
// A not-assessed finding has nothing to cite by definition — that IS
|
||||
// its content, and labelling it a missing citation would read as a
|
||||
// defect in the report rather than the gap it is reporting.
|
||||
//
|
||||
// Anything else with no evidence is loud on purpose: a conclusion with
|
||||
// nothing behind it should look worse than one with a citation.
|
||||
out.push(f.determination === 'not-assessed'
|
||||
? ' (nothing was observed)'
|
||||
: ' (no evidence attached)')
|
||||
continue
|
||||
}
|
||||
if (mode === 'inline') {
|
||||
out.push(` evidence: ${cite(f.evidence[0]!)}${f.evidence.length > 1 ? ` (+${f.evidence.length - 1} more)` : ''}`)
|
||||
} else {
|
||||
for (const e of f.evidence) {
|
||||
for (const line of citeVerbose(e)) out.push(` ${line}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shown.length === 0 && r.findings.length > 0) {
|
||||
out.push('')
|
||||
out.push(` ${r.findings.length} finding(s) below the ${opts.minSeverity} threshold, not shown.`)
|
||||
}
|
||||
if (r.findings.length === 0 && verdict === 'pass') {
|
||||
out.push('')
|
||||
// Scope-neutral on purpose: "49 compose services was checked" is the
|
||||
// grammar you get from interpolating a count into a sentence, and the
|
||||
// count is already on the line above.
|
||||
out.push('✓ Every item examined met the criterion.')
|
||||
}
|
||||
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll several capabilities into one portfolio line.
|
||||
*
|
||||
* The same asymmetry as the per-report case: a capability that was never run
|
||||
* must not average away as a pass, so it drags the roll-up to not-assessed
|
||||
* rather than being skipped.
|
||||
*/
|
||||
export function formatRollUp(reports: CapabilityReport[]): string {
|
||||
const out: string[] = []
|
||||
const verdict = rollUp(reports.map(overall))
|
||||
out.push(`Portfolio — ${reports.length} capabilit${reports.length === 1 ? 'y' : 'ies'} · ${label(verdict)}`)
|
||||
for (const r of reports) {
|
||||
const v = overall(r)
|
||||
const fails = r.findings.filter(f => f.determination === 'fail').length
|
||||
out.push(` ${label(v).padEnd(12)} ${r.capability.padEnd(18)} ${r.examined} examined, ${fails} not met`)
|
||||
}
|
||||
return out.join('\n')
|
||||
}
|
||||
Reference in New Issue
Block a user