First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,12 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
@@ -0,0 +1,9 @@
|
||||
node_modules/
|
||||
dist/
|
||||
*.log
|
||||
.DS_Store
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.npmrc
|
||||
dist-cjs/
|
||||
@@ -0,0 +1,6 @@
|
||||
stages: [test]
|
||||
|
||||
test:
|
||||
stage: test
|
||||
image: node:22-alpine
|
||||
script: [npm ci, npm run build, npm test]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Changelog
|
||||
|
||||
## 0.1.2 - 2026-09-28
|
||||
|
||||
First public release under MIT.
|
||||
|
||||
- `Determination`: `pass`, `fail`, `not-applicable` and `not-assessed`.
|
||||
`rollUp([])` returns `not-assessed`, never `pass`.
|
||||
- `Evidence`: file, record, command, measurement and document sources, with
|
||||
`cite()` and `citeVerbose()` for rendering. Measurements carry their sample
|
||||
count.
|
||||
- `describeMeasurement()`: reads a zero-sample or uncomputable measurement as
|
||||
such.
|
||||
- `CapabilityReport`: `overall()` derives the verdict, and `examined === 0`
|
||||
forces `not-assessed`.
|
||||
- `validateReport()`: flags conclusions that carry no evidence.
|
||||
- `formatReport()` and `formatRollUp()`: worst-first rendering with citations
|
||||
inline.
|
||||
@@ -0,0 +1,21 @@
|
||||
# Contributing
|
||||
|
||||
Issues and merge requests are welcome at
|
||||
https://gitlab.com/extant2000/evidence-record.
|
||||
|
||||
## Ground rules
|
||||
|
||||
- A test that cannot fail proves nothing. If you fix a bug, add a test and
|
||||
check that it fails against the unfixed code before you submit.
|
||||
- Measure, don't assume. Two modules with the same line count can still be
|
||||
different programs.
|
||||
- Keep dependencies minimal. Every new runtime dependency needs a reason.
|
||||
- Explain why in comments, not what. The what is already in the code.
|
||||
- A breaking change needs a major version bump and a note in CHANGELOG.md.
|
||||
|
||||
## Before you open a merge request
|
||||
|
||||
Run both of these and make sure they pass:
|
||||
|
||||
npm run build
|
||||
npm test
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Extant 2000 LLC
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,136 @@
|
||||
# @extant2000/evidence-record
|
||||
|
||||
A small vocabulary for checks that have to show their work: four-way
|
||||
determinations, cited evidence, and reports that derive their own verdict.
|
||||
|
||||
Zero runtime dependencies. Pure functions. Ships ESM and CommonJS.
|
||||
|
||||
## Why it exists
|
||||
|
||||
Automated checks tend to fail their readers in the same few ways. This package
|
||||
gives every check one shared way to avoid them.
|
||||
|
||||
1. **A boolean hides the case that matters most.** `passed: true` can mean "we
|
||||
looked and it held" or "there was nothing to look at". Those need to stay
|
||||
apart, so a determination has four values, and three of them are not a
|
||||
pass.
|
||||
2. Findings that do not say where they came from ask the reader to take them
|
||||
on trust. Every finding here carries `Evidence`: a file and line, a database
|
||||
row, a command, a measurement, or a document.
|
||||
3. A measurement without its sample count is not evidence. An availability of
|
||||
100% over zero samples and one over 8,640 samples are different claims, so
|
||||
the sample count travels with the number.
|
||||
4. A report with zero findings is not a pass unless something was examined.
|
||||
The verdict is computed from the report, and `examined === 0` always gives
|
||||
`not-assessed`.
|
||||
|
||||
The unflattering outputs stay unflattering. A missing value is not a pass, and
|
||||
nothing in this package turns one into a pass.
|
||||
|
||||
## Determinations
|
||||
|
||||
```ts
|
||||
type Determination = 'pass' | 'fail' | 'not-applicable' | 'not-assessed'
|
||||
```
|
||||
|
||||
`rollUp` combines several. Any fail wins. An empty list is `not-assessed`,
|
||||
because `[].every(...)` being true is the trap this type closes. One unassessed
|
||||
item keeps the roll-up at `not-assessed`, so healthy items cannot dilute it.
|
||||
|
||||
```ts
|
||||
rollUp([]) // 'not-assessed'
|
||||
rollUp(['pass', 'not-assessed']) // 'not-assessed'
|
||||
rollUp(['pass', 'pass', 'fail']) // 'fail'
|
||||
```
|
||||
|
||||
`label(d)` gives a short badge (`MET`, `NOT MET`, `N/A`, `NOT ASSESSED`).
|
||||
`describe(d, subject)` gives a sentence that still makes sense when pasted into
|
||||
an email on its own.
|
||||
|
||||
## Evidence
|
||||
|
||||
```ts
|
||||
import { evidence, cite, describeMeasurement } from '@extant2000/evidence-record'
|
||||
|
||||
evidence.file('docker-compose.yml', 42, 'ports: ["2222:22"]')
|
||||
evidence.record('requirements', 'a1b2c3', 'parent_id', 'null')
|
||||
evidence.command('docker compose config', 0)
|
||||
evidence.measurement('uptime', null, 0, { unit: '%', window: '2026-07' })
|
||||
evidence.document('NIST SP 800-53 Rev 5', 'AC-2')
|
||||
```
|
||||
|
||||
`cite()` renders each kind in the form people already read it in: `path:line`
|
||||
for source, `table#id (field)` for rows, `$ command → exit 0` for commands.
|
||||
`citeVerbose()` adds the excerpt and note. `describeMeasurement()` reads the
|
||||
null case out loud: "No samples for Availability in 2026-07; it cannot be
|
||||
computed."
|
||||
|
||||
## Reports
|
||||
|
||||
```ts
|
||||
import { evidence, overall, validateReport, formatReport } from '@extant2000/evidence-record'
|
||||
|
||||
const report = {
|
||||
capability: 'PortCheck',
|
||||
scope: '12 compose services',
|
||||
examined: 12,
|
||||
findings: [
|
||||
{ id: 'f1', summary: 'SSH published on a host port', determination: 'fail',
|
||||
severity: 'critical', evidence: [evidence.file('docker-compose.yml', 42, 'ports: ["2222:22"]')] },
|
||||
{ id: 'f2', summary: 'Admin UI bound to loopback only', determination: 'pass',
|
||||
severity: 'info', evidence: [evidence.file('docker-compose.yml', 88)] },
|
||||
],
|
||||
}
|
||||
|
||||
overall(report) // 'fail', computed from the findings
|
||||
validateReport(report) // [] when every conclusion carries a citation
|
||||
formatReport(report) // worst first, one citation per finding
|
||||
```
|
||||
|
||||
Output of `formatReport(report)`:
|
||||
|
||||
```
|
||||
PortCheck — 12 compose services
|
||||
NOT MET · 12 items examined · 2 findings
|
||||
|
||||
✗ CRITICAL SSH published on a host port [NOT MET]
|
||||
evidence: docker-compose.yml:42
|
||||
◦ info Admin UI bound to loopback only [MET]
|
||||
evidence: docker-compose.yml:88
|
||||
```
|
||||
|
||||
The same report with nothing examined:
|
||||
|
||||
```
|
||||
PortCheck — no services found
|
||||
NOT ASSESSED · 0 items examined · 0 findings
|
||||
|
||||
PortCheck's scope was not assessed. This is not a pass — an unassessed control is not an implemented control.
|
||||
```
|
||||
|
||||
`validateReport` flags a finding that reaches `pass`, `fail` or
|
||||
`not-applicable` with no evidence, plus duplicate ids and empty summaries. A
|
||||
`not-assessed` finding may have no evidence, since having nothing to cite is
|
||||
what it reports. `formatReport` accepts `{ evidence: 'inline' | 'full' | 'none',
|
||||
minSeverity }`. `formatRollUp(reports)` gives one line per report and a
|
||||
combined verdict, where an unrun report pulls the total to `not-assessed`.
|
||||
|
||||
## Conformance
|
||||
|
||||
A check built on this package conforms when its own tests assert that:
|
||||
|
||||
- `validateReport(report)` is empty for every report it can produce;
|
||||
- an empty input produces `not-assessed`, and the rendered output contains
|
||||
neither `✓` nor the word "clean";
|
||||
- every `fail` finding carries at least one `Evidence`;
|
||||
- the rendered output of a failure names a file, a row, or a command.
|
||||
|
||||
## Install
|
||||
|
||||
```
|
||||
npm install @extant2000/evidence-record
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT. Copyright (c) 2026 Extant 2000 LLC.
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# Security policy
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
Please report vulnerabilities privately by email to security@extant2000.com.
|
||||
Do not open a public issue or merge request for a security problem.
|
||||
|
||||
Include the affected version, a description of the issue, and steps or a test
|
||||
that reproduce it if you have them.
|
||||
|
||||
We aim to acknowledge every report within 5 business days.
|
||||
Generated
+1497
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "@extant2000/evidence-record",
|
||||
"version": "0.1.2",
|
||||
"private": false,
|
||||
"description": "Shared evidence and determination vocabulary for checks that must show their work.",
|
||||
"license": "MIT",
|
||||
"author": "Extant 2000 LLC",
|
||||
"type": "module",
|
||||
"main": "./dist-cjs/index.js",
|
||||
"files": [
|
||||
"dist",
|
||||
"dist-cjs",
|
||||
"src",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://gitlab.com/extant2000/evidence-record.git"
|
||||
},
|
||||
"homepage": "https://gitlab.com/extant2000/evidence-record",
|
||||
"bugs": {
|
||||
"url": "https://gitlab.com/extant2000/evidence-record/-/issues"
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js",
|
||||
"require": "./dist-cjs/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && tsc -p tsconfig.cjs.json && node -e \"require('fs').writeFileSync('dist-cjs/package.json', JSON.stringify({type:'commonjs'}))\"",
|
||||
"test": "vitest run",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.6.0",
|
||||
"vitest": "^2.1.0"
|
||||
},
|
||||
"module": "./dist/index.js"
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* The four things a check can conclude — and the three of them that are not
|
||||
* a pass.
|
||||
*
|
||||
* A boolean cannot carry this. `passed: true` collapses "we checked and it
|
||||
* held" together with "there was nothing to check", and those are the two
|
||||
* cases a reviewer most needs kept apart. An earlier version of these
|
||||
* libraries said VACUOUS for the second, which is precise and unreadable; this
|
||||
* replaces the jargon without softening the meaning.
|
||||
*/
|
||||
export type Determination =
|
||||
/** Something was examined against the criterion, and it held. */
|
||||
| 'pass'
|
||||
/** Something was examined against the criterion, and it did not hold. */
|
||||
| 'fail'
|
||||
/** The criterion was examined and genuinely does not apply here. */
|
||||
| 'not-applicable'
|
||||
/** Nothing was examined. Never a pass. */
|
||||
| 'not-assessed'
|
||||
|
||||
/**
|
||||
* One-word label for a column or badge.
|
||||
*
|
||||
* `not-assessed` deliberately does NOT get a neutral word like "skipped" —
|
||||
* "skipped" reads as a decision someone made, and this is usually an absence
|
||||
* nobody noticed.
|
||||
*/
|
||||
export function label(d: Determination): string {
|
||||
switch (d) {
|
||||
case 'pass': return 'MET'
|
||||
case 'fail': return 'NOT MET'
|
||||
case 'not-applicable': return 'N/A'
|
||||
case 'not-assessed': return 'NOT ASSESSED'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A full sentence a reviewer can read without a glossary.
|
||||
*
|
||||
* `subject` names what was being checked, so the sentence stands alone when
|
||||
* it is lifted out of the report and pasted into an email — which is how
|
||||
* these findings actually travel.
|
||||
*/
|
||||
export function describe(d: Determination, subject = 'This item'): string {
|
||||
switch (d) {
|
||||
case 'pass':
|
||||
return `${subject} was checked and met the criterion.`
|
||||
case 'fail':
|
||||
return `${subject} was checked and did not meet the criterion.`
|
||||
case 'not-applicable':
|
||||
return `${subject} was examined and the criterion does not apply to it.`
|
||||
case 'not-assessed':
|
||||
return `${subject} was not assessed. This is not a pass — an unassessed control is not an implemented control.`
|
||||
}
|
||||
}
|
||||
|
||||
/** True only for the one determination that is genuinely a clean result. */
|
||||
export function isPass(d: Determination): boolean {
|
||||
return d === 'pass'
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll several determinations into one.
|
||||
*
|
||||
* Order matters and is deliberate:
|
||||
* any fail → fail
|
||||
* nothing examined → not-assessed (NOT a pass; this is the vacuous case)
|
||||
* everything N/A → not-applicable
|
||||
* otherwise → pass
|
||||
*
|
||||
* The empty list returns `not-assessed` rather than `pass`. `[].every(...)`
|
||||
* being true is the exact trap this type exists to close.
|
||||
*/
|
||||
export function rollUp(ds: Determination[]): Determination {
|
||||
if (ds.some(d => d === 'fail')) return 'fail'
|
||||
if (ds.length === 0) return 'not-assessed'
|
||||
if (ds.some(d => d === 'not-assessed')) return 'not-assessed'
|
||||
if (ds.every(d => d === 'not-applicable')) return 'not-applicable'
|
||||
return 'pass'
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Where a finding came from.
|
||||
*
|
||||
* The rule this type exists to enforce: a finding names the artefact that
|
||||
* produced it. SideDoor already did this — it prints the service, the port
|
||||
* binding and the compose file — and it is noticeably the most useful of these
|
||||
* outputs to read. A result that is asserted without a source leaves the
|
||||
* reader to take it on trust, and an unverified claim should count as absent.
|
||||
*/
|
||||
|
||||
/** A location in source or configuration. */
|
||||
export interface FileEvidence {
|
||||
kind: 'file'
|
||||
path: string
|
||||
/** 1-indexed. Omitted when the finding is about the file as a whole. */
|
||||
line?: number
|
||||
/** The matched text. Keep it short — one line, trimmed. */
|
||||
excerpt?: string
|
||||
}
|
||||
|
||||
/** A row in a database. */
|
||||
export interface RecordEvidence {
|
||||
kind: 'record'
|
||||
table: string
|
||||
id: string
|
||||
/** The column that carried the value the finding turns on. */
|
||||
field?: string
|
||||
value?: string
|
||||
}
|
||||
|
||||
/** Output of a command that was actually run. */
|
||||
export interface CommandEvidence {
|
||||
kind: 'command'
|
||||
command: string
|
||||
exitCode?: number
|
||||
excerpt?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A number that was measured, together with how much data stood behind it.
|
||||
*
|
||||
* `samples` is not decoration. An availability of 100% over zero samples and
|
||||
* an availability of 100% over 8,640 samples are different claims, and only
|
||||
* one of them is evidence.
|
||||
*/
|
||||
export interface MeasurementEvidence {
|
||||
kind: 'measurement'
|
||||
metric: string
|
||||
/** Null when it could not be computed. Never substitute a flattering default. */
|
||||
value: number | null
|
||||
unit?: string
|
||||
samples: number
|
||||
/** e.g. "2026-07-01 → 2026-07-31". */
|
||||
window?: string
|
||||
}
|
||||
|
||||
/** A written source: a policy, a standard, a control catalogue. */
|
||||
export interface DocumentEvidence {
|
||||
kind: 'document'
|
||||
title: string
|
||||
section?: string
|
||||
url?: string
|
||||
}
|
||||
|
||||
export type EvidenceSource =
|
||||
| FileEvidence
|
||||
| RecordEvidence
|
||||
| CommandEvidence
|
||||
| MeasurementEvidence
|
||||
| DocumentEvidence
|
||||
|
||||
export interface Evidence {
|
||||
source: EvidenceSource
|
||||
/** ISO 8601. When the observation was made, if the caller tracks it. */
|
||||
observedAt?: string
|
||||
/** One line on why this artefact supports the finding. */
|
||||
note?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A short, copy-pasteable citation — the thing a reader scans for.
|
||||
*
|
||||
* Formats follow the conventions each kind is already read in: `path:line`
|
||||
* for source (clickable in most terminals), `table#id` for rows.
|
||||
*/
|
||||
export function cite(e: Evidence): string {
|
||||
const s = e.source
|
||||
switch (s.kind) {
|
||||
case 'file':
|
||||
return s.line ? `${s.path}:${s.line}` : s.path
|
||||
case 'record':
|
||||
return s.field ? `${s.table}#${s.id} (${s.field})` : `${s.table}#${s.id}`
|
||||
case 'command':
|
||||
return s.exitCode === undefined ? `$ ${s.command}` : `$ ${s.command} → exit ${s.exitCode}`
|
||||
case 'measurement': {
|
||||
const v = s.value === null ? 'not computable' : `${s.value}${s.unit ?? ''}`
|
||||
const n = `${s.samples} sample${s.samples === 1 ? '' : 's'}`
|
||||
return s.window ? `${s.metric} = ${v} (${n}, ${s.window})` : `${s.metric} = ${v} (${n})`
|
||||
}
|
||||
case 'document':
|
||||
return s.section ? `${s.title} § ${s.section}` : s.title
|
||||
}
|
||||
}
|
||||
|
||||
/** Longer form: the citation plus whatever context was attached to it. */
|
||||
export function citeVerbose(e: Evidence): string[] {
|
||||
const out = [cite(e)]
|
||||
const s = e.source
|
||||
if ((s.kind === 'file' || s.kind === 'command') && s.excerpt) out.push(` ${s.excerpt.trim()}`)
|
||||
if (s.kind === 'record' && s.value !== undefined) out.push(` value: ${s.value}`)
|
||||
if (e.note) out.push(` ${e.note}`)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Convenience constructors, so call sites stay one line. */
|
||||
export const evidence = {
|
||||
file(path: string, line?: number, excerpt?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'file', path, line, excerpt }, note }
|
||||
},
|
||||
record(table: string, id: string, field?: string, value?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'record', table, id, field, value }, note }
|
||||
},
|
||||
command(command: string, exitCode?: number, excerpt?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'command', command, exitCode, excerpt }, note }
|
||||
},
|
||||
measurement(
|
||||
metric: string,
|
||||
value: number | null,
|
||||
samples: number,
|
||||
opts: { unit?: string, window?: string, note?: string } = {},
|
||||
): Evidence {
|
||||
return {
|
||||
source: { kind: 'measurement', metric, value, samples, unit: opts.unit, window: opts.window },
|
||||
note: opts.note,
|
||||
}
|
||||
},
|
||||
document(title: string, section?: string, url?: string, note?: string): Evidence {
|
||||
return { source: { kind: 'document', title, section, url }, note }
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a measurement out loud, including the case where there is nothing to
|
||||
* read.
|
||||
*
|
||||
* `uptime null, met=false` was correct and unreadable. A reader should not
|
||||
* have to know that zero samples is not 100% in order to understand the line.
|
||||
*/
|
||||
export function describeMeasurement(s: MeasurementEvidence): string {
|
||||
if (s.samples === 0) {
|
||||
return `No samples for ${s.metric}${s.window ? ` in ${s.window}` : ''}; it cannot be computed.`
|
||||
}
|
||||
if (s.value === null) {
|
||||
return `${s.metric} could not be computed from the ${s.samples} available sample${s.samples === 1 ? '' : 's'}.`
|
||||
}
|
||||
return `${s.metric} is ${s.value}${s.unit ?? ''} over ${s.samples} sample${s.samples === 1 ? '' : 's'}${s.window ? ` (${s.window})` : ''}.`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export type { Determination } from './determination.js'
|
||||
export { label, describe, isPass, rollUp } from './determination.js'
|
||||
|
||||
export type {
|
||||
Evidence,
|
||||
EvidenceSource,
|
||||
FileEvidence,
|
||||
RecordEvidence,
|
||||
CommandEvidence,
|
||||
MeasurementEvidence,
|
||||
DocumentEvidence,
|
||||
} from './evidence.js'
|
||||
export { cite, citeVerbose, evidence, describeMeasurement } from './evidence.js'
|
||||
|
||||
export type {
|
||||
Severity,
|
||||
Finding,
|
||||
CapabilityReport,
|
||||
ReportProblem,
|
||||
FormatOptions,
|
||||
} from './report.js'
|
||||
export { overall, validateReport, formatReport, formatRollUp } from './report.js'
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
import { type Determination, label, describe, rollUp } from './determination.js'
|
||||
import { type Evidence, cite, citeVerbose } from './evidence.js'
|
||||
|
||||
export type Severity = 'critical' | 'high' | 'medium' | 'info'
|
||||
|
||||
const RANK: Record<Severity, number> = { critical: 0, high: 1, medium: 2, info: 3 }
|
||||
|
||||
export interface Finding {
|
||||
/**
|
||||
* Stable within one report only. These identify a row in THIS run, not a
|
||||
* durable record — do not persist them as entity keys.
|
||||
*/
|
||||
id: string
|
||||
/** One line, in the reader's language, not the checker's. */
|
||||
summary: string
|
||||
determination: Determination
|
||||
severity: Severity
|
||||
/**
|
||||
* What produced this finding. Required, and allowed to be empty only for
|
||||
* `not-assessed` — where the whole point is that nothing was observed.
|
||||
*/
|
||||
evidence: Evidence[]
|
||||
/** Optional paragraph: why it matters, what to do. */
|
||||
detail?: string
|
||||
}
|
||||
|
||||
export interface CapabilityReport {
|
||||
/** The capability that produced this, e.g. "SideDoor". */
|
||||
capability: string
|
||||
/** What was examined, in a phrase: "49 compose services on the build host". */
|
||||
scope: string
|
||||
/** How many items were actually examined. Zero forces `not-assessed`. */
|
||||
examined: number
|
||||
findings: Finding[]
|
||||
/** Free-form lines printed under the summary — caveats, exclusions. */
|
||||
notes?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The report's overall determination, derived rather than asserted.
|
||||
*
|
||||
* Deriving it closes the gap where a capability reports zero findings and a
|
||||
* reader infers a pass. Zero findings over zero examined items is
|
||||
* `not-assessed`, and it says so.
|
||||
*/
|
||||
export function overall(r: CapabilityReport): Determination {
|
||||
if (r.examined === 0) return 'not-assessed'
|
||||
if (r.findings.length === 0) return 'pass'
|
||||
return rollUp(r.findings.map(f => f.determination))
|
||||
}
|
||||
|
||||
export interface ReportProblem {
|
||||
findingId: string
|
||||
problem: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the report against the standard before it is shown to anyone.
|
||||
*
|
||||
* This is the enforcement half of "evidence attached, not asserted". A
|
||||
* capability that concludes something without naming what it looked at fails
|
||||
* here, in its own test suite, rather than in front of a reviewer.
|
||||
*/
|
||||
export function validateReport(r: CapabilityReport): ReportProblem[] {
|
||||
const problems: ReportProblem[] = []
|
||||
const seen = new Set<string>()
|
||||
|
||||
for (const f of r.findings) {
|
||||
if (seen.has(f.id)) problems.push({ findingId: f.id, problem: 'duplicate finding id' })
|
||||
seen.add(f.id)
|
||||
|
||||
if (f.determination !== 'not-assessed' && f.evidence.length === 0) {
|
||||
problems.push({
|
||||
findingId: f.id,
|
||||
problem: `determination "${f.determination}" carries no evidence — an asserted result is not a verified one`,
|
||||
})
|
||||
}
|
||||
if (!f.summary.trim()) {
|
||||
problems.push({ findingId: f.id, problem: 'empty summary' })
|
||||
}
|
||||
}
|
||||
|
||||
if (r.examined > 0 && r.findings.length === 0 && r.notes?.length === 0) {
|
||||
// Not a problem, but worth surfacing: a clean run should still say what
|
||||
// it looked at, and `scope` is the only place that lives.
|
||||
if (!r.scope.trim()) problems.push({ findingId: '(report)', problem: 'clean run with no scope described' })
|
||||
}
|
||||
|
||||
return problems
|
||||
}
|
||||
|
||||
export interface FormatOptions {
|
||||
/**
|
||||
* `inline` — one citation per finding (default; fits a CI log)
|
||||
* `full` — every citation, with excerpts and notes
|
||||
* `none` — findings only, for a summary email
|
||||
*/
|
||||
evidence?: 'inline' | 'full' | 'none'
|
||||
/** Drop findings below this severity. Defaults to showing everything. */
|
||||
minSeverity?: Severity
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a report for a terminal, a CI log, or an email.
|
||||
*
|
||||
* Worst first, so a critical never hides under a page of info lines, and the
|
||||
* unflattering cases stay unflattering: a not-assessed run leads with the
|
||||
* fact that nothing was checked instead of a green tick.
|
||||
*/
|
||||
export function formatReport(r: CapabilityReport, opts: FormatOptions = {}): string {
|
||||
const mode = opts.evidence ?? 'inline'
|
||||
const floor = opts.minSeverity ? RANK[opts.minSeverity] : RANK.info
|
||||
const verdict = overall(r)
|
||||
const out: string[] = []
|
||||
|
||||
out.push(`${r.capability} — ${r.scope}`)
|
||||
out.push(` ${label(verdict)} · ${r.examined} item${r.examined === 1 ? '' : 's'} examined · ${r.findings.length} finding${r.findings.length === 1 ? '' : 's'}`)
|
||||
|
||||
if (verdict === 'not-assessed' && r.examined === 0) {
|
||||
out.push('')
|
||||
out.push(` ${describe('not-assessed', `${r.capability}'s scope`)}`)
|
||||
for (const n of r.notes ?? []) out.push(` ${n}`)
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
for (const n of r.notes ?? []) out.push(` ${n}`)
|
||||
|
||||
const shown = r.findings
|
||||
.filter(f => RANK[f.severity] <= floor)
|
||||
.sort((a, b) => RANK[a.severity] - RANK[b.severity] || a.id.localeCompare(b.id))
|
||||
|
||||
if (shown.length > 0) out.push('')
|
||||
|
||||
for (const f of shown) {
|
||||
const tag = f.severity === 'critical' ? '✗ CRITICAL'
|
||||
: f.severity === 'high' ? '✗ HIGH '
|
||||
: f.severity === 'medium' ? '· MEDIUM '
|
||||
: '◦ info '
|
||||
out.push(`${tag} ${f.summary} [${label(f.determination)}]`)
|
||||
if (f.detail) out.push(` ${f.detail}`)
|
||||
|
||||
if (mode === 'none') continue
|
||||
|
||||
if (f.evidence.length === 0) {
|
||||
// A not-assessed finding has nothing to cite by definition — that IS
|
||||
// its content, and labelling it a missing citation would read as a
|
||||
// defect in the report rather than the gap it is reporting.
|
||||
//
|
||||
// Anything else with no evidence is loud on purpose: a conclusion with
|
||||
// nothing behind it should look worse than one with a citation.
|
||||
out.push(f.determination === 'not-assessed'
|
||||
? ' (nothing was observed)'
|
||||
: ' (no evidence attached)')
|
||||
continue
|
||||
}
|
||||
if (mode === 'inline') {
|
||||
out.push(` evidence: ${cite(f.evidence[0]!)}${f.evidence.length > 1 ? ` (+${f.evidence.length - 1} more)` : ''}`)
|
||||
} else {
|
||||
for (const e of f.evidence) {
|
||||
for (const line of citeVerbose(e)) out.push(` ${line}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shown.length === 0 && r.findings.length > 0) {
|
||||
out.push('')
|
||||
out.push(` ${r.findings.length} finding(s) below the ${opts.minSeverity} threshold, not shown.`)
|
||||
}
|
||||
if (r.findings.length === 0 && verdict === 'pass') {
|
||||
out.push('')
|
||||
// Scope-neutral on purpose: "49 compose services was checked" is the
|
||||
// grammar you get from interpolating a count into a sentence, and the
|
||||
// count is already on the line above.
|
||||
out.push('✓ Every item examined met the criterion.')
|
||||
}
|
||||
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll several capabilities into one portfolio line.
|
||||
*
|
||||
* The same asymmetry as the per-report case: a capability that was never run
|
||||
* must not average away as a pass, so it drags the roll-up to not-assessed
|
||||
* rather than being skipped.
|
||||
*/
|
||||
export function formatRollUp(reports: CapabilityReport[]): string {
|
||||
const out: string[] = []
|
||||
const verdict = rollUp(reports.map(overall))
|
||||
out.push(`Portfolio — ${reports.length} capabilit${reports.length === 1 ? 'y' : 'ies'} · ${label(verdict)}`)
|
||||
for (const r of reports) {
|
||||
const v = overall(r)
|
||||
const fails = r.findings.filter(f => f.determination === 'fail').length
|
||||
out.push(` ${label(v).padEnd(12)} ${r.capability.padEnd(18)} ${r.examined} examined, ${fails} not met`)
|
||||
}
|
||||
return out.join('\n')
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
import { describe as suite, it, expect } from 'vitest'
|
||||
import {
|
||||
type CapabilityReport,
|
||||
cite,
|
||||
citeVerbose,
|
||||
describe as describeDetermination,
|
||||
describeMeasurement,
|
||||
evidence,
|
||||
formatReport,
|
||||
formatRollUp,
|
||||
label,
|
||||
overall,
|
||||
rollUp,
|
||||
validateReport,
|
||||
} from '../src/index.js'
|
||||
|
||||
suite('determination', () => {
|
||||
it('never renders a non-pass as a pass', () => {
|
||||
expect(label('not-assessed')).toBe('NOT ASSESSED')
|
||||
expect(label('not-applicable')).toBe('N/A')
|
||||
expect(describeDetermination('not-assessed')).toContain('not a pass')
|
||||
})
|
||||
|
||||
it('rolls an empty set up to not-assessed, not pass', () => {
|
||||
// [].every(...) === true is the exact trap this closes.
|
||||
expect(rollUp([])).toBe('not-assessed')
|
||||
})
|
||||
|
||||
it('lets one failure dominate', () => {
|
||||
expect(rollUp(['pass', 'pass', 'fail'])).toBe('fail')
|
||||
})
|
||||
|
||||
it('does not let an unassessed item average away', () => {
|
||||
expect(rollUp(['pass', 'not-assessed'])).toBe('not-assessed')
|
||||
})
|
||||
|
||||
it('reports all-N/A as N/A rather than pass', () => {
|
||||
expect(rollUp(['not-applicable', 'not-applicable'])).toBe('not-applicable')
|
||||
})
|
||||
})
|
||||
|
||||
suite('citations', () => {
|
||||
it('cites source as path:line so a terminal can open it', () => {
|
||||
expect(cite(evidence.file('docker-compose.yml', 42, 'ports: ["2222:22"]'))).toBe('docker-compose.yml:42')
|
||||
})
|
||||
|
||||
it('cites a row as table#id with the deciding column', () => {
|
||||
expect(cite(evidence.record('requirements', 'a1b2c3', 'parent_id', 'null')))
|
||||
.toBe('requirements#a1b2c3 (parent_id)')
|
||||
})
|
||||
|
||||
it('carries sample count into the citation, because 100% of nothing is not 100%', () => {
|
||||
const none = evidence.measurement('uptime', null, 0, { unit: '%', window: '2026-07' })
|
||||
const real = evidence.measurement('uptime', 98.3, 8640, { unit: '%', window: '2026-07' })
|
||||
expect(cite(none)).toContain('not computable')
|
||||
expect(cite(none)).toContain('0 samples')
|
||||
expect(cite(real)).toContain('98.3%')
|
||||
expect(cite(real)).toContain('8640 samples')
|
||||
})
|
||||
|
||||
it('reads a zero-sample measurement as uncomputable, not as a value', () => {
|
||||
const s = evidence.measurement('Availability', null, 0, { window: '2026-07' }).source
|
||||
expect(describeMeasurement(s as never)).toBe(
|
||||
'No samples for Availability in 2026-07; it cannot be computed.',
|
||||
)
|
||||
})
|
||||
|
||||
it('includes excerpt and note in the verbose form', () => {
|
||||
const lines = citeVerbose(evidence.file('server/api/x.ts', 7, 'const q = sql(raw)', 'unparameterised'))
|
||||
expect(lines[0]).toBe('server/api/x.ts:7')
|
||||
expect(lines.join('\n')).toContain('const q = sql(raw)')
|
||||
expect(lines.join('\n')).toContain('unparameterised')
|
||||
})
|
||||
})
|
||||
|
||||
const clean: CapabilityReport = {
|
||||
capability: 'SideDoor',
|
||||
scope: '49 compose services',
|
||||
examined: 49,
|
||||
findings: [],
|
||||
}
|
||||
|
||||
suite('report', () => {
|
||||
it('derives not-assessed from an empty scan instead of reporting clean', () => {
|
||||
const empty: CapabilityReport = { capability: 'SideDoor', scope: 'nothing', examined: 0, findings: [] }
|
||||
expect(overall(empty)).toBe('not-assessed')
|
||||
const out = formatReport(empty)
|
||||
expect(out).toContain('NOT ASSESSED')
|
||||
expect(out).not.toContain('✓')
|
||||
})
|
||||
|
||||
it('reports a genuine clean run as a pass', () => {
|
||||
expect(overall(clean)).toBe('pass')
|
||||
expect(formatReport(clean)).toContain('✓')
|
||||
})
|
||||
|
||||
it('rejects a conclusion with no evidence behind it', () => {
|
||||
const asserted: CapabilityReport = {
|
||||
capability: 'CertPack',
|
||||
scope: '3 controls',
|
||||
examined: 3,
|
||||
findings: [{ id: 'f1', summary: 'AC-2 implemented', determination: 'pass', severity: 'info', evidence: [] }],
|
||||
}
|
||||
const problems = validateReport(asserted)
|
||||
expect(problems).toHaveLength(1)
|
||||
expect(problems[0]!.problem).toContain('no evidence')
|
||||
})
|
||||
|
||||
it('allows a not-assessed finding to carry no evidence', () => {
|
||||
const unassessed: CapabilityReport = {
|
||||
capability: 'CertPack',
|
||||
scope: '3 controls',
|
||||
examined: 3,
|
||||
findings: [{ id: 'f1', summary: 'AC-2', determination: 'not-assessed', severity: 'high', evidence: [] }],
|
||||
}
|
||||
expect(validateReport(unassessed)).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('catches duplicate finding ids', () => {
|
||||
const dup: CapabilityReport = {
|
||||
capability: 'X', scope: 's', examined: 2,
|
||||
findings: [
|
||||
{ id: 'f1', summary: 'a', determination: 'pass', severity: 'info', evidence: [evidence.file('a.ts')] },
|
||||
{ id: 'f1', summary: 'b', determination: 'pass', severity: 'info', evidence: [evidence.file('b.ts')] },
|
||||
],
|
||||
}
|
||||
expect(validateReport(dup).some(p => p.problem === 'duplicate finding id')).toBe(true)
|
||||
})
|
||||
|
||||
it('shows a missing citation loudly rather than silently', () => {
|
||||
const r: CapabilityReport = {
|
||||
capability: 'X', scope: 's', examined: 1,
|
||||
findings: [{ id: 'f1', summary: 'something is wrong', determination: 'fail', severity: 'high', evidence: [] }],
|
||||
}
|
||||
expect(formatReport(r)).toContain('(no evidence attached)')
|
||||
})
|
||||
|
||||
it('does not label an unassessed finding as a missing citation', () => {
|
||||
// Having nothing to cite IS the content of a not-assessed finding.
|
||||
// Calling it a missing citation reads as a defect in the report rather
|
||||
// than the gap the report is describing.
|
||||
const r: CapabilityReport = {
|
||||
capability: 'X', scope: 's', examined: 1,
|
||||
findings: [{ id: 'f1', summary: 'rule covered nothing', determination: 'not-assessed', severity: 'medium', evidence: [] }],
|
||||
}
|
||||
const out = formatReport(r)
|
||||
expect(out).toContain('(nothing was observed)')
|
||||
expect(out).not.toContain('(no evidence attached)')
|
||||
})
|
||||
|
||||
it('sorts worst first so a critical cannot hide under info lines', () => {
|
||||
const r: CapabilityReport = {
|
||||
capability: 'SideDoor', scope: '2 services', examined: 2,
|
||||
findings: [
|
||||
{ id: 'f2', summary: 'loopback bind', determination: 'pass', severity: 'info', evidence: [evidence.file('a.yml', 3)] },
|
||||
{ id: 'f1', summary: 'SSH published on a host port', determination: 'fail', severity: 'critical', evidence: [evidence.file('b.yml', 9, '2222:22')] },
|
||||
],
|
||||
}
|
||||
const out = formatReport(r)
|
||||
expect(out.indexOf('SSH published')).toBeLessThan(out.indexOf('loopback bind'))
|
||||
expect(overall(r)).toBe('fail')
|
||||
})
|
||||
|
||||
it('prints every citation in full mode', () => {
|
||||
const r: CapabilityReport = {
|
||||
capability: 'X', scope: 's', examined: 1,
|
||||
findings: [{
|
||||
id: 'f1', summary: 'drift', determination: 'fail', severity: 'high',
|
||||
evidence: [evidence.file('a.ts', 1), evidence.file('b.ts', 2)],
|
||||
}],
|
||||
}
|
||||
expect(formatReport(r, { evidence: 'inline' })).toContain('(+1 more)')
|
||||
const full = formatReport(r, { evidence: 'full' })
|
||||
expect(full).toContain('a.ts:1')
|
||||
expect(full).toContain('b.ts:2')
|
||||
})
|
||||
})
|
||||
|
||||
suite('roll-up', () => {
|
||||
it('does not let healthy capabilities dilute an unrun one', () => {
|
||||
const unrun: CapabilityReport = { capability: 'UnrunCheck', scope: 'none', examined: 0, findings: [] }
|
||||
const out = formatRollUp([clean, unrun])
|
||||
expect(out).toContain('NOT ASSESSED')
|
||||
expect(rollUp([overall(clean), overall(unrun)])).toBe('not-assessed')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "Node",
|
||||
"outDir": "dist-cjs",
|
||||
"declaration": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022", "module": "ES2022", "moduleResolution": "bundler",
|
||||
"declaration": true, "outDir": "dist", "rootDir": "src",
|
||||
"strict": true, "skipLibCheck": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user