First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+294
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* AdversarialSuite — generate the edge, misuse and abuse cases a happy-path
|
||||
* test suite does not contain, and report which of them nobody has covered.
|
||||
*
|
||||
* This library does not run tests. It derives the cases an interface implies
|
||||
* and checks them against the cases that exist, because the failure it
|
||||
* targets is not a failing test — it is a passing suite with a hole in it.
|
||||
* Every generator below comes from a class of defect seen in real systems:
|
||||
*
|
||||
* - **Empty collection.** `[].every(...)` is `true`, so a requirement with
|
||||
* zero verifications reported as verified, and a gate with zero matching
|
||||
* files reported as passing. The empty case is the single most productive
|
||||
* input in this whole set.
|
||||
* - **Absent optional field.** A provider silently dropped a requested
|
||||
* expansion; the response stayed valid and the field was simply missing.
|
||||
* Nothing asserted its presence.
|
||||
* - **Boundary + 1.** A URL filter died past roughly 400 identifiers and a
|
||||
* result set capped silently at 1000 rows. Both are fine at the sizes
|
||||
* anyone tests with.
|
||||
* - **Null-vs-zero.** Zero samples is not 100%. A null composite became a
|
||||
* truthy all-null object.
|
||||
* - **Repeated identical call.** An idempotency key derived from a literal
|
||||
* replayed a deleted object for 24 hours.
|
||||
* - **Wrong-tenant identifier.** A well-formed id belonging to somebody
|
||||
* else is the authorisation case that syntactic validation waves through.
|
||||
*
|
||||
* Pure: no test runner, no network.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CapabilityReport,
|
||||
type Finding,
|
||||
type FormatOptions,
|
||||
type Severity,
|
||||
evidence,
|
||||
formatReport as renderReport,
|
||||
} from '@extant2000/evidence-record'
|
||||
|
||||
export type ParamKind =
|
||||
| 'collection'
|
||||
| 'string'
|
||||
| 'number'
|
||||
| 'boolean'
|
||||
| 'identifier'
|
||||
| 'optional'
|
||||
|
||||
export interface Param {
|
||||
name: string
|
||||
kind: ParamKind
|
||||
required?: boolean
|
||||
/** Documented maximum — length, count, or value. Boundary cases use it. */
|
||||
max?: number
|
||||
/** True when the value is scoped to a tenant, user or organisation. */
|
||||
tenantScoped?: boolean
|
||||
}
|
||||
|
||||
export interface Operation {
|
||||
name: string
|
||||
params: Param[]
|
||||
/** True when repeating the call must not repeat its effect. */
|
||||
idempotent?: boolean
|
||||
/** True when the operation writes. Abuse cases weigh heavier. */
|
||||
mutates?: boolean
|
||||
/** Where the operation is defined — cited on every generated case. */
|
||||
where: string
|
||||
}
|
||||
|
||||
export type CaseCategory = 'edge' | 'misuse' | 'abuse'
|
||||
|
||||
export interface AdversarialCase {
|
||||
id: string
|
||||
operation: string
|
||||
category: CaseCategory
|
||||
/** What to do. */
|
||||
input: string
|
||||
/** What must happen. Written as a requirement, not a hope. */
|
||||
expectation: string
|
||||
/** Why this case exists — the defect class it comes from. */
|
||||
rationale: string
|
||||
severity: Severity
|
||||
}
|
||||
|
||||
const CAT_SEVERITY: Record<CaseCategory, Severity> = {
|
||||
edge: 'medium',
|
||||
misuse: 'high',
|
||||
abuse: 'critical',
|
||||
}
|
||||
|
||||
function mk(
|
||||
op: Operation,
|
||||
suffix: string,
|
||||
category: CaseCategory,
|
||||
input: string,
|
||||
expectation: string,
|
||||
rationale: string,
|
||||
severity?: Severity,
|
||||
): AdversarialCase {
|
||||
return {
|
||||
id: `${op.name}/${suffix}`,
|
||||
operation: op.name,
|
||||
category,
|
||||
input,
|
||||
expectation,
|
||||
rationale,
|
||||
severity: severity ?? CAT_SEVERITY[category],
|
||||
}
|
||||
}
|
||||
|
||||
function casesForParam(op: Operation, p: Param): AdversarialCase[] {
|
||||
const out: AdversarialCase[] = []
|
||||
|
||||
if (p.kind === 'collection') {
|
||||
// The single most productive input in this set.
|
||||
out.push(mk(op, `${p.name}/empty`, 'edge',
|
||||
`${p.name} = []`,
|
||||
'Reports "nothing was checked", never a pass.',
|
||||
'[].every(...) is true, so an empty collection satisfies every universal assertion made about it. A requirement with zero verifications reported as verified this way.',
|
||||
'high'))
|
||||
|
||||
out.push(mk(op, `${p.name}/single`, 'edge',
|
||||
`${p.name} = [one item]`,
|
||||
'Reports that a comparison needs at least two members, rather than a clean result.',
|
||||
'One member cannot disagree with itself, so a comparison over a single element trivially succeeds.'))
|
||||
|
||||
if (p.max) {
|
||||
out.push(mk(op, `${p.name}/over-max`, 'misuse',
|
||||
`${p.name} with ${p.max + 1} items`,
|
||||
'Rejects, paginates or chunks — never silently truncates.',
|
||||
`Documented maximum is ${p.max}. A URL filter can fail past a few hundred identifiers, and a result set can be capped silently at 1000 rows; both behave correctly at the sizes anyone tests with.`))
|
||||
}
|
||||
}
|
||||
|
||||
if (p.kind === 'number') {
|
||||
out.push(mk(op, `${p.name}/zero`, 'edge',
|
||||
`${p.name} = 0`,
|
||||
'Distinguishes zero from absent. Zero samples is not 100%.',
|
||||
'A zero denominator produced a flattering default rather than an abstention.',
|
||||
'high'))
|
||||
out.push(mk(op, `${p.name}/negative`, 'misuse',
|
||||
`${p.name} = -1`, 'Rejects rather than coercing.',
|
||||
'Negative values reach arithmetic that assumed non-negative input.'))
|
||||
if (p.max !== undefined) {
|
||||
out.push(mk(op, `${p.name}/over-max`, 'misuse',
|
||||
`${p.name} = ${p.max + 1}`, `Rejects above the documented maximum of ${p.max}.`,
|
||||
'Off-by-one at a documented boundary.'))
|
||||
}
|
||||
}
|
||||
|
||||
if (p.kind === 'optional' || !p.required) {
|
||||
out.push(mk(op, `${p.name}/absent`, 'edge',
|
||||
`${p.name} omitted entirely`,
|
||||
'Behaves correctly, and does not report success for work it skipped.',
|
||||
'A provider silently dropped a requested expansion. The response stayed structurally valid and the field was simply absent, so schema validation passed while the feature was dead.',
|
||||
'high'))
|
||||
out.push(mk(op, `${p.name}/null`, 'edge',
|
||||
`${p.name} = null`,
|
||||
'Treats null as absent, not as a present empty value.',
|
||||
'A null composite deserialised into a truthy object with all-null fields, which passed every presence check made on it.'))
|
||||
}
|
||||
|
||||
if (p.kind === 'string') {
|
||||
out.push(mk(op, `${p.name}/empty-string`, 'edge',
|
||||
`${p.name} = ""`, 'Distinguishes empty from absent.',
|
||||
'An empty string is falsy and present at the same time.'))
|
||||
out.push(mk(op, `${p.name}/unicode`, 'misuse',
|
||||
`${p.name} with combining marks, RTL overrides and a 4-byte emoji`,
|
||||
'Stores and returns the value unchanged; length limits count characters consistently.',
|
||||
'Byte-length limits truncate mid-codepoint.'))
|
||||
if (p.max) {
|
||||
out.push(mk(op, `${p.name}/over-max`, 'misuse',
|
||||
`${p.name} of length ${p.max + 1}`, `Rejects above ${p.max} rather than truncating silently.`,
|
||||
'Silent truncation is data loss that looks like success.'))
|
||||
}
|
||||
}
|
||||
|
||||
if (p.kind === 'identifier') {
|
||||
out.push(mk(op, `${p.name}/malformed`, 'misuse',
|
||||
`${p.name} = "not-an-id"`, 'Rejects with a clear error.',
|
||||
'Syntactic validation is the easy half.'))
|
||||
out.push(mk(op, `${p.name}/nonexistent`, 'misuse',
|
||||
`${p.name} = a well-formed id that does not exist`,
|
||||
'Returns not-found, and does not leak whether the id exists elsewhere.',
|
||||
'Distinguishing "absent" from "not yours" through error text is an enumeration oracle.'))
|
||||
|
||||
if (p.tenantScoped) {
|
||||
out.push(mk(op, `${p.name}/wrong-tenant`, 'abuse',
|
||||
`${p.name} = a well-formed id belonging to another tenant`,
|
||||
'Denies. The response is indistinguishable from the not-found case.',
|
||||
'The authorisation case syntactic validation waves through: the id is valid, it exists, and it is not yours. This is the check a shared-table suite exists to enforce.'))
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
function operationCases(op: Operation): AdversarialCase[] {
|
||||
const out = op.params.flatMap(p => casesForParam(op, p))
|
||||
|
||||
if (op.idempotent) {
|
||||
out.push(mk(op, 'replay/identical', 'edge',
|
||||
'The same call twice with the same idempotency key',
|
||||
'The second call returns the first result. It does not repeat the effect, and it does not return a deleted object.',
|
||||
'A key derived from a literal rather than from the parameters replayed a stale — and in one case deleted — object for 24 hours.',
|
||||
'high'))
|
||||
out.push(mk(op, 'replay/changed-params', 'misuse',
|
||||
'The same idempotency key with different parameters',
|
||||
'Rejects. It does not silently serve the first response for a different request.',
|
||||
'A literal key with changed parameters failed in-flight requests for a full day.'))
|
||||
}
|
||||
|
||||
if (op.mutates) {
|
||||
out.push(mk(op, 'concurrent/double-submit', 'abuse',
|
||||
'Two identical writes issued simultaneously',
|
||||
'Exactly one takes effect, or both are rejected. Never two.',
|
||||
'A double submit is the cheapest abuse there is and the least often tested.'))
|
||||
out.push(mk(op, 'partial-failure', 'edge',
|
||||
'A failure injected midway through a multi-step write',
|
||||
'Leaves no partially-applied state, and reports the failure rather than a success.',
|
||||
'A partially applied write reports success and is discovered later by its inconsistency.',
|
||||
'high'))
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
/** Derive the adversarial cases an operation implies. */
|
||||
export function generateCases(operations: Operation[]): AdversarialCase[] {
|
||||
return (operations ?? []).flatMap(operationCases)
|
||||
}
|
||||
|
||||
export interface Coverage {
|
||||
/** Case ids that already have a test. */
|
||||
covered: Iterable<string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Report which implied cases are covered.
|
||||
*
|
||||
* An uncovered case is `not-assessed`, not `fail`: nothing has been shown to
|
||||
* be broken. What has been shown is that nobody looked — which is the state
|
||||
* a passing suite is least able to distinguish from working.
|
||||
*/
|
||||
export function assessCoverage(
|
||||
operations: Operation[],
|
||||
coverage: Coverage = { covered: [] },
|
||||
): CapabilityReport {
|
||||
const cases = generateCases(operations)
|
||||
const covered = new Set(coverage.covered)
|
||||
|
||||
if (cases.length === 0) {
|
||||
return {
|
||||
capability: 'AdversarialSuite',
|
||||
scope: 'no operations described',
|
||||
examined: 0,
|
||||
findings: [],
|
||||
notes: ['No operations were described, so no cases were derived.'],
|
||||
}
|
||||
}
|
||||
|
||||
const findings: Finding[] = cases.map(c => {
|
||||
const isCovered = covered.has(c.id)
|
||||
const op = operations.find(o => o.name === c.operation)!
|
||||
return {
|
||||
id: c.id,
|
||||
summary: `${c.category}: ${c.input}`,
|
||||
determination: isCovered ? 'pass' : 'not-assessed',
|
||||
severity: isCovered ? 'info' : c.severity,
|
||||
detail: isCovered ? undefined : `Expected behaviour: ${c.expectation} — ${c.rationale}`,
|
||||
evidence: [evidence.file(op.where, undefined, c.operation,
|
||||
isCovered ? 'case covered' : 'case implied by the interface, no test found')],
|
||||
}
|
||||
})
|
||||
|
||||
const gap = cases.length - cases.filter(c => covered.has(c.id)).length
|
||||
return {
|
||||
capability: 'AdversarialSuite',
|
||||
scope: `${cases.length} case(s) implied by ${operations.length} operation(s)`,
|
||||
examined: cases.length,
|
||||
findings,
|
||||
notes: [
|
||||
`${cases.length - gap} of ${cases.length} implied cases covered; ${gap} with no test found`,
|
||||
'An uncovered case is not a failure. It is the absence of a check, which a passing suite cannot distinguish from working.',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/** Render the derived suite for a terminal, CI log or test plan. */
|
||||
export function formatSuite(
|
||||
operations: Operation[],
|
||||
coverage: Coverage = { covered: [] },
|
||||
opts: FormatOptions = {},
|
||||
): string {
|
||||
return renderReport(assessCoverage(operations, coverage), opts)
|
||||
}
|
||||
Reference in New Issue
Block a user