First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,289 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
BANDS,
|
||||
CAP_RULES,
|
||||
DIMENSIONS,
|
||||
bandFor,
|
||||
scoreDimension,
|
||||
scoreProduct,
|
||||
scoreSuite,
|
||||
type DimensionAssessment,
|
||||
type DimensionKey,
|
||||
type ProductAssessment,
|
||||
formatProduct,
|
||||
productReport,
|
||||
} from '../src/index.js'
|
||||
import { validateReport } from '@extant2000/evidence-record'
|
||||
|
||||
const ev = [{ ref: 'test/foo.test.ts:12' }]
|
||||
|
||||
/** A product scoring `n` on every dimension, with evidence. */
|
||||
function flat(product: string, n: number, tier: 'A' | 'B' | 'C' = 'B'): ProductAssessment {
|
||||
const dimensions = {} as Record<DimensionKey, DimensionAssessment>
|
||||
for (const d of DIMENSIONS) dimensions[d] = { raw: n, evidence: ev }
|
||||
return { product, tier, dimensions }
|
||||
}
|
||||
|
||||
/** A healthy product with one dimension replaced, for isolating its output. */
|
||||
function assessmentWith(dim: DimensionKey, a: DimensionAssessment): ProductAssessment {
|
||||
const p = flat('app', 60)
|
||||
p.dimensions[dim] = a
|
||||
return p
|
||||
}
|
||||
|
||||
describe('bandFor', () => {
|
||||
it('maps the documented boundaries', () => {
|
||||
expect(bandFor(0).name).toBe('Prototype')
|
||||
expect(bandFor(19).name).toBe('Prototype')
|
||||
expect(bandFor(20).name).toBe('Alpha')
|
||||
expect(bandFor(40).name).toBe('Beta')
|
||||
expect(bandFor(60).name).toBe('Launch-capable, with debt')
|
||||
expect(bandFor(75).name).toBe('Production')
|
||||
expect(bandFor(90).name).toBe('Mature')
|
||||
expect(bandFor(100).name).toBe('Mature')
|
||||
})
|
||||
|
||||
it('covers 0-100 with no gaps', () => {
|
||||
for (let s = 0; s <= 100; s++) expect(bandFor(s)).toBeDefined()
|
||||
})
|
||||
|
||||
it('has contiguous non-overlapping bands', () => {
|
||||
for (let i = 1; i < BANDS.length; i++) {
|
||||
expect(BANDS[i]!.min).toBe(BANDS[i - 1]!.max + 1)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('scoreDimension — anti-inflation rule 3 (cap by worst defect)', () => {
|
||||
it('CATCHES THE CHARITABLE SCORE: an unrotated leaked secret caps Security at 45', () => {
|
||||
// A hand scorer can note that a known-leaked, unrotated secret caps the
|
||||
// dimension at 45 and still record ~70. Charity is not available here.
|
||||
const r = scoreDimension('security', {
|
||||
raw: 70,
|
||||
openCaps: ['sec.leaked-secret'],
|
||||
evidence: ev,
|
||||
})
|
||||
expect(r.raw).toBe(70)
|
||||
expect(r.score).toBe(45)
|
||||
expect(r.band.name).toBe('Beta')
|
||||
expect(r.boundBy?.id).toBe('sec.leaked-secret')
|
||||
})
|
||||
|
||||
it('applies the LOWEST open cap, not the first or last', () => {
|
||||
const r = scoreDimension('security', {
|
||||
raw: 90,
|
||||
openCaps: ['sec.no-dep-monitoring', 'sec.cross-tenant', 'sec.no-restore'],
|
||||
evidence: ev,
|
||||
})
|
||||
expect(r.score).toBe(30)
|
||||
expect(r.boundBy?.id).toBe('sec.cross-tenant')
|
||||
})
|
||||
|
||||
it('leaves a score below the cap untouched', () => {
|
||||
const r = scoreDimension('security', {
|
||||
raw: 25, openCaps: ['sec.leaked-secret'], evidence: ev,
|
||||
})
|
||||
expect(r.score).toBe(25)
|
||||
expect(r.boundBy).toBeUndefined()
|
||||
})
|
||||
|
||||
it('rejects a cap belonging to another dimension', () => {
|
||||
expect(() => scoreDimension('bugs', { raw: 80, openCaps: ['sec.leaked-secret'] }))
|
||||
.toThrow(/belongs to security/)
|
||||
})
|
||||
|
||||
it('rejects an unknown cap id rather than ignoring it', () => {
|
||||
// Silently ignoring a typo'd cap would inflate the score — the exact
|
||||
// failure mode this library exists to prevent.
|
||||
expect(() => scoreDimension('security', { raw: 80, openCaps: ['sec.tpyo'] }))
|
||||
.toThrow(/Unknown cap rule/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('scoreDimension — anti-inflation rule 5 (evidence above 70)', () => {
|
||||
it('withholds an un-evidenced score above the threshold', () => {
|
||||
const r = scoreDimension('bugs', { raw: 85 })
|
||||
expect(r.score).toBe(70)
|
||||
expect(r.evidenceWithheld).toBe(true)
|
||||
})
|
||||
|
||||
it('allows an evidenced score above the threshold', () => {
|
||||
const r = scoreDimension('bugs', { raw: 85, evidence: ev })
|
||||
expect(r.score).toBe(85)
|
||||
expect(r.evidenceWithheld).toBe(false)
|
||||
})
|
||||
|
||||
it('does not touch an un-evidenced score at or below the threshold', () => {
|
||||
const r = scoreDimension('bugs', { raw: 70 })
|
||||
expect(r.score).toBe(70)
|
||||
expect(r.evidenceWithheld).toBe(false)
|
||||
})
|
||||
|
||||
it('an empty evidence array is not evidence', () => {
|
||||
expect(scoreDimension('bugs', { raw: 85, evidence: [] }).score).toBe(70)
|
||||
})
|
||||
})
|
||||
|
||||
describe('scoreProduct', () => {
|
||||
it('reports the LOWEST dimension as overall, not the average', () => {
|
||||
const a = flat('app', 80)
|
||||
a.dimensions.security = { raw: 40, evidence: ev }
|
||||
const r = scoreProduct(a)
|
||||
expect(r.overall).toBe(40)
|
||||
expect(r.weakest).toBe('security')
|
||||
expect(r.average).toBe(72) // (40+80*4)/5
|
||||
expect(r.band.name).toBe('Beta')
|
||||
})
|
||||
|
||||
it('a single capped dimension pins the whole product', () => {
|
||||
// Four strong dimensions must not average away one disqualifying hole.
|
||||
const a = flat('api', 88)
|
||||
a.dimensions.security = { raw: 88, openCaps: ['sec.cross-tenant'], evidence: ev }
|
||||
const r = scoreProduct(a)
|
||||
expect(r.overall).toBe(30)
|
||||
expect(r.band.name).toBe('Alpha')
|
||||
})
|
||||
|
||||
it('assigns the tier weight', () => {
|
||||
expect(scoreProduct(flat('x', 80, 'A')).weight).toBe(2)
|
||||
expect(scoreProduct(flat('x', 80, 'B')).weight).toBe(1.5)
|
||||
expect(scoreProduct(flat('x', 80, 'C')).weight).toBe(1)
|
||||
})
|
||||
|
||||
it('throws on a missing dimension rather than scoring a partial product', () => {
|
||||
const a = flat('x', 80)
|
||||
delete (a.dimensions as Partial<Record<DimensionKey, DimensionAssessment>>).usability
|
||||
expect(() => scoreProduct(a)).toThrow(/Missing assessment/)
|
||||
})
|
||||
|
||||
it('carries the binding constraint and next action through', () => {
|
||||
const a = flat('x', 80)
|
||||
a.dimensions.bugs = {
|
||||
raw: 68, evidence: ev,
|
||||
bindingConstraint: 'server/api/** handlers untested',
|
||||
nextAction: 'behaviour-asserting coverage',
|
||||
}
|
||||
const bugs = scoreProduct(a).dimensions.find(d => d.dimension === 'bugs')!
|
||||
expect(bugs.bindingConstraint).toBe('server/api/** handlers untested')
|
||||
expect(bugs.nextAction).toBe('behaviour-asserting coverage')
|
||||
})
|
||||
})
|
||||
|
||||
describe('scoreSuite', () => {
|
||||
it('weights by tier', () => {
|
||||
// A-tier (2) at 90, C-tier (1) at 60 → (90*2 + 60*1) / 3 = 80
|
||||
const r = scoreSuite([flat('big', 90, 'A'), flat('small', 60, 'C')])
|
||||
expect(r.dimensions.security).toBe(80)
|
||||
})
|
||||
|
||||
it('applies a suite-wide cap BEFORE averaging', () => {
|
||||
// Averaging first would let healthy products dilute a flaw affecting all
|
||||
// of them: (90+90)/2 = 90, capped after → 90. Capping first → 45.
|
||||
const r = scoreSuite(
|
||||
[flat('a', 90, 'A'), flat('b', 90, 'A')],
|
||||
{ suiteCaps: ['sec.leaked-secret'] },
|
||||
)
|
||||
expect(r.dimensions.security).toBe(45)
|
||||
})
|
||||
|
||||
it('leads with the lowest suite dimension', () => {
|
||||
const a = flat('a', 85)
|
||||
a.dimensions.consistency = { raw: 63, evidence: ev }
|
||||
const r = scoreSuite([a])
|
||||
expect(r.overall).toBe(63)
|
||||
expect(r.weakest).toBe('consistency')
|
||||
})
|
||||
|
||||
it('throws on an empty suite instead of returning a flattering zero', () => {
|
||||
expect(() => scoreSuite([])).toThrow(/at least one product/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('rubric integrity', () => {
|
||||
it('every cap rule has a unique id', () => {
|
||||
const ids = CAP_RULES.map(c => c.id)
|
||||
expect(new Set(ids).size).toBe(ids.length)
|
||||
})
|
||||
|
||||
it('every cap belongs to a known dimension and sits in 0-100', () => {
|
||||
for (const c of CAP_RULES) {
|
||||
expect(DIMENSIONS).toContain(c.dimension)
|
||||
expect(c.max).toBeGreaterThanOrEqual(0)
|
||||
expect(c.max).toBeLessThanOrEqual(100)
|
||||
}
|
||||
})
|
||||
|
||||
it('every dimension carries at least one cap', () => {
|
||||
for (const d of DIMENSIONS) {
|
||||
expect(CAP_RULES.some(c => c.dimension === d)).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('conformance with the evidence-record standard', () => {
|
||||
const withCapEvidence = scoreDimension('security', {
|
||||
raw: 70,
|
||||
openCaps: [{
|
||||
id: CAP_RULES.find(c => c.dimension === 'security')!.id,
|
||||
evidence: [{ ref: 'docs/secrets-review.md:12', note: 'exposed key, never rotated' }],
|
||||
}],
|
||||
})
|
||||
|
||||
it('a cap cites what establishes it', () => {
|
||||
// The gap this closes: "CAPPED sec.leaked-secret" with nothing behind it.
|
||||
expect(withCapEvidence.boundBy!.evidence).toHaveLength(1)
|
||||
const text = formatProduct(scoreProduct(assessmentWith('security', {
|
||||
raw: 70,
|
||||
openCaps: [{
|
||||
id: CAP_RULES.find(c => c.dimension === 'security')!.id,
|
||||
evidence: [{ ref: 'docs/secrets-review.md:12', note: 'never rotated' }],
|
||||
}],
|
||||
})), { evidence: 'full' })
|
||||
expect(text).toContain('docs/secrets-review.md:12')
|
||||
expect(text).toContain('never rotated')
|
||||
})
|
||||
|
||||
it('still applies a cap that cites nothing, and says it cited nothing', () => {
|
||||
// An unproven cap is a smaller error than an unapplied one.
|
||||
const bare = scoreDimension('security', {
|
||||
raw: 90,
|
||||
openCaps: [CAP_RULES.find(c => c.dimension === 'security')!.id],
|
||||
})
|
||||
expect(bare.score).toBeLessThan(90)
|
||||
const text = formatProduct(scoreProduct(assessmentWith('security', {
|
||||
raw: 90,
|
||||
openCaps: [CAP_RULES.find(c => c.dimension === 'security')!.id],
|
||||
})), { evidence: 'full' })
|
||||
expect(text).toContain('No evidence was recorded for this cap')
|
||||
})
|
||||
|
||||
it('converts a legacy path:line ref into a real citation', () => {
|
||||
const d = scoreDimension('security', { raw: 50, evidence: [{ ref: 'server/api/x.ts:9' }] })
|
||||
expect(d.evidence[0]!.source).toMatchObject({ kind: 'file', path: 'server/api/x.ts', line: 9 })
|
||||
})
|
||||
|
||||
it('a score withheld for lack of evidence is not-assessed, not a pass', () => {
|
||||
const p = scoreProduct(assessmentWith('security', { raw: 95 }))
|
||||
const f = productReport(p).findings.find(x => x.id.endsWith('.security'))!
|
||||
expect(f.determination).toBe('not-assessed')
|
||||
expect(formatProduct(p)).toContain('no evidence was cited')
|
||||
})
|
||||
|
||||
it('every conclusion carries a citation', () => {
|
||||
expect(validateReport(productReport(scoreProduct(assessmentWith('security', { raw: 60 }))))).toEqual([])
|
||||
})
|
||||
|
||||
it('labels the average so it cannot be read as the headline', () => {
|
||||
const text = formatProduct(scoreProduct(assessmentWith('security', { raw: 60 })))
|
||||
expect(text).toContain('never as the headline')
|
||||
expect(text).toContain('set by the weakest dimension')
|
||||
})
|
||||
})
|
||||
|
||||
describe('band rendering', () => {
|
||||
it('prints the band name, not a stringified object', () => {
|
||||
const text = formatProduct(scoreProduct(flat('app', 80)))
|
||||
expect(text).not.toContain('[object Object]')
|
||||
expect(text).toContain('Production')
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user