290 lines
10 KiB
TypeScript
290 lines
10 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
import {
|
|
BANDS,
|
|
CAP_RULES,
|
|
DIMENSIONS,
|
|
bandFor,
|
|
scoreDimension,
|
|
scoreProduct,
|
|
scoreSuite,
|
|
type DimensionAssessment,
|
|
type DimensionKey,
|
|
type ProductAssessment,
|
|
formatProduct,
|
|
productReport,
|
|
} from '../src/index.js'
|
|
import { validateReport } from '@extant2000/evidence-record'
|
|
|
|
const ev = [{ ref: 'test/foo.test.ts:12' }]
|
|
|
|
/** A product scoring `n` on every dimension, with evidence. */
|
|
function flat(product: string, n: number, tier: 'A' | 'B' | 'C' = 'B'): ProductAssessment {
|
|
const dimensions = {} as Record<DimensionKey, DimensionAssessment>
|
|
for (const d of DIMENSIONS) dimensions[d] = { raw: n, evidence: ev }
|
|
return { product, tier, dimensions }
|
|
}
|
|
|
|
/** A healthy product with one dimension replaced, for isolating its output. */
|
|
function assessmentWith(dim: DimensionKey, a: DimensionAssessment): ProductAssessment {
|
|
const p = flat('app', 60)
|
|
p.dimensions[dim] = a
|
|
return p
|
|
}
|
|
|
|
describe('bandFor', () => {
|
|
it('maps the documented boundaries', () => {
|
|
expect(bandFor(0).name).toBe('Prototype')
|
|
expect(bandFor(19).name).toBe('Prototype')
|
|
expect(bandFor(20).name).toBe('Alpha')
|
|
expect(bandFor(40).name).toBe('Beta')
|
|
expect(bandFor(60).name).toBe('Launch-capable, with debt')
|
|
expect(bandFor(75).name).toBe('Production')
|
|
expect(bandFor(90).name).toBe('Mature')
|
|
expect(bandFor(100).name).toBe('Mature')
|
|
})
|
|
|
|
it('covers 0-100 with no gaps', () => {
|
|
for (let s = 0; s <= 100; s++) expect(bandFor(s)).toBeDefined()
|
|
})
|
|
|
|
it('has contiguous non-overlapping bands', () => {
|
|
for (let i = 1; i < BANDS.length; i++) {
|
|
expect(BANDS[i]!.min).toBe(BANDS[i - 1]!.max + 1)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('scoreDimension — anti-inflation rule 3 (cap by worst defect)', () => {
|
|
it('CATCHES THE CHARITABLE SCORE: an unrotated leaked secret caps Security at 45', () => {
|
|
// A hand scorer can note that a known-leaked, unrotated secret caps the
|
|
// dimension at 45 and still record ~70. Charity is not available here.
|
|
const r = scoreDimension('security', {
|
|
raw: 70,
|
|
openCaps: ['sec.leaked-secret'],
|
|
evidence: ev,
|
|
})
|
|
expect(r.raw).toBe(70)
|
|
expect(r.score).toBe(45)
|
|
expect(r.band.name).toBe('Beta')
|
|
expect(r.boundBy?.id).toBe('sec.leaked-secret')
|
|
})
|
|
|
|
it('applies the LOWEST open cap, not the first or last', () => {
|
|
const r = scoreDimension('security', {
|
|
raw: 90,
|
|
openCaps: ['sec.no-dep-monitoring', 'sec.cross-tenant', 'sec.no-restore'],
|
|
evidence: ev,
|
|
})
|
|
expect(r.score).toBe(30)
|
|
expect(r.boundBy?.id).toBe('sec.cross-tenant')
|
|
})
|
|
|
|
it('leaves a score below the cap untouched', () => {
|
|
const r = scoreDimension('security', {
|
|
raw: 25, openCaps: ['sec.leaked-secret'], evidence: ev,
|
|
})
|
|
expect(r.score).toBe(25)
|
|
expect(r.boundBy).toBeUndefined()
|
|
})
|
|
|
|
it('rejects a cap belonging to another dimension', () => {
|
|
expect(() => scoreDimension('bugs', { raw: 80, openCaps: ['sec.leaked-secret'] }))
|
|
.toThrow(/belongs to security/)
|
|
})
|
|
|
|
it('rejects an unknown cap id rather than ignoring it', () => {
|
|
// Silently ignoring a typo'd cap would inflate the score — the exact
|
|
// failure mode this library exists to prevent.
|
|
expect(() => scoreDimension('security', { raw: 80, openCaps: ['sec.tpyo'] }))
|
|
.toThrow(/Unknown cap rule/)
|
|
})
|
|
})
|
|
|
|
describe('scoreDimension — anti-inflation rule 5 (evidence above 70)', () => {
|
|
it('withholds an un-evidenced score above the threshold', () => {
|
|
const r = scoreDimension('bugs', { raw: 85 })
|
|
expect(r.score).toBe(70)
|
|
expect(r.evidenceWithheld).toBe(true)
|
|
})
|
|
|
|
it('allows an evidenced score above the threshold', () => {
|
|
const r = scoreDimension('bugs', { raw: 85, evidence: ev })
|
|
expect(r.score).toBe(85)
|
|
expect(r.evidenceWithheld).toBe(false)
|
|
})
|
|
|
|
it('does not touch an un-evidenced score at or below the threshold', () => {
|
|
const r = scoreDimension('bugs', { raw: 70 })
|
|
expect(r.score).toBe(70)
|
|
expect(r.evidenceWithheld).toBe(false)
|
|
})
|
|
|
|
it('an empty evidence array is not evidence', () => {
|
|
expect(scoreDimension('bugs', { raw: 85, evidence: [] }).score).toBe(70)
|
|
})
|
|
})
|
|
|
|
describe('scoreProduct', () => {
|
|
it('reports the LOWEST dimension as overall, not the average', () => {
|
|
const a = flat('app', 80)
|
|
a.dimensions.security = { raw: 40, evidence: ev }
|
|
const r = scoreProduct(a)
|
|
expect(r.overall).toBe(40)
|
|
expect(r.weakest).toBe('security')
|
|
expect(r.average).toBe(72) // (40+80*4)/5
|
|
expect(r.band.name).toBe('Beta')
|
|
})
|
|
|
|
it('a single capped dimension pins the whole product', () => {
|
|
// Four strong dimensions must not average away one disqualifying hole.
|
|
const a = flat('api', 88)
|
|
a.dimensions.security = { raw: 88, openCaps: ['sec.cross-tenant'], evidence: ev }
|
|
const r = scoreProduct(a)
|
|
expect(r.overall).toBe(30)
|
|
expect(r.band.name).toBe('Alpha')
|
|
})
|
|
|
|
it('assigns the tier weight', () => {
|
|
expect(scoreProduct(flat('x', 80, 'A')).weight).toBe(2)
|
|
expect(scoreProduct(flat('x', 80, 'B')).weight).toBe(1.5)
|
|
expect(scoreProduct(flat('x', 80, 'C')).weight).toBe(1)
|
|
})
|
|
|
|
it('throws on a missing dimension rather than scoring a partial product', () => {
|
|
const a = flat('x', 80)
|
|
delete (a.dimensions as Partial<Record<DimensionKey, DimensionAssessment>>).usability
|
|
expect(() => scoreProduct(a)).toThrow(/Missing assessment/)
|
|
})
|
|
|
|
it('carries the binding constraint and next action through', () => {
|
|
const a = flat('x', 80)
|
|
a.dimensions.bugs = {
|
|
raw: 68, evidence: ev,
|
|
bindingConstraint: 'server/api/** handlers untested',
|
|
nextAction: 'behaviour-asserting coverage',
|
|
}
|
|
const bugs = scoreProduct(a).dimensions.find(d => d.dimension === 'bugs')!
|
|
expect(bugs.bindingConstraint).toBe('server/api/** handlers untested')
|
|
expect(bugs.nextAction).toBe('behaviour-asserting coverage')
|
|
})
|
|
})
|
|
|
|
describe('scoreSuite', () => {
|
|
it('weights by tier', () => {
|
|
// A-tier (2) at 90, C-tier (1) at 60 → (90*2 + 60*1) / 3 = 80
|
|
const r = scoreSuite([flat('big', 90, 'A'), flat('small', 60, 'C')])
|
|
expect(r.dimensions.security).toBe(80)
|
|
})
|
|
|
|
it('applies a suite-wide cap BEFORE averaging', () => {
|
|
// Averaging first would let healthy products dilute a flaw affecting all
|
|
// of them: (90+90)/2 = 90, capped after → 90. Capping first → 45.
|
|
const r = scoreSuite(
|
|
[flat('a', 90, 'A'), flat('b', 90, 'A')],
|
|
{ suiteCaps: ['sec.leaked-secret'] },
|
|
)
|
|
expect(r.dimensions.security).toBe(45)
|
|
})
|
|
|
|
it('leads with the lowest suite dimension', () => {
|
|
const a = flat('a', 85)
|
|
a.dimensions.consistency = { raw: 63, evidence: ev }
|
|
const r = scoreSuite([a])
|
|
expect(r.overall).toBe(63)
|
|
expect(r.weakest).toBe('consistency')
|
|
})
|
|
|
|
it('throws on an empty suite instead of returning a flattering zero', () => {
|
|
expect(() => scoreSuite([])).toThrow(/at least one product/)
|
|
})
|
|
})
|
|
|
|
describe('rubric integrity', () => {
|
|
it('every cap rule has a unique id', () => {
|
|
const ids = CAP_RULES.map(c => c.id)
|
|
expect(new Set(ids).size).toBe(ids.length)
|
|
})
|
|
|
|
it('every cap belongs to a known dimension and sits in 0-100', () => {
|
|
for (const c of CAP_RULES) {
|
|
expect(DIMENSIONS).toContain(c.dimension)
|
|
expect(c.max).toBeGreaterThanOrEqual(0)
|
|
expect(c.max).toBeLessThanOrEqual(100)
|
|
}
|
|
})
|
|
|
|
it('every dimension carries at least one cap', () => {
|
|
for (const d of DIMENSIONS) {
|
|
expect(CAP_RULES.some(c => c.dimension === d)).toBe(true)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('conformance with the evidence-record standard', () => {
|
|
const withCapEvidence = scoreDimension('security', {
|
|
raw: 70,
|
|
openCaps: [{
|
|
id: CAP_RULES.find(c => c.dimension === 'security')!.id,
|
|
evidence: [{ ref: 'docs/secrets-review.md:12', note: 'exposed key, never rotated' }],
|
|
}],
|
|
})
|
|
|
|
it('a cap cites what establishes it', () => {
|
|
// The gap this closes: "CAPPED sec.leaked-secret" with nothing behind it.
|
|
expect(withCapEvidence.boundBy!.evidence).toHaveLength(1)
|
|
const text = formatProduct(scoreProduct(assessmentWith('security', {
|
|
raw: 70,
|
|
openCaps: [{
|
|
id: CAP_RULES.find(c => c.dimension === 'security')!.id,
|
|
evidence: [{ ref: 'docs/secrets-review.md:12', note: 'never rotated' }],
|
|
}],
|
|
})), { evidence: 'full' })
|
|
expect(text).toContain('docs/secrets-review.md:12')
|
|
expect(text).toContain('never rotated')
|
|
})
|
|
|
|
it('still applies a cap that cites nothing, and says it cited nothing', () => {
|
|
// An unproven cap is a smaller error than an unapplied one.
|
|
const bare = scoreDimension('security', {
|
|
raw: 90,
|
|
openCaps: [CAP_RULES.find(c => c.dimension === 'security')!.id],
|
|
})
|
|
expect(bare.score).toBeLessThan(90)
|
|
const text = formatProduct(scoreProduct(assessmentWith('security', {
|
|
raw: 90,
|
|
openCaps: [CAP_RULES.find(c => c.dimension === 'security')!.id],
|
|
})), { evidence: 'full' })
|
|
expect(text).toContain('No evidence was recorded for this cap')
|
|
})
|
|
|
|
it('converts a legacy path:line ref into a real citation', () => {
|
|
const d = scoreDimension('security', { raw: 50, evidence: [{ ref: 'server/api/x.ts:9' }] })
|
|
expect(d.evidence[0]!.source).toMatchObject({ kind: 'file', path: 'server/api/x.ts', line: 9 })
|
|
})
|
|
|
|
it('a score withheld for lack of evidence is not-assessed, not a pass', () => {
|
|
const p = scoreProduct(assessmentWith('security', { raw: 95 }))
|
|
const f = productReport(p).findings.find(x => x.id.endsWith('.security'))!
|
|
expect(f.determination).toBe('not-assessed')
|
|
expect(formatProduct(p)).toContain('no evidence was cited')
|
|
})
|
|
|
|
it('every conclusion carries a citation', () => {
|
|
expect(validateReport(productReport(scoreProduct(assessmentWith('security', { raw: 60 }))))).toEqual([])
|
|
})
|
|
|
|
it('labels the average so it cannot be read as the headline', () => {
|
|
const text = formatProduct(scoreProduct(assessmentWith('security', { raw: 60 })))
|
|
expect(text).toContain('never as the headline')
|
|
expect(text).toContain('set by the weakest dimension')
|
|
})
|
|
})
|
|
|
|
describe('band rendering', () => {
|
|
it('prints the band name, not a stringified object', () => {
|
|
const text = formatProduct(scoreProduct(flat('app', 80)))
|
|
expect(text).not.toContain('[object Object]')
|
|
expect(text).toContain('Production')
|
|
})
|
|
})
|