First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,98 @@
|
||||
import type {
|
||||
PolicyReport,
|
||||
PolicyRule,
|
||||
RuleResult,
|
||||
SourceFile,
|
||||
Violation,
|
||||
} from './types.js'
|
||||
|
||||
function matchesPath(path: string, includes: string[]): boolean {
|
||||
return includes.some(i => path.includes(i))
|
||||
}
|
||||
|
||||
/**
|
||||
* A regex with the `g` flag carries `lastIndex` between `.test()` calls, so the
|
||||
* same pattern reused across files silently starts matching from wherever the
|
||||
* previous file left off — producing false passes that look like clean runs.
|
||||
* Test against a fresh, non-global copy every time.
|
||||
*/
|
||||
function test(re: RegExp, content: string): boolean {
|
||||
return new RegExp(re.source, re.flags.replace(/[gy]/g, '')).test(content)
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate the trigger match so a finding can cite a line, not just a file.
|
||||
*
|
||||
* Scans line by line rather than counting newlines before a whole-file match
|
||||
* index, because the trigger may legitimately span nothing more than one line
|
||||
* and this keeps the excerpt exactly what the reader will see in the editor.
|
||||
*/
|
||||
function locate(re: RegExp, content: string): { line: number, excerpt: string } | undefined {
|
||||
const bare = new RegExp(re.source, re.flags.replace(/[gy]/g, ''))
|
||||
const lines = content.split('\n')
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (bare.test(lines[i]!)) return { line: i + 1, excerpt: lines[i]!.trim().slice(0, 120) }
|
||||
}
|
||||
// The trigger matched the file but not any single line — a multi-line
|
||||
// pattern. Cite the file without a line rather than inventing one.
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Evaluate one rule over already-loaded files. Pure — no I/O, no git. */
|
||||
export function evaluateRule(rule: PolicyRule, files: SourceFile[]): RuleResult {
|
||||
let considered = 0
|
||||
let subject = 0
|
||||
let exempted = 0
|
||||
const violations: Violation[] = []
|
||||
const compliantPaths: string[] = []
|
||||
const exemptedPaths: string[] = []
|
||||
|
||||
for (const f of files) {
|
||||
if (!matchesPath(f.path, rule.pathIncludes)) continue
|
||||
considered++
|
||||
|
||||
if (!test(rule.trigger, f.content)) continue
|
||||
subject++
|
||||
|
||||
if (rule.allow.some(a => test(a, f.content))) {
|
||||
compliantPaths.push(f.path)
|
||||
continue
|
||||
}
|
||||
|
||||
if (f.content.includes(rule.exemptMarker)) {
|
||||
exempted++
|
||||
exemptedPaths.push(f.path)
|
||||
continue
|
||||
}
|
||||
|
||||
const at = locate(rule.trigger, f.content)
|
||||
violations.push({
|
||||
ruleId: rule.id,
|
||||
path: f.path,
|
||||
reason: rule.description,
|
||||
line: at?.line,
|
||||
excerpt: at?.excerpt,
|
||||
})
|
||||
}
|
||||
|
||||
return { rule, considered, subject, exempted, violations, compliantPaths, exemptedPaths }
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate every rule.
|
||||
*
|
||||
* `vacuous` is reported separately from `passed`. A run where no changed file
|
||||
* was subject to any rule proves nothing — treating it as a pass is how a gate
|
||||
* ends up green for months while covering nothing, which is the same defect as
|
||||
* a test that cannot fail.
|
||||
*/
|
||||
export function evaluatePolicy(rules: PolicyRule[], files: SourceFile[]): PolicyReport {
|
||||
const results = rules.map(r => evaluateRule(r, files))
|
||||
const violations = results.flatMap(r => r.violations)
|
||||
return {
|
||||
results,
|
||||
violations,
|
||||
passed: violations.length === 0,
|
||||
vacuous: results.every(r => r.subject === 0),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from './types.js'
|
||||
export * from './evaluate.js'
|
||||
export * from './rules.js'
|
||||
export * from './report.js'
|
||||
@@ -0,0 +1,90 @@
|
||||
import {
|
||||
type CapabilityReport,
|
||||
type Finding,
|
||||
evidence,
|
||||
formatReport as renderReport,
|
||||
type FormatOptions,
|
||||
} from '@extant2000/evidence-record'
|
||||
import type { PolicyReport, RuleResult } from './types.js'
|
||||
|
||||
/**
|
||||
* One finding per rule, in the shared vocabulary.
|
||||
*
|
||||
* The three-way split is the point of this library and now has three-way
|
||||
* types to carry it:
|
||||
*
|
||||
* nothing was subject to the rule -> not-assessed (used to print VACUOUS)
|
||||
* files were subject and complied -> pass, citing the files it checked
|
||||
* files were subject and did not -> fail, citing file:line and the line
|
||||
*
|
||||
* "Nothing changed that this rule covers" and "the policy holds" are different
|
||||
* statements, and a gate that prints ✓ for both is indistinguishable from one
|
||||
* that does nothing.
|
||||
*/
|
||||
function findingFor(r: RuleResult): Finding {
|
||||
const doc = evidence.document(r.rule.description, r.rule.id)
|
||||
|
||||
if (r.violations.length > 0) {
|
||||
return {
|
||||
id: r.rule.id,
|
||||
summary: `${r.rule.id} — ${r.violations.length} file(s) subject to the rule did not satisfy it`,
|
||||
determination: 'fail',
|
||||
severity: 'high',
|
||||
detail: r.rule.remedy.join(' '),
|
||||
evidence: [
|
||||
...r.violations.map(v => evidence.file(v.path, v.line, v.excerpt, v.reason)),
|
||||
doc,
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
if (r.subject === 0) {
|
||||
return {
|
||||
id: r.rule.id,
|
||||
summary: `${r.rule.id} — ${r.considered} file(s) examined, none subject to the rule`,
|
||||
determination: 'not-assessed',
|
||||
severity: 'medium',
|
||||
detail: 'Nothing was checked against this rule. That is not evidence the policy holds.',
|
||||
// Deliberately empty: the whole content of this finding is that there
|
||||
// was nothing to observe.
|
||||
evidence: [],
|
||||
}
|
||||
}
|
||||
|
||||
const ex = r.exempted ? `, ${r.exempted} exempted` : ''
|
||||
return {
|
||||
id: r.rule.id,
|
||||
summary: `${r.rule.id} — ${r.subject} file(s) checked and compliant${ex}`,
|
||||
determination: 'pass',
|
||||
severity: 'info',
|
||||
evidence: [
|
||||
...r.compliantPaths.map(p => evidence.file(p, undefined, undefined, 'satisfied the rule')),
|
||||
// Exemptions are evidence too, and printing them is what keeps an
|
||||
// exemption from being a quiet way to pass.
|
||||
...r.exemptedPaths.map(p => evidence.file(p, undefined, undefined, `exempted via "${r.rule.exemptMarker}"`)),
|
||||
doc,
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert a policy run into the portfolio-standard report shape. */
|
||||
export function toReport(report: PolicyReport): CapabilityReport {
|
||||
const considered = report.results.reduce((n, r) => n + r.considered, 0)
|
||||
return {
|
||||
capability: 'PolicyEnforced',
|
||||
scope: `${report.results.length} rule(s) over ${considered} file(s)`,
|
||||
examined: considered,
|
||||
findings: report.results.map(findingFor),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a report for a CI log.
|
||||
*
|
||||
* Delegates to the shared renderer. The distinction that used to be carried
|
||||
* by the word VACUOUS is now carried by NOT ASSESSED plus a sentence, which
|
||||
* means the same thing without requiring a glossary.
|
||||
*/
|
||||
export function formatReport(report: PolicyReport, opts: FormatOptions = {}): string {
|
||||
return renderReport(toReport(report), opts)
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
import type { PolicyRule } from './types.js'
|
||||
|
||||
/**
|
||||
* Rule factories for three common gates.
|
||||
*
|
||||
* Each takes the parts that vary per app. An org-scoping gate differs between
|
||||
* apps precisely because these lists differ (one app guards with
|
||||
* `requireOrg` and tenant columns, another with RBAC capability checks), so
|
||||
* they are parameters, not constants.
|
||||
*/
|
||||
|
||||
export interface OrgScopingOptions {
|
||||
/** Accessors that BYPASS row-level security. */
|
||||
serviceClients?: RegExp
|
||||
/** Named guards that make a service-role handler safe. */
|
||||
guards: RegExp
|
||||
/** Optional: inline tenant-column filter that also counts as scoped. */
|
||||
inlineTenantFilter?: RegExp
|
||||
pathIncludes?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* A changed handler using a service-role client (RLS bypassed) must carry an
|
||||
* authorization check or a documented exemption.
|
||||
*
|
||||
* Diff-scoped on purpose: it does not police pre-existing unguarded endpoints
|
||||
* (track those separately), only new or changed code where the author is right there.
|
||||
* A gate nobody trusts is worse than no gate.
|
||||
*
|
||||
* LIMITATION, inherited and worth restating: this confirms a guard is PRESENT,
|
||||
* not that it filters the caller's own rows. Value-correctness is for tests and
|
||||
* review.
|
||||
*/
|
||||
export function orgScopingRule(opts: OrgScopingOptions): PolicyRule {
|
||||
const allow = [opts.guards]
|
||||
if (opts.inlineTenantFilter) allow.push(opts.inlineTenantFilter)
|
||||
|
||||
return {
|
||||
id: 'org-scoping',
|
||||
description: 'service-role handler with no authorization check or exemption',
|
||||
pathIncludes: opts.pathIncludes ?? ['server/api/'],
|
||||
trigger: opts.serviceClients
|
||||
?? /useServerSupabase|serverSupabaseServiceRole|createServiceClient|createClient\(/,
|
||||
allow,
|
||||
exemptMarker: 'org-scoping-exempt:',
|
||||
remedy: [
|
||||
'Add an authorization guard, or scope the query by a tenant column.',
|
||||
'If the route is genuinely cross-tenant (cron, token, webhook, admin),',
|
||||
'document it: // org-scoping-exempt: <reason>',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A changed file adding a dynamic `.in()` filter must chunk it.
|
||||
*
|
||||
* The bug: supabase-js puts every id in the query string, so past roughly 400
|
||||
* UUIDs the request exceeds the header limit and dies — returning null, which
|
||||
* `data || []` turns into a convincing empty state on top of real rows. It does
|
||||
* not error. That is why this is a gate and not a code review note.
|
||||
*/
|
||||
export function inOverflowRule(opts: { chunkHelpers: RegExp, pathIncludes?: string[] }): PolicyRule {
|
||||
return {
|
||||
id: 'in-overflow',
|
||||
description: 'dynamic .in() filter with no chunking helper or exemption',
|
||||
pathIncludes: opts.pathIncludes ?? ['server/', 'app/'],
|
||||
// `.in('col', someVariable)` — a literal array is bounded and fine.
|
||||
trigger: /\.in\(\s*['"][^'"]+['"]\s*,\s*(?!\[)[A-Za-z_$]/,
|
||||
allow: [opts.chunkHelpers],
|
||||
exemptMarker: 'in-overflow-ok:',
|
||||
remedy: [
|
||||
'Chunk the filter (selectInChunks or equivalent), or document why the',
|
||||
'list is bounded: // in-overflow-ok: <reason>',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/** A rule asserting a stated claim has an enforcement mechanism behind it. */
|
||||
export interface ClaimOptions {
|
||||
id: string
|
||||
/** Where the claim is made (marketing copy, docs, policy pages). */
|
||||
claimPaths: string[]
|
||||
/** The claim text. */
|
||||
claim: RegExp
|
||||
/** Evidence that it is actually enforced. */
|
||||
enforcedBy: RegExp
|
||||
remedy?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The general "advertised == implemented" shape.
|
||||
*
|
||||
* The typical case: uptime or SLA language appears on pricing and marketing
|
||||
* pages with no server-side enforcement anywhere. A public claim with nothing
|
||||
* behind it is a compliance risk, not just a documentation gap.
|
||||
*/
|
||||
export function claimRule(opts: ClaimOptions): PolicyRule {
|
||||
return {
|
||||
id: opts.id,
|
||||
description: 'public claim with no enforcement behind it',
|
||||
pathIncludes: opts.claimPaths,
|
||||
trigger: opts.claim,
|
||||
allow: [opts.enforcedBy],
|
||||
exemptMarker: `${opts.id}-exempt:`,
|
||||
remedy: opts.remedy ?? [
|
||||
'Either implement the enforcement, or remove/soften the claim.',
|
||||
`If enforcement lives elsewhere, document it: // ${opts.id}-exempt: <where>`,
|
||||
],
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* A policy that must hold in the running codebase, not merely in a document.
|
||||
*
|
||||
* Policy gates written as shell scripts tend to share one engine and differ
|
||||
* only in their patterns. An org-scoping gate copied across several apps can
|
||||
* legitimately end up with a distinct copy per app, and that divergence is
|
||||
* CORRECT, not drift: one app scopes by an `org_id` tenant column, another has
|
||||
* no tenant columns at all and authorises via RBAC capability checks. Forcing
|
||||
* one script would break one of them.
|
||||
*
|
||||
* So the engine is shared and the policy is data.
|
||||
*/
|
||||
|
||||
export interface PolicyRule {
|
||||
/** Stable id, used in output and to reference a rule. */
|
||||
id: string
|
||||
/** One line: what must be true. */
|
||||
description: string
|
||||
|
||||
/**
|
||||
* Only files whose path matches one of these substrings/patterns are
|
||||
* considered. Kept deliberately simple — a rule that needs a real glob
|
||||
* engine is usually a rule that wants narrowing instead.
|
||||
*/
|
||||
pathIncludes: string[]
|
||||
|
||||
/**
|
||||
* The file is SUBJECT to this rule only if its content matches. This is the
|
||||
* risky construct — a service-role client, an unbounded `.in()`, and so on.
|
||||
*/
|
||||
trigger: RegExp
|
||||
|
||||
/**
|
||||
* Any match here makes the file compliant. Multiple entries are OR-ed,
|
||||
* because there is usually more than one legitimate way to satisfy a policy.
|
||||
*/
|
||||
allow: RegExp[]
|
||||
|
||||
/**
|
||||
* A comment marker that documents a deliberate exemption, e.g.
|
||||
* `org-scoping-exempt:`. Exemptions are part of a workable gate: without one,
|
||||
* the legitimate cross-tenant cron job forces someone to disable the check
|
||||
* entirely.
|
||||
*/
|
||||
exemptMarker: string
|
||||
|
||||
/** Lines printed under a violation telling the author how to fix it. */
|
||||
remedy: string[]
|
||||
}
|
||||
|
||||
export interface SourceFile {
|
||||
path: string
|
||||
content: string
|
||||
}
|
||||
|
||||
export interface Violation {
|
||||
ruleId: string
|
||||
path: string
|
||||
/** Why it failed, in one line. */
|
||||
reason: string
|
||||
/**
|
||||
* 1-indexed line where the trigger matched — the construct that made this
|
||||
* file subject to the rule. A violation that names only the file sends the
|
||||
* reader hunting; this points at the line that did it.
|
||||
*/
|
||||
line?: number
|
||||
/** The matched line, trimmed. */
|
||||
excerpt?: string
|
||||
}
|
||||
|
||||
export interface RuleResult {
|
||||
rule: PolicyRule
|
||||
/** Files the rule actually examined (matched pathIncludes). */
|
||||
considered: number
|
||||
/** Files that were subject to it (matched the trigger). */
|
||||
subject: number
|
||||
/** Files excused by an exemption marker. */
|
||||
exempted: number
|
||||
violations: Violation[]
|
||||
/**
|
||||
* Paths that were subject to the rule and satisfied it.
|
||||
*
|
||||
* Retained so a PASS can cite what it checked. A gate that reports "✓ 3
|
||||
* files checked" without naming them is an asserted result, and this
|
||||
* library's whole premise is that written policy and enforced policy are
|
||||
* different things.
|
||||
*/
|
||||
compliantPaths: string[]
|
||||
/** Paths excused by the exemption marker — counted, never hidden. */
|
||||
exemptedPaths: string[]
|
||||
}
|
||||
|
||||
export interface PolicyReport {
|
||||
results: RuleResult[]
|
||||
violations: Violation[]
|
||||
passed: boolean
|
||||
/**
|
||||
* True when there was nothing to check. Distinguished from `passed` on
|
||||
* purpose: "no changed files matched" is not evidence the policy holds, and
|
||||
* a report that conflates them is the same lie as a test that cannot fail.
|
||||
*/
|
||||
vacuous: boolean
|
||||
}
|
||||
Reference in New Issue
Block a user