First public release
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+282
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* SelfServe — decide whether partner-initiated provisioning is safe to run,
|
||||
* and whether the queue running it is actually healthy.
|
||||
*
|
||||
* **This does not provision anything.** It assumes a provisioning worker
|
||||
* that already exists (one that claims jobs via `FOR UPDATE SKIP LOCKED`,
|
||||
* renders a per-tenant compose stack, brings it up and wires DNS).
|
||||
* Reimplementing that here would be duplication. What is missing is the half that decides whether to let it
|
||||
* run — which is the half that keeps self-service from being an outage.
|
||||
*
|
||||
* Four checks, each from a property of that kind of deployment:
|
||||
*
|
||||
* 1. **Dormant is not proven.** The worker ships behind
|
||||
* `DISABLE_PROVISION_WORKER=1`. Built, enabled, and proven in production
|
||||
* are three states, and reporting a dormant capability as ready is how
|
||||
* the first real tenant becomes the test.
|
||||
*
|
||||
* 2. **Self-serve without a ceiling is a denial of service you run against
|
||||
* yourself.** Every tenant is a compose stack on one host. A signup loop
|
||||
* with no quota fills the host, and on a shared host a full disk wedges
|
||||
* every container, not only the new one.
|
||||
*
|
||||
* 3. **A claimed job looks like progress.** `SKIP LOCKED` means a worker
|
||||
* that dies mid-job leaves that job claimed forever. Not queued, not
|
||||
* failed, nothing retrying it — the one state no dashboard alarms on.
|
||||
*
|
||||
* 4. **Provisioned is not reachable.** DNS wired and the stack up is not
|
||||
* the same as the tenant's URL serving.
|
||||
*
|
||||
* Pure: no clock, no I/O. `now` is a parameter.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CapabilityReport,
|
||||
type Finding,
|
||||
type FormatOptions,
|
||||
evidence,
|
||||
formatReport as renderReport,
|
||||
} from '@extant2000/evidence-record'
|
||||
|
||||
export interface CapacityLimits {
|
||||
/** Hard ceiling on tenants for this host. */
|
||||
maxTenants?: number
|
||||
/** Ceiling per account, so one account cannot take the host. */
|
||||
maxPerAccount?: number
|
||||
/** Free disk that must REMAIN after provisioning. */
|
||||
minFreeDiskBytes?: number
|
||||
}
|
||||
|
||||
export interface HostState {
|
||||
tenants: number
|
||||
/** `null` when not measured — never treated as plentiful. */
|
||||
freeDiskBytes: number | null
|
||||
/** Where these readings came from. */
|
||||
where: string
|
||||
}
|
||||
|
||||
export interface ProvisioningRequest {
|
||||
account: string
|
||||
tenantName: string
|
||||
/** Tenants this account already has. */
|
||||
existingForAccount: number
|
||||
/** Estimated footprint of the new stack. `null` when unknown. */
|
||||
estimatedDiskBytes: number | null
|
||||
where: string
|
||||
}
|
||||
|
||||
export type JobState = 'queued' | 'claimed' | 'done' | 'failed'
|
||||
|
||||
export interface Job {
|
||||
id: string
|
||||
state: JobState
|
||||
/** ISO 8601, when a worker claimed it. */
|
||||
claimedAt?: string
|
||||
claimedBy?: string
|
||||
where: string
|
||||
}
|
||||
|
||||
export interface ServiceState {
|
||||
/** False when the worker ships dormant behind a kill flag. */
|
||||
workerEnabled: boolean
|
||||
/** True once a real tenant has gone end to end in production. */
|
||||
provenInProduction?: boolean
|
||||
/** Seconds a job may stay claimed before it is presumed abandoned. */
|
||||
maxClaimSeconds?: number
|
||||
/** Whether provisioned tenants are verified reachable afterwards. */
|
||||
verifiesReachability?: boolean
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_CLAIM = 900
|
||||
|
||||
const fmt = (n: number): string =>
|
||||
n >= 1e9 ? `${(n / 1e9).toFixed(1)} GB` : n >= 1e6 ? `${(n / 1e6).toFixed(0)} MB` : `${n} B`
|
||||
|
||||
/**
|
||||
* Whether the service may accept a new provisioning request.
|
||||
*
|
||||
* The absence of a limit is itself a finding. A quota that was never set is
|
||||
* not an unlimited quota, it is an unconsidered one — and "unlimited" on a
|
||||
* single host is not a decision anyone would make deliberately.
|
||||
*/
|
||||
export function preflight(
|
||||
request: ProvisioningRequest,
|
||||
host: HostState,
|
||||
limits: CapacityLimits = {},
|
||||
): Finding[] {
|
||||
const out: Finding[] = []
|
||||
const cite = evidence.file(host.where, undefined,
|
||||
`${host.tenants} tenants, ${host.freeDiskBytes === null ? 'disk unmeasured' : `${fmt(host.freeDiskBytes)} free`}`)
|
||||
|
||||
const noLimits = limits.maxTenants === undefined
|
||||
&& limits.maxPerAccount === undefined
|
||||
&& limits.minFreeDiskBytes === undefined
|
||||
|
||||
if (noLimits) {
|
||||
out.push({
|
||||
id: 'capacity/no-limits',
|
||||
summary: 'No capacity limits are declared',
|
||||
determination: 'fail',
|
||||
severity: 'critical',
|
||||
detail: 'Self-serve provisioning with no ceiling is a denial of service you run against yourself. Every tenant is a stack on one host, and a signup loop fills it — a quota that was never set is not unlimited, it is unconsidered.',
|
||||
evidence: [cite],
|
||||
})
|
||||
}
|
||||
|
||||
if (limits.maxTenants !== undefined && host.tenants + 1 > limits.maxTenants) {
|
||||
out.push({
|
||||
id: 'capacity/host',
|
||||
summary: `Host is at ${host.tenants} of ${limits.maxTenants} tenants`,
|
||||
determination: 'fail',
|
||||
severity: 'high',
|
||||
detail: 'Provisioning would exceed the host ceiling.',
|
||||
evidence: [cite],
|
||||
})
|
||||
}
|
||||
|
||||
if (limits.maxPerAccount !== undefined && request.existingForAccount + 1 > limits.maxPerAccount) {
|
||||
out.push({
|
||||
id: 'capacity/account',
|
||||
summary: `${request.account} has ${request.existingForAccount} of ${limits.maxPerAccount} allowed tenants`,
|
||||
determination: 'fail',
|
||||
severity: 'high',
|
||||
detail: 'A per-account ceiling is what stops one account taking the host.',
|
||||
evidence: [evidence.file(request.where, undefined, `account ${request.account}`)],
|
||||
})
|
||||
}
|
||||
|
||||
if (limits.minFreeDiskBytes !== undefined) {
|
||||
if (host.freeDiskBytes === null || request.estimatedDiskBytes === null) {
|
||||
out.push({
|
||||
id: 'capacity/disk',
|
||||
summary: 'Disk headroom after provisioning cannot be computed',
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: `${host.freeDiskBytes === null ? 'Free disk was not measured' : 'The new stack has no size estimate'}. An unmeasured resource is not a plentiful one, and a full disk on this host wedges every container, not only the new tenant.`,
|
||||
evidence: [cite],
|
||||
})
|
||||
} else {
|
||||
const after = host.freeDiskBytes - request.estimatedDiskBytes
|
||||
if (after < limits.minFreeDiskBytes) {
|
||||
out.push({
|
||||
id: 'capacity/disk',
|
||||
summary: `${fmt(after)} free after provisioning, below the ${fmt(limits.minFreeDiskBytes)} floor`,
|
||||
determination: 'fail',
|
||||
severity: 'critical',
|
||||
detail: 'A full disk on this host wedges every container, not only the new tenant.',
|
||||
evidence: [cite],
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (out.length === 0) {
|
||||
out.push({
|
||||
id: 'capacity',
|
||||
summary: `Capacity available for ${request.tenantName}`,
|
||||
determination: 'pass',
|
||||
severity: 'info',
|
||||
evidence: [cite],
|
||||
})
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Jobs claimed and never finished.
|
||||
*
|
||||
* `SKIP LOCKED` claiming means a worker that dies mid-job leaves that job
|
||||
* claimed forever: not queued, not failed, nothing retries it. It is the one
|
||||
* state that looks like progress, which is why nothing alarms on it.
|
||||
*/
|
||||
export function stuckJobs(jobs: Job[], now: string, maxClaimSeconds = DEFAULT_MAX_CLAIM): Job[] {
|
||||
const n = Date.parse(now)
|
||||
return (jobs ?? []).filter(j => {
|
||||
if (j.state !== 'claimed' || !j.claimedAt) return false
|
||||
const t = Date.parse(j.claimedAt)
|
||||
if (!Number.isFinite(t) || !Number.isFinite(n)) return false
|
||||
return (n - t) / 1000 > maxClaimSeconds
|
||||
})
|
||||
}
|
||||
|
||||
export interface AssessOptions {
|
||||
/** ISO 8601. Injected, never read from the clock. */
|
||||
now: string
|
||||
request?: ProvisioningRequest
|
||||
host?: HostState
|
||||
limits?: CapacityLimits
|
||||
jobs?: Job[]
|
||||
}
|
||||
|
||||
/** Assess whether self-service provisioning is safe to run right now. */
|
||||
export function assessService(service: ServiceState, opts: AssessOptions): CapabilityReport {
|
||||
const findings: Finding[] = []
|
||||
const maxClaim = service.maxClaimSeconds ?? DEFAULT_MAX_CLAIM
|
||||
|
||||
if (!service.workerEnabled) {
|
||||
findings.push({
|
||||
id: 'worker/dormant',
|
||||
summary: 'The provisioning worker is disabled',
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: 'Built, enabled and proven in production are three states. A dormant capability reported as ready is how the first real tenant becomes the test.',
|
||||
evidence: [evidence.document('DISABLE_PROVISION_WORKER', undefined, undefined, 'the worker ships dormant behind this flag')],
|
||||
})
|
||||
} else if (!service.provenInProduction) {
|
||||
findings.push({
|
||||
id: 'worker/unproven',
|
||||
summary: 'The worker is enabled but has never completed a real tenant',
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: 'Enabled is not proven. Until one tenant has gone end to end in production, the first customer is the integration test.',
|
||||
evidence: [],
|
||||
})
|
||||
}
|
||||
|
||||
if (opts.request && opts.host) {
|
||||
findings.push(...preflight(opts.request, opts.host, opts.limits ?? {}))
|
||||
}
|
||||
|
||||
const jobs = opts.jobs ?? []
|
||||
const stuck = stuckJobs(jobs, opts.now, maxClaim)
|
||||
for (const j of stuck) {
|
||||
findings.push({
|
||||
id: `job/${j.id}`,
|
||||
summary: `job ${j.id} has been claimed by ${j.claimedBy ?? 'a worker'} since ${j.claimedAt}`,
|
||||
determination: 'fail',
|
||||
severity: 'critical',
|
||||
detail: `Claimed longer than ${Math.round(maxClaim / 60)} minutes. With SKIP LOCKED claiming, a worker that died mid-job leaves it claimed forever — not queued, not failed, nothing retrying it. It is the one state that looks like progress.`,
|
||||
evidence: [evidence.record('provisioning_jobs', j.id, 'state', 'claimed', j.where)],
|
||||
})
|
||||
}
|
||||
|
||||
if (service.verifiesReachability !== true) {
|
||||
findings.push({
|
||||
id: 'verify/reachability',
|
||||
summary: 'Provisioned tenants are not verified reachable',
|
||||
determination: 'not-assessed',
|
||||
severity: 'high',
|
||||
detail: "DNS wired and the stack up is not the same as the tenant's URL serving. Without a post-provision probe, the first person to discover a broken tenant is the tenant.",
|
||||
evidence: [],
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
capability: 'SelfServe',
|
||||
scope: `worker ${service.workerEnabled ? 'enabled' : 'dormant'}, ${jobs.length} job(s) in flight`,
|
||||
examined: 1 + (opts.request ? 1 : 0) + jobs.length,
|
||||
findings,
|
||||
notes: [
|
||||
`${stuck.length} stuck job(s) of ${jobs.filter(j => j.state === 'claimed').length} claimed`,
|
||||
'This library gates provisioning; it does not perform it — the worker already exists.',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/** Render a self-service assessment for a terminal, CI log or review. */
|
||||
export function formatService(service: ServiceState, opts: AssessOptions & FormatOptions): string {
|
||||
return renderReport(assessService(service, opts), opts)
|
||||
}
|
||||
|
||||
export { DEFAULT_MAX_CLAIM, fmt as formatBytes }
|
||||
Reference in New Issue
Block a user