A live suite cannot be pointed at production any more #33

Merged
paulhitt merged 1 commits from 9-refuse-production-target into main 2026-08-31 18:47:50 +00:00
9 changed files with 168 additions and 14 deletions
@@ -34,8 +34,10 @@ class LiveConversationTest {
* this way; these files did not, so 6 tests here reported green on every
* pipeline while running nothing at all.
*/
private fun needFnLiveFleet() = assumeTrue(
fleet != null, "set FN_LIVE_FLEET to run this against a relay")
private fun needFnLiveFleet() {
LiveTarget.refuseProduction(fleet)
assumeTrue(fleet != null, "set FN_LIVE_FLEET to run this against a relay")
}
/** A signed-up account with a registered device, ready to talk. */
private class Person(
@@ -37,8 +37,10 @@ class LiveKeyLogTest {
* this way; these files did not, so 7 tests here reported green on every
* pipeline while running nothing at all.
*/
private fun needFnLiveFleet() = assumeTrue(
base != null, "set FN_LIVE_FLEET to run this against a relay")
private fun needFnLiveFleet() {
LiveTarget.refuseProduction(base)
assumeTrue(base != null, "set FN_LIVE_FLEET to run this against a relay")
}
private class MapStorage : Vault.Storage {
val records = LinkedHashMap<String, ByteArray>()
@@ -33,8 +33,10 @@ class LiveRelayTest {
* this way; these files did not, so 2 tests here reported green on every
* pipeline while running nothing at all.
*/
private fun needFnLiveRelay() = assumeTrue(
relay != null, "set FN_LIVE_RELAY to run this against a relay")
private fun needFnLiveRelay() {
LiveTarget.refuseProduction(relay)
assumeTrue(relay != null, "set FN_LIVE_RELAY to run this against a relay")
}
@Test
fun `the deployed relay serves the routes this client asks for`() {
@@ -36,8 +36,10 @@ class LiveSealedSenderTest {
* this way; these files did not, so 2 tests here reported green on every
* pipeline while running nothing at all.
*/
private fun needFnLiveFleet() = assumeTrue(
base != null, "set FN_LIVE_FLEET to run this against a relay")
private fun needFnLiveFleet() {
LiveTarget.refuseProduction(base)
assumeTrue(base != null, "set FN_LIVE_FLEET to run this against a relay")
}
private class MapStorage : Vault.Storage {
val records = LinkedHashMap<String, ByteArray>()
@@ -44,8 +44,10 @@ class LiveShapesTest {
* of those is indistinguishable from a suite that works. `assumeTrue`
* reports SKIPPED, which is the truth.
*/
private fun needFleet() = assumeTrue(
System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET to run against the fleet")
private fun needFleet() {
LiveTarget.refuseProduction(System.getenv("FN_LIVE_FLEET"))
assumeTrue(System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET to run against the fleet")
}
private fun needManaged() = assumeTrue(
System.getenv("FN_LIVE_MANAGED") == "1",
@@ -31,8 +31,10 @@ class LiveSignUpTest {
* this way; these files did not, so 1 tests here reported green on every
* pipeline while running nothing at all.
*/
private fun needFnLiveFleet() = assumeTrue(
relay != null, "set FN_LIVE_FLEET to run this against a relay")
private fun needFnLiveFleet() {
LiveTarget.refuseProduction(relay)
assumeTrue(relay != null, "set FN_LIVE_FLEET to run this against a relay")
}
@Test
fun `a new account can register a device and publish prekeys`() {
@@ -0,0 +1,80 @@
package io.flamenet.client
import java.net.URI
/**
* Where a live suite is allowed to point.
*
* 🔴 **On 2026-08-31 a thirty-minute test round created 39 accounts in the
* PRODUCTION database.** They are still there: `jvmt…`, `kt…`, `sl…`, `live…`,
* 55 of 86 accounts on the relay, which made the launch signup metric unusable
* — a real person signing up was invisible inside the burst.
*
* Every one of those suites is correctly written. They are opt-in, and their
* own headers say to point them at the disposable fleet and not at production.
* That was the whole protection, and documentation is not a protection: it
* cannot refuse anything. Something ran them against production regardless, and
* nothing at any layer objected.
*
* ⚠️ **A deny list, not an allow list**, and that is a deliberate trade. An
* allow list would be stricter, and would also refuse a developer's own
* throwaway relay on localhost or a self-hosted instance — which is a normal
* thing to test against and would push people to disable the check. Naming the
* hosts that must never receive test traffic keeps the guard narrow enough to
* survive.
*
* 🔴 A refusal here is an ERROR, never a skip. "This run was pointed at
* production" is not a precondition that happens to be unmet; it is a mistake
* that has already done damage on one occasion, and a skip is how it would go
* unnoticed on the next.
*/
object LiveTarget {
/**
* 🔴 **The disposable fleet lives on the same domain as production**, so it
* is named explicitly and checked FIRST. Without this, denying
* `flamenet.io` and its subdomains refuses `relay1.flamenet.io` — the exact
* place these suites are supposed to point — and the guard blocks every
* legitimate run while appearing to block only production. A test caught
* that before it shipped, which is the argument for testing a guard rather
* than reading it.
*/
private val DISPOSABLE = listOf(
"relay1.flamenet.io",
"relay2.flamenet.io",
"relay3.flamenet.io",
)
/**
* Hosts that serve real accounts and real people's messages.
*
* ⚠️ The bare domain is in here on purpose: a production host added later
* is denied by default rather than needing somebody to remember this file.
* That is only safe because [DISPOSABLE] names the exceptions above.
*/
private val PRODUCTION = listOf(
"flamenetmessenger.com",
"relay.flamenet.io",
"accounts.flamenet.io",
"flamenet.io",
)
/**
* Throw if [url] names production. Null is fine — that is "not configured",
* which the caller turns into an announced skip.
*/
fun refuseProduction(url: String?) {
val host = url?.let { runCatching { URI(it).host }.getOrNull() }
?.lowercase()
?: return
if (DISPOSABLE.any { host == it }) return
// `endsWith(".$it")` so a subdomain cannot slip past a bare-host match.
if (PRODUCTION.any { host == it || host.endsWith(".$it") }) {
error(
"refusing to run a live suite against $host — that is production. " +
"These tests create real accounts. Point FN_LIVE_FLEET at the " +
"disposable fleet (relay1.flamenet.io) or a throwaway relay.",
)
}
}
}
@@ -0,0 +1,60 @@
package io.flamenet.client
import kotlin.test.Test
import kotlin.test.assertFailsWith
/**
* The guard that stops a live suite creating accounts on production.
*
* 🔴 This is the test that runs in every pipeline, unlike the live suites it
* protects — those skip without `FN_LIVE_FLEET`, so if the rule itself were
* only exercised there it would never be checked at all.
*/
class LiveTargetTest {
@Test
fun `production hosts are refused`() {
for (url in listOf(
"https://flamenetmessenger.com",
"https://relay.flamenet.io",
"https://accounts.flamenet.io",
"https://flamenet.io",
// ⚠️ Subdomains too. A deny list matching only the bare host would
// wave through anything in front of it.
"https://api.flamenetmessenger.com",
// Scheme and port must not matter — the host is the whole question.
"http://relay.flamenet.io:8080/e2e",
)) {
assertFailsWith<IllegalStateException>("$url was allowed") {
LiveTarget.refuseProduction(url)
}
}
}
@Test
fun `the disposable fleet and local relays are allowed`() {
for (url in listOf(
"https://relay1.flamenet.io",
"https://relay2.flamenet.io",
"https://relay3.flamenet.io",
"http://localhost:8080",
"http://127.0.0.1:18973",
// A self-hoster's own instance. Refusing this would push people to
// disable the check, which is worse than not having it.
"https://relay.example.org",
)) {
LiveTarget.refuseProduction(url)
}
}
/**
* ⚠️ Null is "not configured", which the caller turns into an announced
* skip. It must not throw, or an unconfigured machine cannot run the suite
* at all.
*/
@Test
fun `an unset target is not an error`() {
LiveTarget.refuseProduction(null)
LiveTarget.refuseProduction("not a url at all")
}
}
@@ -30,8 +30,10 @@ class LiveTwoProductsTest {
System.getenv("FN_LIVE_MANAGED") == "1",
"set FN_LIVE_MANAGED=1 — this creates a real account")
private fun needFleet() = assumeTrue(
System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET")
private fun needFleet() {
LiveTarget.refuseProduction(System.getenv("FN_LIVE_FLEET"))
assumeTrue(System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET")
}
/**
* ⚠️ **The Messenger-only case, which is the one that matters.** A relay