From 201e09aaa3780c7b921f2baa48242aceef80a04e Mon Sep 17 00:00:00 2001 From: Paul Hitt Date: Mon, 31 Aug 2026 14:23:46 -0400 Subject: [PATCH] A live suite cannot be pointed at production any more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 39 accounts were created in the production database in half an hour on 2026-08-31, and 55 of the 86 accounts on that relay are now test accounts. The launch signup metric became unusable: a real person signing up was invisible inside the burst. Every one of those suites is correctly written. They are opt-in, and their own headers say to point them at the disposable fleet and not at production. That was the entire protection, and a comment cannot refuse anything. `LiveTarget.refuseProduction` is checked by every live suite before it does anything, and throws rather than skips. "This run was pointed at production" is not an unmet precondition — it is a mistake that has already done damage once, and a skip is how it goes unnoticed the next time. 🔴 The unit test found a real bug in the guard before it shipped. The disposable fleet lives on the SAME DOMAIN as production, so denying `flamenet.io` and its subdomains refused `relay1.flamenet.io` — the exact place these suites are meant to point. The guard would have blocked every legitimate run while appearing to block only production. The fleet is now named explicitly and checked first, which keeps the bare domain in the deny list so a production host added later is refused by default. ⚠️ A deny list rather than an allow list, deliberately. An allow list would also refuse a developer's localhost or a self-hoster's own relay, which are normal things to test against, and a guard that gets in the way is one somebody turns off. Proven both directions rather than read: FN_LIVE_FLEET=https://flamenetmessenger.com -> IllegalStateException: refusing to run a live suite against flamenetmessenger.com FN_LIVE_FLEET unset -> tests=1 skipped=1 failures=0 The guard's own test runs in every pipeline, unlike the suites it protects — those skip without FN_LIVE_FLEET, so a rule exercised only there would never be checked at all. --- .../flamenet/client/LiveConversationTest.kt | 6 +- .../io/flamenet/client/LiveKeyLogTest.kt | 6 +- .../io/flamenet/client/LiveRelayTest.kt | 6 +- .../flamenet/client/LiveSealedSenderTest.kt | 6 +- .../io/flamenet/client/LiveShapesTest.kt | 6 +- .../io/flamenet/client/LiveSignUpTest.kt | 6 +- .../kotlin/io/flamenet/client/LiveTarget.kt | 80 +++++++++++++++++++ .../io/flamenet/client/LiveTargetTest.kt | 60 ++++++++++++++ .../io/flamenet/client/LiveTwoProductsTest.kt | 6 +- 9 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 client/src/test/kotlin/io/flamenet/client/LiveTarget.kt create mode 100644 client/src/test/kotlin/io/flamenet/client/LiveTargetTest.kt diff --git a/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt index f7dbd28..52f5619 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt @@ -34,8 +34,10 @@ class LiveConversationTest { * this way; these files did not, so 6 tests here reported green on every * pipeline while running nothing at all. */ - private fun needFnLiveFleet() = assumeTrue( - fleet != null, "set FN_LIVE_FLEET to run this against a relay") + private fun needFnLiveFleet() { + LiveTarget.refuseProduction(fleet) + assumeTrue(fleet != null, "set FN_LIVE_FLEET to run this against a relay") + } /** A signed-up account with a registered device, ready to talk. */ private class Person( diff --git a/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt index 1642253..03dd1b6 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt @@ -37,8 +37,10 @@ class LiveKeyLogTest { * this way; these files did not, so 7 tests here reported green on every * pipeline while running nothing at all. */ - private fun needFnLiveFleet() = assumeTrue( - base != null, "set FN_LIVE_FLEET to run this against a relay") + private fun needFnLiveFleet() { + LiveTarget.refuseProduction(base) + assumeTrue(base != null, "set FN_LIVE_FLEET to run this against a relay") + } private class MapStorage : Vault.Storage { val records = LinkedHashMap() diff --git a/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt index 630c191..1a603b7 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt @@ -33,8 +33,10 @@ class LiveRelayTest { * this way; these files did not, so 2 tests here reported green on every * pipeline while running nothing at all. */ - private fun needFnLiveRelay() = assumeTrue( - relay != null, "set FN_LIVE_RELAY to run this against a relay") + private fun needFnLiveRelay() { + LiveTarget.refuseProduction(relay) + assumeTrue(relay != null, "set FN_LIVE_RELAY to run this against a relay") + } @Test fun `the deployed relay serves the routes this client asks for`() { diff --git a/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt index a20c245..88eceae 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt @@ -36,8 +36,10 @@ class LiveSealedSenderTest { * this way; these files did not, so 2 tests here reported green on every * pipeline while running nothing at all. */ - private fun needFnLiveFleet() = assumeTrue( - base != null, "set FN_LIVE_FLEET to run this against a relay") + private fun needFnLiveFleet() { + LiveTarget.refuseProduction(base) + assumeTrue(base != null, "set FN_LIVE_FLEET to run this against a relay") + } private class MapStorage : Vault.Storage { val records = LinkedHashMap() diff --git a/client/src/test/kotlin/io/flamenet/client/LiveShapesTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveShapesTest.kt index bdc6581..75bde53 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveShapesTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveShapesTest.kt @@ -44,8 +44,10 @@ class LiveShapesTest { * of those is indistinguishable from a suite that works. `assumeTrue` * reports SKIPPED, which is the truth. */ - private fun needFleet() = assumeTrue( - System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET to run against the fleet") + private fun needFleet() { + LiveTarget.refuseProduction(System.getenv("FN_LIVE_FLEET")) + assumeTrue(System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET to run against the fleet") + } private fun needManaged() = assumeTrue( System.getenv("FN_LIVE_MANAGED") == "1", diff --git a/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt index c0d4889..cfd13a9 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt @@ -31,8 +31,10 @@ class LiveSignUpTest { * this way; these files did not, so 1 tests here reported green on every * pipeline while running nothing at all. */ - private fun needFnLiveFleet() = assumeTrue( - relay != null, "set FN_LIVE_FLEET to run this against a relay") + private fun needFnLiveFleet() { + LiveTarget.refuseProduction(relay) + assumeTrue(relay != null, "set FN_LIVE_FLEET to run this against a relay") + } @Test fun `a new account can register a device and publish prekeys`() { diff --git a/client/src/test/kotlin/io/flamenet/client/LiveTarget.kt b/client/src/test/kotlin/io/flamenet/client/LiveTarget.kt new file mode 100644 index 0000000..19f521e --- /dev/null +++ b/client/src/test/kotlin/io/flamenet/client/LiveTarget.kt @@ -0,0 +1,80 @@ +package io.flamenet.client + +import java.net.URI + +/** + * Where a live suite is allowed to point. + * + * 🔴 **On 2026-08-31 a thirty-minute test round created 39 accounts in the + * PRODUCTION database.** They are still there: `jvmt…`, `kt…`, `sl…`, `live…`, + * 55 of 86 accounts on the relay, which made the launch signup metric unusable + * — a real person signing up was invisible inside the burst. + * + * Every one of those suites is correctly written. They are opt-in, and their + * own headers say to point them at the disposable fleet and not at production. + * That was the whole protection, and documentation is not a protection: it + * cannot refuse anything. Something ran them against production regardless, and + * nothing at any layer objected. + * + * ⚠️ **A deny list, not an allow list**, and that is a deliberate trade. An + * allow list would be stricter, and would also refuse a developer's own + * throwaway relay on localhost or a self-hosted instance — which is a normal + * thing to test against and would push people to disable the check. Naming the + * hosts that must never receive test traffic keeps the guard narrow enough to + * survive. + * + * 🔴 A refusal here is an ERROR, never a skip. "This run was pointed at + * production" is not a precondition that happens to be unmet; it is a mistake + * that has already done damage on one occasion, and a skip is how it would go + * unnoticed on the next. + */ +object LiveTarget { + + /** + * 🔴 **The disposable fleet lives on the same domain as production**, so it + * is named explicitly and checked FIRST. Without this, denying + * `flamenet.io` and its subdomains refuses `relay1.flamenet.io` — the exact + * place these suites are supposed to point — and the guard blocks every + * legitimate run while appearing to block only production. A test caught + * that before it shipped, which is the argument for testing a guard rather + * than reading it. + */ + private val DISPOSABLE = listOf( + "relay1.flamenet.io", + "relay2.flamenet.io", + "relay3.flamenet.io", + ) + + /** + * Hosts that serve real accounts and real people's messages. + * + * ⚠️ The bare domain is in here on purpose: a production host added later + * is denied by default rather than needing somebody to remember this file. + * That is only safe because [DISPOSABLE] names the exceptions above. + */ + private val PRODUCTION = listOf( + "flamenetmessenger.com", + "relay.flamenet.io", + "accounts.flamenet.io", + "flamenet.io", + ) + + /** + * Throw if [url] names production. Null is fine — that is "not configured", + * which the caller turns into an announced skip. + */ + fun refuseProduction(url: String?) { + val host = url?.let { runCatching { URI(it).host }.getOrNull() } + ?.lowercase() + ?: return + if (DISPOSABLE.any { host == it }) return + // `endsWith(".$it")` so a subdomain cannot slip past a bare-host match. + if (PRODUCTION.any { host == it || host.endsWith(".$it") }) { + error( + "refusing to run a live suite against $host — that is production. " + + "These tests create real accounts. Point FN_LIVE_FLEET at the " + + "disposable fleet (relay1.flamenet.io) or a throwaway relay.", + ) + } + } +} diff --git a/client/src/test/kotlin/io/flamenet/client/LiveTargetTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveTargetTest.kt new file mode 100644 index 0000000..20980e1 --- /dev/null +++ b/client/src/test/kotlin/io/flamenet/client/LiveTargetTest.kt @@ -0,0 +1,60 @@ +package io.flamenet.client + +import kotlin.test.Test +import kotlin.test.assertFailsWith + +/** + * The guard that stops a live suite creating accounts on production. + * + * 🔴 This is the test that runs in every pipeline, unlike the live suites it + * protects — those skip without `FN_LIVE_FLEET`, so if the rule itself were + * only exercised there it would never be checked at all. + */ +class LiveTargetTest { + + @Test + fun `production hosts are refused`() { + for (url in listOf( + "https://flamenetmessenger.com", + "https://relay.flamenet.io", + "https://accounts.flamenet.io", + "https://flamenet.io", + // ⚠️ Subdomains too. A deny list matching only the bare host would + // wave through anything in front of it. + "https://api.flamenetmessenger.com", + // Scheme and port must not matter — the host is the whole question. + "http://relay.flamenet.io:8080/e2e", + )) { + assertFailsWith("$url was allowed") { + LiveTarget.refuseProduction(url) + } + } + } + + @Test + fun `the disposable fleet and local relays are allowed`() { + for (url in listOf( + "https://relay1.flamenet.io", + "https://relay2.flamenet.io", + "https://relay3.flamenet.io", + "http://localhost:8080", + "http://127.0.0.1:18973", + // A self-hoster's own instance. Refusing this would push people to + // disable the check, which is worse than not having it. + "https://relay.example.org", + )) { + LiveTarget.refuseProduction(url) + } + } + + /** + * ⚠️ Null is "not configured", which the caller turns into an announced + * skip. It must not throw, or an unconfigured machine cannot run the suite + * at all. + */ + @Test + fun `an unset target is not an error`() { + LiveTarget.refuseProduction(null) + LiveTarget.refuseProduction("not a url at all") + } +} diff --git a/client/src/test/kotlin/io/flamenet/client/LiveTwoProductsTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveTwoProductsTest.kt index 52fb019..0ddd7c6 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveTwoProductsTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveTwoProductsTest.kt @@ -30,8 +30,10 @@ class LiveTwoProductsTest { System.getenv("FN_LIVE_MANAGED") == "1", "set FN_LIVE_MANAGED=1 — this creates a real account") - private fun needFleet() = assumeTrue( - System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET") + private fun needFleet() { + LiveTarget.refuseProduction(System.getenv("FN_LIVE_FLEET")) + assumeTrue(System.getenv("FN_LIVE_FLEET") != null, "set FN_LIVE_FLEET") + } /** * ⚠️ **The Messenger-only case, which is the one that matters.** A relay -- 2.54.0