From 5242a506b3635e49c7a73d4192c14008e34b8a0e Mon Sep 17 00:00:00 2001 From: Paul Hitt Date: Mon, 31 Aug 2026 13:54:39 -0400 Subject: [PATCH] Eighteen tests reported PASSED while running nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `val url = base ?: return` from an unmet precondition makes a test that did nothing report as a pass. FN_LIVE_FLEET is never set in CI, so eighteen tests across five files did this on every pipeline: LiveKeyLogTest 7, LiveConversationTest 6, LiveRelayTest 2, LiveSealedSenderTest 2, LiveSignUpTest 1. 🔴 This repository had already written the rule down. LiveShapesTest uses `assumeTrue` and says why, in its own words: a suite of tests that did nothing is indistinguishable from a suite that works. Two files followed that and five did not, so this is a consistency fix rather than a new idea. It nearly reached the specification. `LiveSealedSenderTest` was the evidence §15 would have cited for "Android sealed sender, wired end to end" — two tests, zero skipped, zero failures, on a run that did nothing. Only reading the source shows that, and a status table is exactly where nobody would. ⚠️ The suite now looks worse and is not. Client goes from "277 tests, 0 failures" to "277 tests, 28 skipped" — the 28 were always doing nothing; the number was wrong before, not now. 578 of 606 actually run. --- .../flamenet/client/LiveConversationTest.kt | 31 +++++++++++++---- .../io/flamenet/client/LiveKeyLogTest.kt | 34 +++++++++++++++---- .../io/flamenet/client/LiveRelayTest.kt | 19 +++++++++-- .../flamenet/client/LiveSealedSenderTest.kt | 19 +++++++++-- .../io/flamenet/client/LiveSignUpTest.kt | 16 ++++++++- 5 files changed, 101 insertions(+), 18 deletions(-) diff --git a/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt index af8a473..f7dbd28 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveConversationTest.kt @@ -1,5 +1,6 @@ package io.flamenet.client +import org.junit.jupiter.api.Assumptions.assumeTrue import io.flamenet.e2e.DeviceIdentity import io.flamenet.e2e.Vault import javax.crypto.KeyGenerator @@ -24,6 +25,18 @@ class LiveConversationTest { private val fleet: String? = System.getenv("FN_LIVE_FLEET") + + /** + * ⚠️ **Skips are announced, not silent.** `return` from an unmet + * precondition makes a test that did nothing report as PASSED, and a suite + * of those is indistinguishable from a suite that works. `assumeTrue` + * reports SKIPPED, which is the truth. `LiveShapesTest` has always done it + * this way; these files did not, so 6 tests here reported green on every + * pipeline while running nothing at all. + */ + private fun needFnLiveFleet() = assumeTrue( + fleet != null, "set FN_LIVE_FLEET to run this against a relay") + /** A signed-up account with a registered device, ready to talk. */ private class Person( val handle: String, @@ -88,7 +101,8 @@ class LiveConversationTest { @Test fun `two people exchange messages through a real relay`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "a") val bob = signUp(base, "b") @@ -128,7 +142,8 @@ class LiveConversationTest { */ @Test fun `polling twice does not deliver the same message twice`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "c") val bob = signUp(base, "d") @@ -146,7 +161,8 @@ class LiveConversationTest { */ @Test fun `a message reaches every device a person has`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "e") val bob = signUp(base, "f") @@ -176,7 +192,8 @@ class LiveConversationTest { */ @Test fun `the relay never sees the plaintext`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "g") val bob = signUp(base, "h") @@ -202,7 +219,8 @@ class LiveConversationTest { */ @Test fun `an image crosses end to end and the relay cannot read it`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "i") val bob = signUp(base, "j") @@ -242,7 +260,8 @@ class LiveConversationTest { */ @Test fun `a group is created, described and messaged`() { - val base = fleet ?: return + needFnLiveFleet() + val base = fleet!! val alice = signUp(base, "k") val bob = signUp(base, "l") val carol = signUp(base, "m") diff --git a/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt index c51cba9..1642253 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveKeyLogTest.kt @@ -1,5 +1,6 @@ package io.flamenet.client +import org.junit.jupiter.api.Assumptions.assumeTrue import io.flamenet.e2e.DeviceIdentity import io.flamenet.e2e.Vault import javax.crypto.KeyGenerator @@ -27,6 +28,18 @@ class LiveKeyLogTest { private val base: String? = System.getenv("FN_LIVE_FLEET") + + /** + * ⚠️ **Skips are announced, not silent.** `return` from an unmet + * precondition makes a test that did nothing report as PASSED, and a suite + * of those is indistinguishable from a suite that works. `assumeTrue` + * reports SKIPPED, which is the truth. `LiveShapesTest` has always done it + * this way; these files did not, so 7 tests here reported green on every + * pipeline while running nothing at all. + */ + private fun needFnLiveFleet() = assumeTrue( + base != null, "set FN_LIVE_FLEET to run this against a relay") + private class MapStorage : Vault.Storage { val records = LinkedHashMap() override fun read(name: String) = records[name] @@ -61,7 +74,8 @@ class LiveKeyLogTest { @Test fun `a key this relay really published verifies against its real log`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "a") val bob = Actor(url, "b") @@ -88,7 +102,8 @@ class LiveKeyLogTest { */ @Test fun `a key the relay never published is refused by the real log`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "c") val bob = Actor(url, "d") @@ -118,7 +133,8 @@ class LiveKeyLogTest { */ @Test fun `a later head reconciles with the head we pinned earlier`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "e") val bob = Actor(url, "f") @@ -147,7 +163,8 @@ class LiveKeyLogTest { */ @Test fun `a root we never saw at our pinned size is a fork`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "h") val bob = Actor(url, "i") @@ -178,7 +195,8 @@ class LiveKeyLogTest { */ @Test fun `a contact carrying the peer's address verifies a key logged under it`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "l") val bob = Actor(url, "m") @@ -228,7 +246,8 @@ class LiveKeyLogTest { */ @Test fun `a peer whose address changes is reported as an identity change`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "n") val bob = Actor(url, "o") @@ -256,7 +275,8 @@ class LiveKeyLogTest { */ @Test fun `a send refuses when enforcement is on and the key is not logged`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "j") val bob = Actor(url, "k") diff --git a/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt index ed55190..630c191 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveRelayTest.kt @@ -1,5 +1,6 @@ package io.flamenet.client +import org.junit.jupiter.api.Assumptions.assumeTrue import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -23,9 +24,22 @@ class LiveRelayTest { private val relay: String? = System.getenv("FN_LIVE_RELAY") + + /** + * ⚠️ **Skips are announced, not silent.** `return` from an unmet + * precondition makes a test that did nothing report as PASSED, and a suite + * of those is indistinguishable from a suite that works. `assumeTrue` + * reports SKIPPED, which is the truth. `LiveShapesTest` has always done it + * this way; these files did not, so 2 tests here reported green on every + * pipeline while running nothing at all. + */ + private fun needFnLiveRelay() = assumeTrue( + relay != null, "set FN_LIVE_RELAY to run this against a relay") + @Test fun `the deployed relay serves the routes this client asks for`() { - val base = relay ?: return + needFnLiveRelay() + val base = relay!! val head = RelayClient(base, username = "", appPassword = "").logHead() // `audience` is the hostname the relay signs under, and a mismatch @@ -39,7 +53,8 @@ class LiveRelayTest { @Test fun `an inclusion proof from the deployed log verifies`() { - val base = relay ?: return + needFnLiveRelay() + val base = relay!! val client = RelayClient(base, username = "", appPassword = "") val head = client.logHead() val size = head.getInt("size") diff --git a/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt index 419f56a..a20c245 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveSealedSenderTest.kt @@ -1,5 +1,6 @@ package io.flamenet.client +import org.junit.jupiter.api.Assumptions.assumeTrue import io.flamenet.e2e.DeviceIdentity import io.flamenet.e2e.SealedSender import io.flamenet.e2e.Vault @@ -26,6 +27,18 @@ class LiveSealedSenderTest { private val base: String? = System.getenv("FN_LIVE_FLEET") + + /** + * ⚠️ **Skips are announced, not silent.** `return` from an unmet + * precondition makes a test that did nothing report as PASSED, and a suite + * of those is indistinguishable from a suite that works. `assumeTrue` + * reports SKIPPED, which is the truth. `LiveShapesTest` has always done it + * this way; these files did not, so 2 tests here reported green on every + * pipeline while running nothing at all. + */ + private fun needFnLiveFleet() = assumeTrue( + base != null, "set FN_LIVE_FLEET to run this against a relay") + private class MapStorage : Vault.Storage { val records = LinkedHashMap() override fun read(name: String) = records[name] @@ -62,7 +75,8 @@ class LiveSealedSenderTest { */ @Test fun `a sealed message arrives and the relay records no sender`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "a") val bob = Actor(url, "b") @@ -114,7 +128,8 @@ class LiveSealedSenderTest { */ @Test fun `rotating the delivery key refuses submissions holding the old one`() { - val url = base ?: return + needFnLiveFleet() + val url = base!! val alice = Actor(url, "c") val bob = Actor(url, "d") diff --git a/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt b/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt index 201af81..c0d4889 100644 --- a/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt +++ b/client/src/test/kotlin/io/flamenet/client/LiveSignUpTest.kt @@ -1,5 +1,6 @@ package io.flamenet.client +import org.junit.jupiter.api.Assumptions.assumeTrue import io.flamenet.e2e.DeviceIdentity import kotlin.test.Test import kotlin.test.assertEquals @@ -21,9 +22,22 @@ class LiveSignUpTest { private val relay: String? = System.getenv("FN_LIVE_FLEET") + + /** + * ⚠️ **Skips are announced, not silent.** `return` from an unmet + * precondition makes a test that did nothing report as PASSED, and a suite + * of those is indistinguishable from a suite that works. `assumeTrue` + * reports SKIPPED, which is the truth. `LiveShapesTest` has always done it + * this way; these files did not, so 1 tests here reported green on every + * pipeline while running nothing at all. + */ + private fun needFnLiveFleet() = assumeTrue( + relay != null, "set FN_LIVE_FLEET to run this against a relay") + @Test fun `a new account can register a device and publish prekeys`() { - val base = relay ?: return + needFnLiveFleet() + val base = relay!! val handle = "jvmt" + (System.currentTimeMillis() % 100_000_000) val account = RelayAccounts(base).signUp( -- 2.54.0