Make the instrumented tests capable of failing

Three separate reasons these suites could never catch a regression, all fixed here. None of them are about the app; all of them are about the tests being unable to report the truth. 1. A skipped test reported as a pass. Thirteen guards were bare returns — `if (!relayReachable()) return` — so with no relay the method returned immediately and the runner recorded PASSED. Now `assumeTrue`, which reports SKIPPED, and the run says "(2 skipped)" where it used to say nothing at all. Only the guards that abandon a whole test were changed; helper loop exits like `if (found) return` are control flow and were left alone. 2. The relay was hardcoded to relay1.flamenet.io — production. Every run that did work created real accounts there. Now `-e relay <host>`, still defaulting to production so an unqualified run is unchanged. 3. Android blocks cleartext HTTP, so a debug build could not reach a relay on the developer's machine even when one was running. A debug-only network security config permits cleartext to 10.0.2.2, 127.0.0.1 and localhost, and nothing else. src/debug, so no release APK is affected. Together these took the sign-up suite from "2 skipped, reported green" to "2 run, 2 failed" — which is worth more, because it is true. What it fails on now (no compose hierarchy at launch) is the next thing to fix. ./gradlew :app:connectedDebugAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
2026-09-02 20:58:26 -04:00
parent 4396c7e8cc
commit 16e0101daf
7 changed files with 136 additions and 21 deletions
@@ -10,7 +10,9 @@ import androidx.compose.ui.test.performClick
 import androidx.compose.ui.test.performTextClearance
 import androidx.compose.ui.test.performTextInput
 import androidx.compose.ui.test.printToString
+import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
@@ -36,7 +38,25 @@ class ChatFlowTest {

    @get:Rule val compose = createAndroidComposeRule<MainActivity>()

-    private val relay = "relay1.flamenet.io"
+    /**
+     * Which relay to drive.
+     *
+     * 🔴 Defaults to the production relay, which is what this has always used —
+     * so an unqualified run still does what it did. Pass `-e relay <host>` to
+     * point it somewhere else, and a LOCAL one is what you want while
+     * developing: these tests create real accounts, and doing that on
+     * production for every run is a cost nobody chose.
+     *
+     *   ./gradlew :app:connectedDebugAndroidTest \
+     *       -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
+     */
+    private val relay: String
+        get() = InstrumentationRegistry.getArguments().getString("relay")
+            ?: "relay1.flamenet.io"
+
+    /** A local relay is plain HTTP; anything else is not. */
+    private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
+        relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
    private val peerHandle = "peerjvmone"
    private lateinit var handle: String

@@ -48,7 +68,7 @@ class ChatFlowTest {

    @Test
    fun aPersonCanAddAContactAndSendThemAMessage() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())

        signUp()

@@ -97,7 +117,7 @@ class ChatFlowTest {
     */
    @Test
    fun aReplyFromSomebodyElseAppearsInTheThread() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())

        signUp()
        compose.onNodeWithTag("add-contact").performClick()
@@ -232,7 +252,7 @@ class ChatFlowTest {
    }.getOrElse { "(could not read the screen: $it)" }

    private fun relayReachable(): Boolean = runCatching {
-        (java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
+        (java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
            .apply { connectTimeout = 8000; readTimeout = 8000 }
            .responseCode in 200..299
    }.getOrDefault(false)
@@ -10,7 +10,9 @@ import androidx.compose.ui.test.performClick
 import androidx.compose.ui.test.performTextClearance
 import androidx.compose.ui.test.performTextInput
 import androidx.compose.ui.test.printToString
+import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
@@ -35,7 +37,25 @@ class CrudSurfaceTest {

    @get:Rule val compose = createAndroidComposeRule<MainActivity>()

-    private val relay = "relay1.flamenet.io"
+    /**
+     * Which relay to drive.
+     *
+     * 🔴 Defaults to the production relay, which is what this has always used —
+     * so an unqualified run still does what it did. Pass `-e relay <host>` to
+     * point it somewhere else, and a LOCAL one is what you want while
+     * developing: these tests create real accounts, and doing that on
+     * production for every run is a cost nobody chose.
+     *
+     *   ./gradlew :app:connectedDebugAndroidTest \
+     *       -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
+     */
+    private val relay: String
+        get() = InstrumentationRegistry.getArguments().getString("relay")
+            ?: "relay1.flamenet.io"
+
+    /** A local relay is plain HTTP; anything else is not. */
+    private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
+        relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
    private val peerHandle = "peerjvmone"
    private lateinit var handle: String

@@ -74,7 +94,7 @@ class CrudSurfaceTest {

    @Test
    fun everyContactActionIsReachable() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUp()

        // Presence is a claim about you, so you can set it.
@@ -110,7 +130,7 @@ class CrudSurfaceTest {

    @Test
    fun theOtherScreensAreReachableAndComplete() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUp()

        // Stories are reachable from the contact list, and say what they are
@@ -143,7 +163,7 @@ class CrudSurfaceTest {

    @Test
    fun aThreadCanSendTextAndPhotos() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUp()
        compose.onNodeWithTag("add-contact").performClick()
        compose.onNodeWithTag("new-contact").performTextInput(peerHandle)
@@ -282,7 +302,7 @@ class CrudSurfaceTest {
    }.getOrElse { "(could not read the screen: $it)" }

    private fun relayReachable(): Boolean = runCatching {
-        (java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
+        (java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
            .apply { connectTimeout = 8000; readTimeout = 8000 }
            .responseCode in 200..299
    }.getOrDefault(false)
@@ -8,7 +8,9 @@ import androidx.compose.ui.test.performClick
 import androidx.compose.ui.test.performTextClearance
 import androidx.compose.ui.test.performTextInput
 import androidx.compose.ui.test.printToString
+import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assume.assumeTrue
 import org.junit.Assert.assertTrue
 import org.junit.Before
 import org.junit.Rule
@@ -40,7 +42,25 @@ class FeatureUseTest {

    @get:Rule val compose = createAndroidComposeRule<MainActivity>()

-    private val relay = "relay1.flamenet.io"
+    /**
+     * Which relay to drive.
+     *
+     * 🔴 Defaults to the production relay, which is what this has always used —
+     * so an unqualified run still does what it did. Pass `-e relay <host>` to
+     * point it somewhere else, and a LOCAL one is what you want while
+     * developing: these tests create real accounts, and doing that on
+     * production for every run is a cost nobody chose.
+     *
+     *   ./gradlew :app:connectedDebugAndroidTest \
+     *       -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
+     */
+    private val relay: String
+        get() = InstrumentationRegistry.getArguments().getString("relay")
+            ?: "relay1.flamenet.io"
+
+    /** A local relay is plain HTTP; anything else is not. */
+    private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
+        relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
    private lateinit var handle: String

    @Before
@@ -59,7 +79,7 @@ class FeatureUseTest {
     */
    @Test
    fun aVoiceNoteRecordsAndCanBeDiscarded() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUpAndOpenAThread()

        compose.onNodeWithTag("record-voice").performClick()
@@ -88,7 +108,7 @@ class FeatureUseTest {
     */
    @Test
    fun aGroupIsCreatedAndListed() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUpAndAddPeer()

        compose.onNodeWithTag("new-group").performClick()
@@ -120,7 +140,7 @@ class FeatureUseTest {
     */
    @Test
    fun theStorySurfaceOpensAndExplainsItself() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUp()

        compose.onNodeWithTag("stories").assertExists()
@@ -143,7 +163,7 @@ class FeatureUseTest {
     */
    @Test
    fun placingACallRaisesTheBannerAndHangingUpClearsIt() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        signUpAndOpenAThread()

        compose.onNodeWithTag("start-call").performClick()
@@ -256,7 +276,7 @@ class FeatureUseTest {
    }.getOrElse { "(could not read the screen: $it)" }

    private fun relayReachable(): Boolean = runCatching {
-        (java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
+        (java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
            .apply { connectTimeout = 8000; readTimeout = 8000 }
            .responseCode in 200..299
    }.getOrDefault(false)
@@ -11,6 +11,7 @@ import androidx.compose.ui.test.performTextInput
 import androidx.compose.ui.test.printToString
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
@@ -75,13 +76,13 @@ class LiveSignUpFleetTest {

    @Test
    fun signUpOnTheManagedRelay() {
-        if (args.getString("live") != "true") return
+        assumeTrue("pass -e live true to run this - it creates real accounts", args.getString("live") == "true")
        signUpLive(managed, "m")
    }

    @Test
    fun signUpOnASharedSelfHostedRelay() {
-        if (args.getString("live") != "true") return
+        assumeTrue("pass -e live true to run this - it creates real accounts", args.getString("live") == "true")
        signUpLive(sharedSelfHosted, "s")
    }

@@ -9,7 +9,9 @@ import androidx.compose.ui.test.printToString
 import androidx.compose.ui.test.performClick
 import androidx.compose.ui.test.performTextClearance
 import androidx.compose.ui.test.performTextInput
+import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
@@ -36,7 +38,25 @@ class SignUpFlowTest {

    @get:Rule val compose = createAndroidComposeRule<MainActivity>()

-    private val relay = "relay1.flamenet.io"
+    /**
+     * Which relay to drive.
+     *
+     * 🔴 Defaults to the production relay, which is what this has always used —
+     * so an unqualified run still does what it did. Pass `-e relay <host>` to
+     * point it somewhere else, and a LOCAL one is what you want while
+     * developing: these tests create real accounts, and doing that on
+     * production for every run is a cost nobody chose.
+     *
+     *   ./gradlew :app:connectedDebugAndroidTest \
+     *       -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
+     */
+    private val relay: String
+        get() = InstrumentationRegistry.getArguments().getString("relay")
+            ?: "relay1.flamenet.io"
+
+    /** A local relay is plain HTTP; anything else is not. */
+    private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
+        relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
    private lateinit var handle: String

    @Before
@@ -155,14 +175,14 @@ class SignUpFlowTest {
    }.getOrElse { "(could not read the screen: $it)" }

    private fun relayReachable(): Boolean = runCatching {
-        (java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
+        (java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
            .apply { connectTimeout = 8000; readTimeout = 8000 }
            .responseCode in 200..299
    }.getOrDefault(false)

    @Test
    fun aPersonCanCreateAnAccountOnASelfHostedRelay() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())

        compose.onNodeWithTag("relay").performTextClearance()
        compose.onNodeWithTag("relay").performTextInput(relay)
@@ -191,7 +211,7 @@ class SignUpFlowTest {
     */
    @Test
    fun signingOutReturnsToSignIn() {
-        if (!relayReachable()) return
+        assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
        aPersonCanCreateAnAccountOnASelfHostedRelay()

        signOutIfSignedIn()
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+  Debug-only overlay. Points debug builds at a network security config that
+  permits cleartext to the loopback addresses, so a local relay is reachable
+  for end-to-end tests. Release builds never see this file.
+-->
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          xmlns:tools="http://schemas.android.com/tools">
+    <application
+        android:networkSecurityConfig="@xml/network_security_config"
+        tools:replace="android:networkSecurityConfig" />
+</manifest>
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+  🔴 DEBUG BUILDS ONLY. This file lives in src/debug and is not part of a
+  release APK, so nothing shipped to anybody is loosened by it.
+
+  Android blocks cleartext HTTP by default, which is right — but it also
+  means a debug build cannot talk to a relay running on the developer's own
+  machine, and so every end-to-end test either points at PRODUCTION or skips.
+  Both of those are worse than a scoped exception: one creates real accounts
+  for every test run, the other reports a pass for a test that never ran.
+
+  Scoped to the loopback addresses only. 10.0.2.2 is how the emulator reaches
+  the host; 127.0.0.1 and localhost are the device itself. Nothing else is
+  permitted, so this cannot quietly enable cleartext to a real server.
+-->
+<network-security-config>
+    <domain-config cleartextTrafficPermitted="true">
+        <domain includeSubdomains="false">10.0.2.2</domain>
+        <domain includeSubdomains="false">127.0.0.1</domain>
+        <domain includeSubdomains="false">localhost</domain>
+    </domain-config>
+</network-security-config>