Make the instrumented tests capable of failing #61

Merged
paulhitt merged 2 commits from fix/a-skipped-instrumented-test-must-not-pass into main 2026-09-04 20:49:05 +00:00
9 changed files with 253 additions and 27 deletions
+3
View File
@@ -225,6 +225,9 @@ dependencies {
androidTestImplementation("androidx.test.ext:junit:1.2.1")
androidTestImplementation("androidx.test:runner:1.6.2")
// GrantPermissionRule: permissions have to be granted before the
// activity launches, or its permission dialog steals focus from Compose.
androidTestImplementation("androidx.test:rules:1.6.1")
androidTestImplementation(platform("androidx.compose:compose-bom:2024.12.01"))
androidTestImplementation("androidx.compose.ui:ui-test-junit4")
debugImplementation("androidx.compose.ui:ui-test-manifest")
@@ -10,7 +10,10 @@ import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.compose.ui.test.printToString
import androidx.test.rule.GrantPermissionRule
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Assume.assumeTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
@@ -34,9 +37,44 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class ChatFlowTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
private val relay = "relay1.flamenet.io"
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
/**
* Which relay to drive.
*
* 🔴 Defaults to the production relay, which is what this has always used —
* so an unqualified run still does what it did. Pass `-e relay <host>` to
* point it somewhere else, and a LOCAL one is what you want while
* developing: these tests create real accounts, and doing that on
* production for every run is a cost nobody chose.
*
* ./gradlew :app:connectedDebugAndroidTest \
* -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
*/
private val relay: String
get() = InstrumentationRegistry.getArguments().getString("relay")
?: "relay1.flamenet.io"
/** A local relay is plain HTTP; anything else is not. */
private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
private val peerHandle = "peerjvmone"
private lateinit var handle: String
@@ -48,7 +86,7 @@ class ChatFlowTest {
@Test
fun aPersonCanAddAContactAndSendThemAMessage() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
@@ -97,7 +135,7 @@ class ChatFlowTest {
*/
@Test
fun aReplyFromSomebodyElseAppearsInTheThread() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
compose.onNodeWithTag("add-contact").performClick()
@@ -232,7 +270,7 @@ class ChatFlowTest {
}.getOrElse { "(could not read the screen: $it)" }
private fun relayReachable(): Boolean = runCatching {
(java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
(java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
.apply { connectTimeout = 8000; readTimeout = 8000 }
.responseCode in 200..299
}.getOrDefault(false)
@@ -10,6 +10,7 @@ import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.compose.ui.test.printToString
import androidx.test.rule.GrantPermissionRule
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Rule
@@ -33,7 +34,24 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class CrossPlatformTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
private val args get() = InstrumentationRegistry.getArguments()
private val relay: String get() = args.getString("relay") ?: "relay1.flamenet.io"
@@ -10,7 +10,10 @@ import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.compose.ui.test.printToString
import androidx.test.rule.GrantPermissionRule
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Assume.assumeTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
@@ -33,9 +36,44 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class CrudSurfaceTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
private val relay = "relay1.flamenet.io"
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
/**
* Which relay to drive.
*
* 🔴 Defaults to the production relay, which is what this has always used —
* so an unqualified run still does what it did. Pass `-e relay <host>` to
* point it somewhere else, and a LOCAL one is what you want while
* developing: these tests create real accounts, and doing that on
* production for every run is a cost nobody chose.
*
* ./gradlew :app:connectedDebugAndroidTest \
* -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
*/
private val relay: String
get() = InstrumentationRegistry.getArguments().getString("relay")
?: "relay1.flamenet.io"
/** A local relay is plain HTTP; anything else is not. */
private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
private val peerHandle = "peerjvmone"
private lateinit var handle: String
@@ -74,7 +112,7 @@ class CrudSurfaceTest {
@Test
fun everyContactActionIsReachable() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
// Presence is a claim about you, so you can set it.
@@ -110,7 +148,7 @@ class CrudSurfaceTest {
@Test
fun theOtherScreensAreReachableAndComplete() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
// Stories are reachable from the contact list, and say what they are
@@ -143,7 +181,7 @@ class CrudSurfaceTest {
@Test
fun aThreadCanSendTextAndPhotos() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
compose.onNodeWithTag("add-contact").performClick()
compose.onNodeWithTag("new-contact").performTextInput(peerHandle)
@@ -282,7 +320,7 @@ class CrudSurfaceTest {
}.getOrElse { "(could not read the screen: $it)" }
private fun relayReachable(): Boolean = runCatching {
(java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
(java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
.apply { connectTimeout = 8000; readTimeout = 8000 }
.responseCode in 200..299
}.getOrDefault(false)
@@ -8,7 +8,10 @@ import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.compose.ui.test.printToString
import androidx.test.rule.GrantPermissionRule
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Assume.assumeTrue
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Rule
@@ -38,9 +41,44 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class FeatureUseTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
private val relay = "relay1.flamenet.io"
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
/**
* Which relay to drive.
*
* 🔴 Defaults to the production relay, which is what this has always used —
* so an unqualified run still does what it did. Pass `-e relay <host>` to
* point it somewhere else, and a LOCAL one is what you want while
* developing: these tests create real accounts, and doing that on
* production for every run is a cost nobody chose.
*
* ./gradlew :app:connectedDebugAndroidTest \
* -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
*/
private val relay: String
get() = InstrumentationRegistry.getArguments().getString("relay")
?: "relay1.flamenet.io"
/** A local relay is plain HTTP; anything else is not. */
private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
private lateinit var handle: String
@Before
@@ -59,7 +97,7 @@ class FeatureUseTest {
*/
@Test
fun aVoiceNoteRecordsAndCanBeDiscarded() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUpAndOpenAThread()
compose.onNodeWithTag("record-voice").performClick()
@@ -88,7 +126,7 @@ class FeatureUseTest {
*/
@Test
fun aGroupIsCreatedAndListed() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUpAndAddPeer()
compose.onNodeWithTag("new-group").performClick()
@@ -120,7 +158,7 @@ class FeatureUseTest {
*/
@Test
fun theStorySurfaceOpensAndExplainsItself() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUp()
compose.onNodeWithTag("stories").assertExists()
@@ -143,7 +181,7 @@ class FeatureUseTest {
*/
@Test
fun placingACallRaisesTheBannerAndHangingUpClearsIt() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
signUpAndOpenAThread()
compose.onNodeWithTag("start-call").performClick()
@@ -256,7 +294,7 @@ class FeatureUseTest {
}.getOrElse { "(could not read the screen: $it)" }
private fun relayReachable(): Boolean = runCatching {
(java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
(java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
.apply { connectTimeout = 8000; readTimeout = 8000 }
.responseCode in 200..299
}.getOrDefault(false)
@@ -9,8 +9,10 @@ import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.compose.ui.test.printToString
import androidx.test.rule.GrantPermissionRule
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Assume.assumeTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
@@ -49,7 +51,24 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class LiveSignUpFleetTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
private val args get() = InstrumentationRegistry.getArguments()
@@ -75,13 +94,13 @@ class LiveSignUpFleetTest {
@Test
fun signUpOnTheManagedRelay() {
if (args.getString("live") != "true") return
assumeTrue("pass -e live true to run this - it creates real accounts", args.getString("live") == "true")
signUpLive(managed, "m")
}
@Test
fun signUpOnASharedSelfHostedRelay() {
if (args.getString("live") != "true") return
assumeTrue("pass -e live true to run this - it creates real accounts", args.getString("live") == "true")
signUpLive(sharedSelfHosted, "s")
}
@@ -9,7 +9,10 @@ import androidx.compose.ui.test.printToString
import androidx.compose.ui.test.performClick
import androidx.compose.ui.test.performTextClearance
import androidx.compose.ui.test.performTextInput
import androidx.test.rule.GrantPermissionRule
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Assume.assumeTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
@@ -34,9 +37,44 @@ import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class SignUpFlowTest {
@get:Rule val compose = createAndroidComposeRule<MainActivity>()
/**
* 🔴 Granted BEFORE the activity launches, and that ordering is the point.
*
* `MainActivity.onCreate` asks for POST_NOTIFICATIONS before it calls
* `setContent`. The permission dialog is a separate activity and takes
* focus, so the Compose rule attaches to a window that has no hierarchy
* yet and every test dies with "No compose hierarchies found in the app".
*
* `order = 0` puts this outside the Compose rule, so the grant happens
* first and no dialog ever appears.
*/
@get:Rule(order = 0)
val permissions: GrantPermissionRule = GrantPermissionRule.grant(
android.Manifest.permission.POST_NOTIFICATIONS,
)
private val relay = "relay1.flamenet.io"
@get:Rule(order = 1)
val compose = createAndroidComposeRule<MainActivity>()
/**
* Which relay to drive.
*
* 🔴 Defaults to the production relay, which is what this has always used —
* so an unqualified run still does what it did. Pass `-e relay <host>` to
* point it somewhere else, and a LOCAL one is what you want while
* developing: these tests create real accounts, and doing that on
* production for every run is a cost nobody chose.
*
* ./gradlew :app:connectedDebugAndroidTest \
* -Pandroid.testInstrumentationRunnerArguments.relay=10.0.2.2:18810
*/
private val relay: String
get() = InstrumentationRegistry.getArguments().getString("relay")
?: "relay1.flamenet.io"
/** A local relay is plain HTTP; anything else is not. */
private val relayScheme: String get() = if (relay.startsWith("10.0.2.2") ||
relay.startsWith("127.0.0.1") || relay.startsWith("localhost")) "http" else "https"
private lateinit var handle: String
@Before
@@ -155,14 +193,14 @@ class SignUpFlowTest {
}.getOrElse { "(could not read the screen: $it)" }
private fun relayReachable(): Boolean = runCatching {
(java.net.URL("https://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
(java.net.URL("$relayScheme://$relay/e2e/log/head").openConnection() as java.net.HttpURLConnection)
.apply { connectTimeout = 8000; readTimeout = 8000 }
.responseCode in 200..299
}.getOrDefault(false)
@Test
fun aPersonCanCreateAnAccountOnASelfHostedRelay() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
compose.onNodeWithTag("relay").performTextClearance()
compose.onNodeWithTag("relay").performTextInput(relay)
@@ -191,7 +229,7 @@ class SignUpFlowTest {
*/
@Test
fun signingOutReturnsToSignIn() {
if (!relayReachable()) return
assumeTrue("the relay is unreachable - skipped, and a skip is not a pass", relayReachable())
aPersonCanCreateAnAccountOnASelfHostedRelay()
signOutIfSignedIn()
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Debug-only overlay. Points debug builds at a network security config that
permits cleartext to the loopback addresses, so a local relay is reachable
for end-to-end tests. Release builds never see this file.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
<application
android:networkSecurityConfig="@xml/network_security_config"
tools:replace="android:networkSecurityConfig" />
</manifest>
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
🔴 DEBUG BUILDS ONLY. This file lives in src/debug and is not part of a
release APK, so nothing shipped to anybody is loosened by it.
Android blocks cleartext HTTP by default, which is right — but it also
means a debug build cannot talk to a relay running on the developer's own
machine, and so every end-to-end test either points at PRODUCTION or skips.
Both of those are worse than a scoped exception: one creates real accounts
for every test run, the other reports a pass for a test that never ran.
Scoped to the loopback addresses only. 10.0.2.2 is how the emulator reaches
the host; 127.0.0.1 and localhost are the device itself. Nothing else is
permitted, so this cannot quietly enable cleartext to a real server.
-->
<network-security-config>
<domain-config cleartextTrafficPermitted="true">
<domain includeSubdomains="false">10.0.2.2</domain>
<domain includeSubdomains="false">127.0.0.1</domain>
<domain includeSubdomains="false">localhost</domain>
</domain-config>
</network-security-config>