Paul Hitt
77a7f3981c
🔴 **Sign-up requires a year of birth and no instrumented test filled it.**
`submit` is `enabled = … && (!registering || birthYear.length == 4)`, so
the button was DISABLED — and `performClick()` on a disabled Compose node
does not throw, it does nothing. Every sign-up in this suite has been
tapping a dead button since the age gate landed, failing two minutes later
with 'nav-contacts did not appear' and no error anywhere on screen.
Nothing was wrong with the app. The suite simply stopped being able to
create an account, and a failing instrumented suite reads as a flaky one.
Also fixes the skip counter in scripts/instrumented.sh: it grepped a bare
SKIPPED, which matches Gradle's own `Task … SKIPPED` lines, so a verdict
written to stop a skip being read as a pass was inventing skips instead.
109 lines
4.8 KiB
Bash
Executable File
109 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run the on-device suites against a relay you named, on an emulator with no
|
|
# history.
|
|
#
|
|
# 🔴 **Two ways an instrumented run silently points at PRODUCTION, and this
|
|
# script closes both.**
|
|
#
|
|
# 1. The suites DEFAULT to a live host when no relay argument is passed —
|
|
# `FeatureUseTest` to relay1.flamenet.io, `LiveSignUpFleetTest` to
|
|
# relay.flamenet.io. A plain `./gradlew connectedAndroidTest` therefore
|
|
# signs accounts up on somebody's real server, and nothing in the output
|
|
# says so.
|
|
# 2. A SAVED EMULATOR SNAPSHOT restores whatever the AVD was last doing.
|
|
# Observed 2026-09-05: a snapshot signed in as `fnreviewbeta` against
|
|
# relay.flamenet.io, restored under a test run that had been told to use a
|
|
# local relay. Every one of the ten tests failed — they could not find the
|
|
# sign-up screen, because the app was already signed in — and the reason
|
|
# was invisible until the failure's screen dump was read.
|
|
#
|
|
# Usage: scripts/instrumented.sh 10.0.2.2:18810 [ClassName ...]
|
|
set -euo pipefail
|
|
|
|
RELAY="${1:-}"
|
|
[ -n "$RELAY" ] || {
|
|
cat >&2 <<'MSG'
|
|
usage: scripts/instrumented.sh <relay-host:port> [TestClass ...]
|
|
|
|
Name the relay. There is deliberately no default: the suites have their own,
|
|
and theirs are live servers.
|
|
|
|
10.0.2.2:18810 the host machine's local relay, as seen from the emulator
|
|
MSG
|
|
exit 2
|
|
}
|
|
shift || true
|
|
|
|
# 🔴 Production is refused, not warned about. A run that creates accounts on
|
|
# relay.flamenet.io cannot be undone by noticing afterwards.
|
|
#
|
|
# ⚠️ relay1/relay2 are the TEST FLEET and are allowed — that is what they are
|
|
# for, and the suites need https because Android blocks cleartext, so a local
|
|
# http relay cannot serve. Loud rather than silent: accounts really are created.
|
|
case "$RELAY" in
|
|
*//relay.flamenet.io*|relay.flamenet.io*)
|
|
echo "REFUSED: $RELAY is the PRODUCTION relay. These suites create accounts." >&2
|
|
echo "Use a test-fleet relay (relay1/relay2) or a local one." >&2
|
|
exit 1;;
|
|
*relay1.flamenet.io*|*relay2.flamenet.io*)
|
|
echo "⚠️ $RELAY is a shared test relay — this run WILL create accounts on it." >&2;;
|
|
esac
|
|
|
|
CLASSES="${*:-io.flamenet.messenger.app.FeatureUseTest,io.flamenet.messenger.app.CrudSurfaceTest,io.flamenet.messenger.app.ChatFlowTest}"
|
|
CLASSES="${CLASSES// /,}"
|
|
|
|
export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
|
|
export JAVA_HOME="${JAVA_HOME:-/opt/homebrew/opt/openjdk@21}"
|
|
ADB="$ANDROID_HOME/platform-tools/adb"
|
|
AVD="${FN_AVD:-fnmsg-android-test}"
|
|
|
|
# ⚠️ `-no-snapshot-load` is the point of this block, and `-no-snapshot-save`
|
|
# keeps the next run clean too. A warm boot is faster and carries the last
|
|
# session's account with it.
|
|
if ! "$ADB" shell getprop sys.boot_completed 2>/dev/null | grep -q 1; then
|
|
echo "### cold-booting $AVD (no snapshot)"
|
|
nohup "$ANDROID_HOME/emulator/emulator" -avd "$AVD" \
|
|
-no-snapshot-load -no-snapshot-save -no-boot-anim \
|
|
-netdelay none -netspeed full >/tmp/fnmsg-emu.log 2>&1 &
|
|
until "$ADB" shell getprop sys.boot_completed 2>/dev/null | grep -q 1; do sleep 5; done
|
|
else
|
|
echo "### emulator already up — clearing the app rather than trusting it"
|
|
"$ADB" uninstall io.flamenet.messenger >/dev/null 2>&1 || true
|
|
"$ADB" uninstall io.flamenet.messenger.test >/dev/null 2>&1 || true
|
|
fi
|
|
|
|
echo "### signed-in account on the device before we start:"
|
|
"$ADB" shell pm list packages 2>/dev/null | grep -i flamenet || echo " (nothing installed — good)"
|
|
|
|
echo "### relay $RELAY · classes $CLASSES"
|
|
set +e
|
|
./gradlew --no-daemon :app:connectedDebugAndroidTest \
|
|
-Pandroid.testInstrumentationRunnerArguments.relay="$RELAY" \
|
|
-Pandroid.testInstrumentationRunnerArguments.class="$CLASSES" 2>&1 \
|
|
| tee /tmp/fnmsg-instrumented.log \
|
|
| grep -E "FAILED|PASSED|SKIPPED|BUILD|Finished [0-9]+ test|Tests on"
|
|
STATUS=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
# 🔴 **A skip is not a pass.** Every one of these suites opens with
|
|
# `assumeTrue(relayReachable())`, so an unreachable relay produces a green
|
|
# build that exercised nothing. Say which happened.
|
|
# ⚠️ Anchored to the TEST line, not a bare "SKIPPED". Gradle prints
|
|
# `> Task :app:preDebugAndroidTestBuild SKIPPED` for up-to-date tasks, so the
|
|
# loose grep reported four skipped tests on a run where none were skipped —
|
|
# a verdict written to stop a skip reading as a pass, inventing skips instead.
|
|
SKIPS=$(grep -cE "^io\.flamenet\..*SKIPPED" /tmp/fnmsg-instrumented.log || true)
|
|
RAN=$(grep -oE "Finished [0-9]+ test" /tmp/fnmsg-instrumented.log | tail -1 || echo "")
|
|
echo
|
|
echo "### $RAN, $SKIPS skipped"
|
|
if [ "$SKIPS" != "0" ] && [ "$STATUS" = "0" ]; then
|
|
echo "### VERDICT: SKIPPED — the relay was unreachable. This is NOT a pass."
|
|
exit 3
|
|
elif [ "$STATUS" = "0" ]; then
|
|
echo "### VERDICT: PASSED on device"
|
|
else
|
|
echo "### VERDICT: FAILED — see /tmp/fnmsg-instrumented.log"
|
|
echo "### and app/build/reports/androidTests/connected/debug/index.html"
|
|
fi
|
|
exit $STATUS
|