Take the e2e poll vectors and check them in CI
The other half of a change that was split so F6 could land alone. `PollTallyVectorTest` was written, passed locally, and failed in CI with three AssertionErrors at line 40, which is "vectors not found". The fixture lives in flamenet-e2e, CI clones that repo at the commit `.e2e-ref` pins, and the file only existed on an unmerged branch. It passed on a developer machine because there is a sibling checkout there and the runner has none, which is exactly the gap the pin-and-clone exists to close. Not a defect in the test; this repo's cross-repo process working. So now, in one commit as `.e2e-ref.md` asks: the pin moves to e2e main, FLAMENET_POLL_VECTORS joins the export list and the existence check that runs before the suite, and the test comes back. That existence check is the reason the loop above it exists. A path that silently does not exist puts the tests straight back where they were and the failure reads as a protocol bug rather than a missing file. Verified both ways it can be reached: with FLAMENET_POLL_VECTORS set, as CI sets it, and with it unset so the sibling-checkout fallback is what answers. client 480, engine 723, app 652, no failures. The vector suite itself: 3 tests, 0 skipped, 0 failures. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Paul Hitt
co-authored by
Claude Opus 5
Claude Opus 5
parent
c0327c8fc0
commit
e48907077c
@@ -1 +1 @@
|
||||
afd0e4fafeeb56a647f254fcee8668effaf4574a
|
||||
be8093f7c76c42cac5b826f40db95db372bdd34b
|
||||
|
||||
+2
-1
@@ -81,13 +81,14 @@ unit-tests:
|
||||
- export FLAMENET_BACKUP_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/backup-v1.json"
|
||||
- export FLAMENET_BACKUP_PAYLOAD_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/backup-payload-v1.json"
|
||||
- export FLAMENET_CONTENT_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/content-v1.json"
|
||||
- export FLAMENET_POLL_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/poll-tally-v1.json"
|
||||
# Checked before the suite runs. A path that silently does not exist puts
|
||||
# the tests straight back where they were, and the failure would read as a
|
||||
# protocol bug rather than a missing file.
|
||||
- |
|
||||
for f in "$FLAMENET_VECTORS" "$FLAMENET_PQ_VECTORS" "$FLAMENET_ACCOUNT_VECTORS" \
|
||||
"$FLAMENET_BACKUP_VECTORS" "$FLAMENET_BACKUP_PAYLOAD_VECTORS" \
|
||||
"$FLAMENET_CONTENT_VECTORS"; do
|
||||
"$FLAMENET_CONTENT_VECTORS" "$FLAMENET_POLL_VECTORS"; do
|
||||
[ -f "$f" ] || { echo "GATE: vector file missing: $f"; exit 1; }
|
||||
done
|
||||
# Clear old results before running, or the gate counts ghosts. The
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package io.flamenet.e2e
|
||||
|
||||
import java.io.File as JFile
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertNotEquals
|
||||
import kotlin.test.assertTrue
|
||||
import org.json.JSONObject
|
||||
|
||||
/**
|
||||
* The poll arithmetic, against the same vectors the iPhone reads.
|
||||
*
|
||||
* Both engines carry a comment saying the width is load-bearing. `Poll.kt`
|
||||
* here, and `Poll.swift` there: "`Int32` with `&+` and `&-`, never `Int`:
|
||||
* Swift's `Int` is 64 bits on every platform this ships to, so the same shares
|
||||
* would add up to a different number and the two engines would disagree about
|
||||
* the result of a vote."
|
||||
*
|
||||
* Both were unit-tested on their own side and **nothing checked that they
|
||||
* agree**, while a mixed iPhone-and-Android group poll is the ordinary case.
|
||||
* A disagreement surfaces as a wrong count rather than as an error: no crash,
|
||||
* no exception, the poll simply closes on the wrong answer and everybody
|
||||
* believes it.
|
||||
*
|
||||
* Two of the four cases in the fixture come out differently if a reader widens
|
||||
* to 64 bits. The other two agree by luck, which is why the wrapping cases are
|
||||
* there: the wrong width passes a poll or two before anybody notices.
|
||||
*/
|
||||
class PollTallyVectorTest {
|
||||
private val vectors: JSONObject by lazy {
|
||||
val candidates =
|
||||
listOfNotNull(
|
||||
System.getenv("FLAMENET_POLL_VECTORS"),
|
||||
"../../engine/test/vectors/poll-tally-v1.json",
|
||||
"../../../flamenet-e2e/test/vectors/poll-tally-v1.json",
|
||||
"../../flamenet-e2e/test/vectors/poll-tally-v1.json",
|
||||
).map { JFile(it) }
|
||||
val found =
|
||||
candidates.firstOrNull { it.exists() }
|
||||
?: throw AssertionError(
|
||||
"poll vectors not found. Looked in: " +
|
||||
candidates.joinToString(", ") { it.absolutePath },
|
||||
)
|
||||
JSONObject(found.readText())
|
||||
}
|
||||
|
||||
private class Case(
|
||||
val name: String,
|
||||
val options: Int,
|
||||
val partials: List<IntArray>,
|
||||
val tally: IntArray,
|
||||
val differs: Boolean,
|
||||
val widened: LongArray?,
|
||||
)
|
||||
|
||||
private fun cases(): List<Case> {
|
||||
val arr = vectors.getJSONArray("cases")
|
||||
return (0 until arr.length()).map { i ->
|
||||
val c = arr.getJSONObject(i)
|
||||
val partials =
|
||||
c.getJSONArray("partials").let { ps ->
|
||||
(0 until ps.length()).map { j ->
|
||||
ps.getJSONArray(j).let { row ->
|
||||
IntArray(row.length()) { k -> row.getInt(k) }
|
||||
}
|
||||
}
|
||||
}
|
||||
val tally = c.getJSONArray("tally").let { t -> IntArray(t.length()) { k -> t.getInt(k) } }
|
||||
val widened =
|
||||
if (c.has("tally_if_widened_to_64_bits")) {
|
||||
c.getJSONArray("tally_if_widened_to_64_bits").let { w ->
|
||||
LongArray(w.length()) { k -> w.getLong(k) }
|
||||
}
|
||||
} else {
|
||||
null
|
||||
}
|
||||
Case(
|
||||
name = c.getString("name"),
|
||||
options = c.getInt("options"),
|
||||
partials = partials,
|
||||
tally = tally,
|
||||
differs = c.optBoolean("differs_without_wrapping", false),
|
||||
widened = widened,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The control. A fixture that did not load leaves nothing to compare, and a
|
||||
* loop over an empty list is the quietest green there is.
|
||||
*/
|
||||
@Test
|
||||
fun `the vectors are there`() {
|
||||
val all = cases()
|
||||
assertTrue(all.size >= 4, "only ${all.size} poll vectors loaded")
|
||||
assertTrue(
|
||||
all.any { it.differs },
|
||||
"no case here distinguishes 32-bit wrapping from 64-bit arithmetic, so " +
|
||||
"these vectors cannot catch the mistake they exist for",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `every pinned tally is the one this engine computes`() {
|
||||
for (one in cases()) {
|
||||
val got = Poll.tally(one.partials, one.options)
|
||||
assertTrue(
|
||||
got.contentEquals(one.tally),
|
||||
"${one.name}: got ${got.toList()}, pinned ${one.tally.toList()}",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* And the wrapping is what produces them. Adding up in Long must give the
|
||||
* WRONG answer for the cases marked as distinguishing the widths, which
|
||||
* pins the reason rather than only the result.
|
||||
*/
|
||||
@Test
|
||||
fun `the cases that need wrapping really do need it`() {
|
||||
var checked = 0
|
||||
for (one in cases().filter { it.differs }) {
|
||||
val widened = one.widened ?: throw AssertionError("${one.name} has no widened answer")
|
||||
val sum = LongArray(one.options)
|
||||
for (p in one.partials) {
|
||||
for (i in 0 until one.options) sum[i] += p[i].toLong()
|
||||
}
|
||||
assertTrue(
|
||||
sum.contentEquals(widened),
|
||||
"${one.name}: 64-bit sum ${sum.toList()}, recorded ${widened.toList()}",
|
||||
)
|
||||
assertNotEquals(
|
||||
one.tally.map { it.toLong() },
|
||||
sum.toList(),
|
||||
"${one.name} is marked as distinguishing the widths and does not",
|
||||
)
|
||||
checked++
|
||||
}
|
||||
assertEquals(2, checked.coerceAtMost(2), "fewer than two wrapping cases were checked")
|
||||
assertTrue(checked >= 2, "only $checked wrapping case(s) were checked")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user