Check the backup payload against the published vector

Both apps were taught one backup shape and nothing checked that they
agreed. Each side's tests asserted against what the OTHER was believed
to write, transcribed by hand from reading its source, which is a test
of one person's reading rather than of interoperability.

`backup-payload-v1.json` in flamenet-e2e is now the authority, and the
bar it sets is exact: decode the canonical payload, encode it again, and
the result must match byte for byte once both are put in a canonical
order. The same vector is checked on the iPhone, so the two are
compared against one file rather than against each other's source.

One real fix came out of writing it. `messageFromShared` wrote
`author` unconditionally, so a one-to-one message that carried no author
came back claiming user zero sent it, and a round trip through this app
changed the file. Storage keeps an author on every message and reads a
missing one as 0, so it is now written only when the file carried one.

The env var is exported in BOTH ci jobs, the pinned one and the one that
runs at vectors HEAD, and added to the gate that checks every vector
file exists before the suite runs. A path that silently does not exist
puts the tests back where they were and the failure reads as a protocol
bug rather than a missing file. `.e2e-ref` moves to the commit that adds
the vector.

Full suite: 1279 tests, no failures. `ci/lint` accepts the pipeline.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.e2e-ref

@@ -1 +1 @@
-32d3a8824b06e53a0204feebedfee6dd9d384e7b
+d8e1826a946fe5a1f956a3bb42dcb55969e3a890

.gitlab-ci.yml

@@ -61,12 +61,13 @@ unit-tests:
     - export FLAMENET_PQ_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/pqx3dh-v2.json"
     - export FLAMENET_ACCOUNT_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/account-kdf-v1.json"
     - export FLAMENET_BACKUP_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/backup-v1.json"
+    - export FLAMENET_BACKUP_PAYLOAD_VECTORS="$CI_PROJECT_DIR/.ci-vectors/test/vectors/backup-payload-v1.json"
     # Checked before the suite runs. A path that silently does not exist puts
     # the tests straight back where they were, and the failure would read as a
     # protocol bug rather than a missing file.
     - |
       for f in "$FLAMENET_VECTORS" "$FLAMENET_PQ_VECTORS" "$FLAMENET_ACCOUNT_VECTORS" \
-               "$FLAMENET_BACKUP_VECTORS"; do
+               "$FLAMENET_BACKUP_VECTORS" "$FLAMENET_BACKUP_PAYLOAD_VECTORS"; do
         [ -f "$f" ] || { echo "GATE: vector file missing: $f"; exit 1; }
       done
     # Clear old results before running, or the gate counts ghosts. The
@@ -158,6 +159,7 @@ vectors-head:
     - export FLAMENET_PQ_VECTORS="$CI_PROJECT_DIR/.ci-vectors-head/test/vectors/pqx3dh-v2.json"
     - export FLAMENET_ACCOUNT_VECTORS="$CI_PROJECT_DIR/.ci-vectors-head/test/vectors/account-kdf-v1.json"
     - export FLAMENET_BACKUP_VECTORS="$CI_PROJECT_DIR/.ci-vectors-head/test/vectors/backup-v1.json"
+    - export FLAMENET_BACKUP_PAYLOAD_VECTORS="$CI_PROJECT_DIR/.ci-vectors-head/test/vectors/backup-payload-v1.json"
     # Only the module that reads vectors. The rest of the suite has nothing
     # to say about whether flamenet-e2e moved, and running it here would double
     # the board's cost to answer a question it cannot answer.

engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt

@@ -143,10 +143,11 @@ object BackupPayload {
                 .put("at", m.optLong("at"))
                 .put("text", m.optString("text"))
         m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) }
-        // The author rides along on one-to-one threads too. The iPhone reads it
-        // only for groups and ignores it elsewhere, which costs nothing and
-        // keeps an Android-to-Android restore whole.
-        if (m.has("author")) out.put("author", m.optLong("author"))
+        // The author rides along on group threads, where it says who spoke.
+        // Only when there is one: this app writes 0 on a one-to-one message,
+        // where the author is whoever is not you, and emitting that would put
+        // a claim in the file that the message came from user zero.
+        m.optLong("author", 0L).takeIf { it != 0L }?.let { out.put("author", it) }
         m.optJSONObject("att")?.let {
             out.put("att", attachmentToShared(it, m.optString("kind", Attachment.IMAGE)))
         }
@@ -307,7 +308,9 @@ object BackupPayload {
                 .put("mine", m.optBoolean("mine"))
                 .put("at", m.optLong("at"))
                 .put("text", m.optString("text"))
-                .put("author", m.optLong("author"))
+        // Storage keeps an author on every message and reads a missing one as
+        // 0, so this is written only when the file carried one.
+        m.optLong("author", 0L).takeIf { it != 0L }?.let { out.put("author", it) }
         m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) }
         m.optJSONObject("att")?.let { att ->
             out.put("kind", att.optString("kind", Attachment.IMAGE))

engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadVectorTest.kt

@@ -0,0 +1,109 @@
+package io.flamenet.e2e
+
+import org.json.JSONArray
+import org.json.JSONObject
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+import java.io.File as JFile
+
+/**
+ * The payload inside a §16 backup, against the published vector.
+ *
+ * §16 pinned the envelope and nothing else, so both apps sealed their own
+ * storage and refused each other's files. Both now claim one shape, and until
+ * this vector existed nothing checked that claim: each side's tests asserted
+ * against what the OTHER was believed to write, transcribed by hand from
+ * reading its source. Agreeing with your own reading of somebody else's code is
+ * not interoperability.
+ *
+ * The bar here is exact. Decode the canonical payload, encode it again, and the
+ * result must match byte for byte after both are put in a canonical order.
+ * Anything looser passes while a field quietly goes missing, which is the whole
+ * failure being guarded against.
+ */
+class BackupPayloadVectorTest {
+    private val vectors: JSONObject by lazy {
+        val candidates =
+            listOfNotNull(
+                System.getenv("FLAMENET_BACKUP_PAYLOAD_VECTORS"),
+                "../../engine/test/vectors/backup-payload-v1.json",
+                "../../../flamenet-e2e/test/vectors/backup-payload-v1.json",
+                "../../flamenet-e2e/test/vectors/backup-payload-v1.json",
+            ).map { JFile(it) }
+        val found =
+            candidates.firstOrNull { it.exists() }
+                ?: throw AssertionError(
+                    "backup payload vectors not found. Looked in: " +
+                        candidates.joinToString(", ") { it.absolutePath },
+                )
+        JSONObject(found.readText())
+    }
+
+    /** Sorted keys, so comparing two payloads compares content and not order. */
+    private fun canonical(value: Any?): String =
+        when (value) {
+            is JSONObject -> {
+                value
+                    .keys()
+                    .asSequence()
+                    .sorted()
+                    .joinToString(",", "{", "}") { k -> JSONObject.quote(k) + ":" + canonical(value.get(k)) }
+            }
+
+            is JSONArray -> {
+                (0 until value.length()).joinToString(",", "[", "]") { canonical(value.get(it)) }
+            }
+
+            // A whole number that arrives as a Double must not print as
+            // "3600.0" on one side and "3600" on the other; the difference is
+            // an artefact of the parser, not of the file.
+            is Double -> {
+                if (value == Math.floor(value) && !value.isInfinite()) value.toLong().toString() else value.toString()
+            }
+
+            is Number -> {
+                value.toString()
+            }
+
+            is String -> {
+                JSONObject.quote(value)
+            }
+
+            else -> {
+                value.toString()
+            }
+        }
+
+    @Test
+    fun `the canonical payload survives this app unchanged`() {
+        val pinned = vectors.getJSONObject("canonical")
+        val here = BackupPayload.toShared(BackupPayload.fromShared(pinned))
+        assertEquals(
+            canonical(pinned),
+            canonical(here),
+            "this app changed the shared payload on a round trip, so a backup written here " +
+                "is not the one an iPhone reads",
+        )
+    }
+
+    @Test
+    fun `the vector says what only one side keeps`() {
+        // A field missing from a vector looks exactly like a field nobody
+        // thought of. Naming them is what makes a gap a decision.
+        val listed = vectors.getJSONArray("divergences")
+        assertTrue(listed.length() > 0)
+        for (i in 0 until listed.length()) {
+            val d = listed.getJSONObject(i)
+            assertTrue(d.getString("field").isNotEmpty())
+            assertTrue(d.getString("detail").length > 40, "a divergence without its consequence is a note")
+        }
+    }
+
+    @Test
+    fun `the outbox is not in the vector, and must not be added`() {
+        // Both apps flush it every poll cycle, so a restored outbox sends at
+        // once; a backup opened months later would deliver forgotten messages.
+        assertTrue(!vectors.getJSONObject("canonical").has("outbox"))
+    }
+}