Merge branch '2-v3-cross-engine' into 'main'

A v2 peer was being sent a post-quantum ratchet step, and v3 was never advertised

See merge request paulhitt/flamenet-messenger-android!4

engine/src/main/kotlin/io/flamenet/e2e/DeviceIdentity.kt

@@ -83,11 +83,26 @@ class DeviceIdentity private constructor(
         /**
          * The highest handshake version this engine implements.
          *
-         * 2 — hybrid X3DH. Not 3: that puts ML-KEM into every ratchet step and
+         * 3 — ML-KEM in every ratchet step (§13), not only the handshake. This
-         * is a separate piece of work. Claiming 3 here would have peers send
+         * read 2 until the ratchet existed, with the note that claiming 3 early
-         * `pqdh`/`pqct` on every message and this engine reject the lot.
+         * would have peers send `pqdh`/`pqct` on every message and this engine
+         * reject the lot. That was right, and it is why the number moves in the
+         * same change as the evidence rather than ahead of it.
+         *
+         * 🔴 **A version is a claim other devices act on.** Peers pin the
+         * highest proven version and the pin is monotonic (§13.4), so a build
+         * that advertises 3 and cannot honour it does not degrade — it strands
+         * every contact that believed it. What backs this number: the Kotlin
+         * ratchet drives a v3 session against the web engine in both directions
+         * (`flamenet-e2e/test/interop-kotlin.mjs`, §13), including the reply
+         * that only opens if the decapsulate-before-adopt ordering is right.
+         *
+         * ⚠️ Claiming 3 does not mean sending 3 to everyone. `E2ESession`
+         * gates the PQ ratchet on the PEER having proven 3 as well, because
+         * every v2 device offers a PQ prekey and none of them can read a
+         * ratchet step.
          */
-        const val PROTOCOL_VERSION = 2
+        const val PROTOCOL_VERSION = 3
 
         fun create(now: Long = System.currentTimeMillis()): DeviceIdentity = DeviceIdentity(
             deviceId = java.util.UUID.randomUUID().toString().lowercase(),

engine/src/main/kotlin/io/flamenet/e2e/E2ESession.kt

@@ -90,17 +90,30 @@ class E2ESession private constructor(
                 .put("spk_id", bundle.signedPreKeyId)
                 .put("opk_id", bundle.oneTimePreKeyId)
 
+            // 🔴 **Offering a PQ prekey is not a claim to §13.** Every v2 device
+            // publishes one; only a v3 device can read a PQ ratchet step. Sending
+            // one to a v2 peer breaks the conversation twice over — their root
+            // step derives without `ss` under the `_v1` info string while ours
+            // used `_v3`, and their canonical header omits `pqdh`/`pqct` so the
+            // AAD does not match either. The first message never opens.
+            //
+            // ⚠️ `bundle.proto` alone is the relay's word for it. `chosen` is
+            // non-null only when the capability signature over THIS proto
+            // verified (§12.7), so the pair together is a proven claim to v3.
+            val postQuantumRatchet =
+                if (chosen != null && bundle.proto >= 3) chosen.publicKey else null
+
             return E2ESession(
                 // 🔴 `chosen?.publicKey` is the PQ prekey X3DH just encapsulated
                 // to, and the ratchet has to start from that exact key or the
                 // responder cannot decapsulate the first step (§13.2).
                 ratchet = DoubleRatchet.initiator(
-                    initiated.sharedSecret, bundle.signedPreKey, chosen?.publicKey),
+                    initiated.sharedSecret, bundle.signedPreKey, postQuantumRatchet),
                 associatedData = X3DH.associatedData(
                     identity.identitySigningPublic, bundle.identitySigning,
                     initiated.pqCiphertext),
                 pendingHandshake = handshake,
-                version = initiated.version,
+                version = if (postQuantumRatchet != null) 3 else initiated.version,
                 // Rides the first envelope only. Every later `msg` is
                 // byte-identical to v1, because the KEM secret is already
                 // folded into the root key and the associated data binds the

engine/src/test/kotlin/io/flamenet/e2e/HybridSessionTest.kt

@@ -52,6 +52,17 @@ class HybridSessionTest {
         return PeerBundle.fromWire(bundle)
     }
 
+    /**
+     * A peer that has proven v2 and only v2 — the released fleet.
+     *
+     * ⚠️ `copy(proto = 2)` alone would NOT do: the capability signature commits
+     * to the version, so a bundle whose proto was edited reads as v1 to every
+     * peer. It has to be re-signed for 2 to be a genuine v2 device.
+     */
+    private fun publishedV2(device: DeviceIdentity, withOneTime: Boolean = true): PeerBundle =
+        published(device, withOneTime).copy(
+            proto = 2, capabilitySignature = device.capabilitySignature(2))
+
     /** A v1 device: exactly what this engine published before ML-KEM existed. */
     private fun publishedV1(device: DeviceIdentity): PeerBundle {
         val opk = device.generateOneTimePreKeys(1).first()
@@ -70,12 +81,13 @@ class HybridSessionTest {
     // MARK: - The happy path
 
     @Test
-    fun `two v2 devices hold a post-quantum conversation`() {
+    fun `two post-quantum devices hold a post-quantum conversation`() {
         val alice = DeviceIdentity.create()
         val bob = DeviceIdentity.create()
 
         val session = E2ESession.initiate(alice, published(bob))
-        assertEquals(2, session.version, "both ends can, so this must be v2")
+        assertEquals(3, session.version,
+                     "both ends implement the ratchet, so this must be v3")
 
         val envelope = session.encrypt("post-quantum hello")
         assertNotNull(envelope.pq, "the KEM ciphertext has to travel with the handshake")
@@ -108,7 +120,7 @@ class HybridSessionTest {
         val bob = DeviceIdentity.create()
 
         val session = E2ESession.initiate(alice, published(bob, withOneTime = false))
-        assertEquals(2, session.version)
+        assertEquals(3, session.version)
         val envelope = session.encrypt("still hybrid")
         assertEquals("spk", envelope.pq!!.kind)
         assertEquals("still hybrid", E2ESession.respond(bob, envelope).decrypt(envelope))
@@ -184,8 +196,12 @@ class HybridSessionTest {
     @Test
     fun `a capability signature does not transfer between versions`() {
         val bob = DeviceIdentity.create()
-        val forVersion3 = published(bob).copy(proto = 3)
+        // 🔴 A version this engine does not claim, so the edit cannot be
-        assertNull(forVersion3.provenPostQuantumPreKey(),
+        // accidentally valid. This read `proto = 3` while the engine published
+        // 2; the constant moving is what makes a hardcoded neighbour like that
+        // stop testing anything.
+        val forAnotherVersion = published(bob).copy(proto = DeviceIdentity.PROTOCOL_VERSION + 1)
+        assertNull(forAnotherVersion.provenPostQuantumPreKey(),
                    "the signature commits to the version it claims")
     }
 
@@ -203,6 +219,57 @@ class HybridSessionTest {
         assertEquals("spk", moved.provenPostQuantumPreKey()?.kind)
     }
 
+    /**
+     * 🔴 **The advertised version and the capability signature must agree.**
+     *
+     * A peer verifies `Ed25519(ik_sig, "FlamenetE2E_Caps_v2" ‖ uint8(proto))`
+     * against the proto in the same bundle. Bumping the constant without the
+     * signature following it produces a bundle every peer silently reads as v1
+     * — `provenPostQuantumPreKey()` returns null and the whole post-quantum
+     * path disappears, with nothing anywhere reporting an error.
+     */
+    @Test
+    fun `the published bundle claims v3 and proves it`() {
+        val device = DeviceIdentity.create()
+        val wire = device.registrationWire()
+        assertEquals(3, wire.getInt("proto"), "this engine implements the v3 ratchet")
+
+        // Round-trip through the peer's own view, which is the only reading
+        // that matters: it re-verifies the signature over the claimed version.
+        val seen = published(device)
+        assertEquals(3, seen.proto)
+        assertNotNull(seen.provenPostQuantumPreKey(),
+                      "the capability signature does not cover the version claimed")
+    }
+
+    /**
+     * 🔴 **A v2 peer must not be sent a v3 ratchet step.**
+     *
+     * The released fleet advertises `proto: 2`: it does post-quantum key
+     * AGREEMENT and has no post-quantum ratchet. Attaching `pqdh`/`pqct` to a
+     * message for one of those devices breaks the conversation twice over —
+     * their root step derives without `ss` and under the `_v1` info string
+     * while ours used `_v3`, and their canonical header omits the two fields
+     * entirely so the AAD does not match either. The first message never opens
+     * and never will.
+     *
+     * ⚠️ Offering a PQ prekey is NOT the same claim as implementing §13. Every
+     * v2 device offers one. The version has to be what gates this, which is
+     * what §13.4 says and what the iOS engine already does.
+     */
+    @Test
+    fun `a peer that only claims v2 is not sent a post-quantum ratchet step`() {
+        val alice = DeviceIdentity.create()
+        val bob = DeviceIdentity.create()
+        // A real v2 bundle: PQ prekeys published, capability signed for 2.
+        val session = E2ESession.initiate(alice, publishedV2(bob))
+        val envelope = session.encrypt("hello")
+        assertNull(envelope.header.pqdh,
+                   "a v2 peer was sent a v3 ratchet step it cannot read")
+        assertNull(envelope.header.pqct,
+                   "a v2 peer was sent a PQ ciphertext it cannot decapsulate")
+    }
+
     // MARK: - Persistence
 
     /**
@@ -283,9 +350,9 @@ class HybridSessionTest {
      * would be picked for v2 by every peer and then fail every handshake.
      */
     @Test
-    fun `a v2 registration publishes the whole block`() {
+    fun `a post-quantum registration publishes the whole block`() {
         val wire = DeviceIdentity.create().registrationWire(pqOneTime = emptyList())
-        assertEquals(2, wire.getInt("proto"))
+        assertEquals(DeviceIdentity.PROTOCOL_VERSION, wire.getInt("proto"))
         for (field in listOf("caps_sig", "pqspk", "pqspk_sig")) {
             assertTrue(wire.getString(field).isNotEmpty(), "$field must be published")
         }

engine/src/test/kotlin/io/flamenet/e2e/SealedRosterTest.kt

@@ -50,12 +50,40 @@ class SealedRosterTest {
     /**
      * 🔴 The property the whole change rests on: the stored bytes say nothing.
      * A dump that yields the member ids has bought nobody anything.
+     *
+     * ⚠️ **The values are long on purpose, and the search covers the decoded
+     * bytes.** This sealed ids `41`/`42` and looked for `"41"`/`"42"` in the
+     * BASE64 — two characters out of a 64-symbol alphabet, so each had about a
+     * 1-in-4096 chance at any position and roughly a **14% chance across a blob
+     * this size**. It failed on exactly that, having passed for days: a coin
+     * flip wearing the clothes of an assertion. The iOS twin had the same bug
+     * and was fixed the same way; this one was missed because only one engine
+     * was looked at.
+     *
+     * Nine-digit ids and a real-sounding name make a chance match impossible
+     * rather than unlikely, and the byte-level search is the one that actually
+     * states the property — base64 could in principle hide a plaintext run the
+     * raw bytes reveal.
      */
     @Test
     fun `the sealed blob reveals nothing`() {
-        val blob = SealedRoster.seal(contents(), SealedRoster.newGroupKey())
+        val secret = SealedRoster.Contents(
-        for (needle in listOf("41", "42", "Book club", "members", "tags")) {
+            members = listOf(918273645L, 564738291L),
+            tags = mapOf("918273645" to SealedRoster.newTag(),
+                         "564738291" to SealedRoster.newTag()),
+            name = "Chemotherapy support",
+        )
+        val blob = SealedRoster.seal(secret, SealedRoster.newGroupKey())
+        val raw = Bytes.unb64(blob)
+
+        for (needle in listOf("918273645", "564738291", "Chemotherapy support",
+                              "members", "tags", "name")) {
             assertFalse(blob.contains(needle), "$needle is readable in the sealed roster")
+            assertFalse(
+                String(raw, Charsets.ISO_8859_1)
+                    .contains(String(needle.toByteArray(Charsets.UTF_8), Charsets.ISO_8859_1)),
+                "$needle is readable in the sealed bytes",
+            )
         }
     }
 

interop/src/main/kotlin/io/flamenet/interop/Main.kt

@@ -6,6 +6,7 @@ import io.flamenet.e2e.DeviceIdentity
 import io.flamenet.e2e.DoubleRatchet
 import io.flamenet.e2e.E2ESession
 import io.flamenet.e2e.Envelope
+import io.flamenet.e2e.PQKEM
 import io.flamenet.e2e.PeerBundle
 import io.flamenet.e2e.Ed25519
 import io.flamenet.e2e.Hkdf
@@ -51,10 +52,24 @@ private class Device(
     val signedPreKey: KeyPair,
     val oneTimePreKeyId: Int,
     val oneTimePreKey: KeyPair,
+    /**
+     * The post-quantum signed prekey (§12.5). Published in the bundle, and the
+     * key an initiator encapsulates to — which is also where the v3 ratchet
+     * starts, because `PQr` must be exactly the key X3DH targeted (§13.2).
+     */
+    val pqSignedPreKeyId: Int = 1,
+    val pqSignedPreKey: PQKEM.KeyPair = PQKEM.generate(),
 )
 
 private class Session(val ratchet: DoubleRatchet, val ad: ByteArray)
 
+// ⚠️ Spelled out here rather than imported: `DeviceIdentity.PQDomain` is
+// private, and a driver that reached into the engine's internals to build a
+// bundle would be checking the engine against itself. These are the strings the
+// SPEC names, so a divergence between engine and spec shows up as a failure.
+private val PQ_SPK_DOMAIN = "FlamenetE2E_PQSPK_v2".toByteArray(Charsets.UTF_8)
+private val CAPS_DOMAIN = "FlamenetE2E_Caps_v2".toByteArray(Charsets.UTF_8)
+
 fun main() {
     val out = System.out.bufferedWriter()
     generateSequence(::readLine).forEach { line ->
@@ -86,7 +101,7 @@ private fun unb64(text: String): ByteArray = java.util.Base64.getDecoder().decod
 private fun JSONObject.bytes(key: String): ByteArray = unb64(getString(key))
 
 private fun handle(req: JSONObject): JSONObject = when (req.getString("cmd")) {
-    "device" -> newDevice()
+    "device" -> newDevice(req)
     "initiate" -> initiate(req)
     "respond" -> respond(req)
     "encrypt" -> encrypt(req)
@@ -122,7 +137,13 @@ private fun handle(req: JSONObject): JSONObject = when (req.getString("cmd")) {
  * exactly that — so the day Kotlin gains PQ, this bundle changing is what makes
  * the assertion fail and get updated.
  */
-private fun newDevice(): JSONObject {
+private fun newDevice(req: JSONObject): JSONObject {
+    // 🔴 A v1 bundle is still a thing this engine must be able to publish, and
+    // still a thing the harness must be able to test against. The fleet holds
+    // released builds that advertise 1 and 2; if this driver could only speak
+    // v3, the legacy paths would stop being covered on the day v3 landed —
+    // which is the day they matter most.
+    val postQuantum = req.optBoolean("pq", true)
     val id = next("kotlin-device-")
     val device = Device(
         deviceId = id,
@@ -146,7 +167,20 @@ private fun newDevice(): JSONObject {
         .put("opk", JSONObject()
             .put("id", device.oneTimePreKeyId)
             .put("pub", b64(X25519.rawFromPublicKey(device.oneTimePreKey.public))))
-        .put("proto", 1)
+        // §12.3/§12.5 — the PQ signed prekey, under its OWN domain string.
+        // Signing the bare key would verify against nothing the peer computes:
+        // the web engine checks `Ed25519(ik_sig, "FlamenetE2E_PQSPK_v2" ‖ pub)`.
+        .putOpt("pqspk_id", if (postQuantum) device.pqSignedPreKeyId else null)
+        .putOpt("pqspk", if (postQuantum) b64(device.pqSignedPreKey.publicKey) else null)
+        .putOpt("pqspk_sig", if (postQuantum) b64(Ed25519.sign(
+            device.ikSig, PQ_SPK_DOMAIN + device.pqSignedPreKey.publicKey)) else null)
+        // 🔴 §13.4 — advertising a version is a SIGNED claim, not a number.
+        // Without `caps_sig` the web engine's `bundleFromWire` discards the
+        // whole v2 block and `negotiate` returns v1, which is exactly what this
+        // harness was quietly doing before.
+        .putOpt("caps_sig", if (postQuantum) b64(Ed25519.sign(
+            device.ikSig, CAPS_DOMAIN + byteArrayOf(3.toByte()))) else null)
+        .put("proto", if (postQuantum) 3 else 1)
 
     return JSONObject().put("device", id).put("bundle", bundle)
 }
@@ -166,6 +200,18 @@ private fun initiate(req: JSONObject): JSONObject {
     }
 
     val opk = bundle.optJSONObject("opk")
+
+    // §12.5 — the peer's PQ prekey, verified against the same identity key that
+    // signed the classical one. An unverified PQ prekey is worth nothing: a
+    // relay that can substitute it chooses the secret.
+    val pqspk = bundle.optString("pqspk").ifEmpty { null }?.let(::unb64)
+    if (pqspk != null) {
+        val sig = bundle.optString("pqspk_sig").ifEmpty { null }?.let(::unb64)
+        if (sig == null || !Ed25519.verify(theirIkSig, sig, pqspk)) {
+            return JSONObject().put("error", "badPQSignature")
+        }
+    }
+
     val initiated = X3DH.initiate(
         identityPrivateDH = me.ikDH.private,
         bundle = X3DH.Bundle(
@@ -173,11 +219,16 @@ private fun initiate(req: JSONObject): JSONObject {
             signedPreKey = spk,
             oneTimePreKey = opk?.let { unb64(it.getString("pub")) },
         ),
+        bobPQ = pqspk,
     )
 
-    // v1 associated data: the two signing identities, initiator first.
+    // The AD binds the PQ ciphertext on v2+, so a substituted one fails the
-    val ad = me.ikSig.publicKey + theirIkSig
+    // AEAD open rather than silently deriving a different key.
-    val ratchet = DoubleRatchet.initiator(initiated.sharedSecret, spk)
+    val ad = X3DH.associatedData(me.ikSig.publicKey, theirIkSig, initiated.pqCiphertext)
+    // 🔴 `pqspk` — the SAME key X3DH just encapsulated to. The responder holds
+    // its seed, so this is the only value the first ratchet step can be
+    // decapsulated against (§13.2).
+    val ratchet = DoubleRatchet.initiator(initiated.sharedSecret, spk, pqspk)
     val id = next("kotlin-session-")
     sessions[id] = Session(ratchet, ad)
 
@@ -188,6 +239,12 @@ private fun initiate(req: JSONObject): JSONObject {
         .put("ek", b64(initiated.ephemeralPublic))
         .put("opk_id", opk?.getInt("id") ?: -1)
         .put("spk_id", bundle.optInt("spk_id", 0))
+        .put("version", initiated.version)
+        .put("pq", initiated.pqCiphertext?.let {
+            JSONObject().put("kind", "spk")
+                .put("id", bundle.optInt("pqspk_id", 0))
+                .put("ct", b64(it))
+        })
         .put("header", headerJson(message.header))
         .put("ct", b64(message.ciphertext))
 }
@@ -199,16 +256,25 @@ private fun respond(req: JSONObject): JSONObject {
 
     val theirIkDH = req.bytes("ik_dh")
     val theirIkSig = req.bytes("ik_sig")
+    // The PQ half, when the initiator sent one. `pq.ct` was encapsulated to our
+    // published PQ prekey, so the seed is what opens it.
+    val pqBlock = req.optJSONObject("pq")
+    val pq = pqBlock?.let { me.pqSignedPreKey.seed to unb64(it.getString("ct")) }
+
     val sharedSecret = X3DH.respond(
         identityPrivateDH = me.ikDH.private,
         signedPreKeyPrivate = me.signedPreKey.private,
         oneTimePrivate = if (usedOpk) me.oneTimePreKey.private else null,
         initiatorIdentityDH = theirIkDH,
         initiatorEphemeral = req.bytes("ek"),
+        pq = pq,
     )
 
-    val ad = theirIkSig + me.ikSig.publicKey
+    val ad = X3DH.associatedData(theirIkSig, me.ikSig.publicKey, pq?.second)
-    val ratchet = DoubleRatchet.responder(sharedSecret, me.signedPreKey)
+    // 🔴 The seed, not the public key. The initiator encapsulated to our PQ
+    // prekey, so decapsulating the first ratchet step needs the private half —
+    // and `PQr` is adopted from that first header, not set here (§13.2).
+    val ratchet = DoubleRatchet.responder(sharedSecret, me.signedPreKey, pq?.first)
     val id = next("kotlin-session-")
     sessions[id] = Session(ratchet, ad)
 
@@ -216,6 +282,7 @@ private fun respond(req: JSONObject): JSONObject {
     return JSONObject()
         .put("session", id)
         .put("ad", b64(ad))
+        .put("version", if (pq == null) 1 else 2)
         .put("text", String(text, Charsets.UTF_8))
 }
 
@@ -234,10 +301,15 @@ private fun decrypt(req: JSONObject): JSONObject {
     return JSONObject().put("text", String(text, Charsets.UTF_8))
 }
 
+// §13.3 — `pqdh`/`pqct` present only on a PQ step, omitted entirely otherwise.
+// `JSONObject.put(String, null)` drops the key, which is the shape the canonical
+// header needs.
 private fun headerJson(header: DoubleRatchet.Header): JSONObject = JSONObject()
     .put("dh", b64(header.dh))
     .put("pn", header.pn)
     .put("n", header.n)
+    .put("pqdh", header.pqdh?.let(::b64))
+    .put("pqct", header.pqct?.let(::b64))
 
 private fun messageFrom(req: JSONObject): DoubleRatchet.Message {
     val header = req.getJSONObject("header")
@@ -246,6 +318,8 @@ private fun messageFrom(req: JSONObject): DoubleRatchet.Message {
             dh = unb64(header.getString("dh")),
             pn = header.getInt("pn"),
             n = header.getInt("n"),
+            pqdh = header.optString("pqdh").ifEmpty { null }?.let(::unb64),
+            pqct = header.optString("pqct").ifEmpty { null }?.let(::unb64),
         ),
         ciphertext = req.bytes("ct"),
     )