From 5a5e41bfa2b9791994932f3c30c4c9d68775f6c5 Mon Sep 17 00:00:00 2001 From: Paul Hitt Date: Thu, 10 Sep 2026 18:11:35 -0400 Subject: [PATCH] Android writes and reads the shared backup format Section 16 pins the envelope and says nothing about what is sealed, so each app sealed its own storage verbatim and stamped the file with the platform that wrote it. Each then refused the other's, in as many words: "import it on a device of the same kind". Somebody moving between an Android phone and an iPhone lost every conversation they had. The iPhone half landed first. This is the other half. `BackupPayload` translates rather than dumps, which is the entire point. This app stores a message under the keys its wire format uses (`secs`, `fwd`, `rx`), keeps one-to-one and group threads in one map under prefixed ids, hangs the attachment kind beside `att` rather than inside it, abbreviates the sizes, and holds expiries in unix seconds. The shared shape spells all of that out and counts in milliseconds. Exporting storage as-is is what made the two formats incompatible in the first place, so a copy would have reproduced the bug with more steps. Federated threads are the one thing that does not fit. This app keys them by address and the iPhone's history is keyed by numeric user id, with a decoder that skips anything else, so an address left in `threads` would be dropped in silence. They travel under `addrThreads` instead: an iPhone ignores the key, an Android phone gets them back, and nothing disappears without somebody having decided it should. Import tells the two formats apart by SHAPE, not by file name, because a name is whatever the person saving it chose. Section 16 announces itself with `alg` and `ct`; the older export announces itself with `magic`. A wrong passphrase on a section 16 file reports a wrong passphrase and does NOT fall through to the legacy reader, which would have answered "that file is not a Flamenet history export" for a file that is one and sent somebody hunting for corruption instead of a typo. Nothing writes the old format any more, and everything still reads it. Seventeen tests: the shared side against the key names the iPhone writes, the round trip back into this app's own storage shape, the unit conversions in both directions, and a payload transcribed from the iPhone's encoder. That last one is the weak point and the test says so: it proves this app reads the shape the iPhone is BELIEVED to write. Only a vector both apps are built against proves they agree, and `backup-v1.json` pins the envelope alone, which is exactly how the two shapes drifted apart. Full suite: 1271 tests, no failures. Co-Authored-By: Claude Opus 5 --- .../flamenet/messenger/app/HistoryBackup.kt | 12 +- .../messenger/app/MessengerViewModel.kt | 42 +- .../src/main/kotlin/io/flamenet/e2e/Backup.kt | 44 ++ .../kotlin/io/flamenet/e2e/BackupPayload.kt | 379 +++++++++++++++++ .../io/flamenet/e2e/BackupPayloadTest.kt | 401 ++++++++++++++++++ 5 files changed, 871 insertions(+), 7 deletions(-) create mode 100644 engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt create mode 100644 engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt diff --git a/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt b/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt index 6d48285..10fe821 100644 --- a/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt +++ b/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt @@ -25,7 +25,7 @@ import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.input.PasswordVisualTransformation import androidx.compose.ui.unit.sp -import io.flamenet.e2e.HistoryExport +import io.flamenet.e2e.BackupPayload import kotlinx.coroutines.launch import java.text.SimpleDateFormat import java.util.Date @@ -71,9 +71,9 @@ fun HistoryBackupSection(model: MessengerViewModel) { Text( "Your history lives only on this phone, the server keeps none, so a " + "new phone starts empty. An export is sealed with a passphrase you " + - "choose and can be imported on another Android phone signed into " + - "this account. Lose the passphrase and the backup cannot be " + - "opened, by anyone.", + "choose and opens on any phone signed into this account, Android " + + "or iPhone. Lose the passphrase and the backup cannot be opened, " + + "by anyone.", fontSize = 12.sp, color = Palette.SecondaryText, ) @@ -124,7 +124,9 @@ fun HistoryBackupSection(model: MessengerViewModel) { } else { pendingExport = bytes val stamp = SimpleDateFormat("yyyy-MM-dd", Locale.US).format(Date()) - saveLauncher.launch("flamenet-history-$stamp.${HistoryExport.FILE_EXTENSION}") + // The shared extension. Backups made by older versions + // still open; this is only what new ones are called. + saveLauncher.launch("flamenet-history-$stamp.${BackupPayload.FILE_EXTENSION}") } } } diff --git a/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt b/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt index 2e3dc31..0d736ae 100644 --- a/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt +++ b/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt @@ -17,6 +17,8 @@ import io.flamenet.client.RelayAccounts import io.flamenet.client.RelayClient import io.flamenet.client.RelayException import io.flamenet.e2e.AccountKdf +import io.flamenet.e2e.Backup +import io.flamenet.e2e.BackupPayload import io.flamenet.e2e.ContactCard import io.flamenet.e2e.Content import io.flamenet.e2e.DeviceIdentity @@ -2641,7 +2643,15 @@ class MessengerViewModel( */ suspend fun exportHistory(passphrase: CharArray): ByteArray? = withContext(Dispatchers.Default) { - vault?.let { HistoryExport.seal(it.exportPayload(), passphrase) } + vault?.let { + // The shared section 16 format, so this file opens on an + // iPhone. The older Android-only export is still READ, below, + // but nothing writes one any more. + val payload = BackupPayload.toShared(it.exportPayload()) + Backup.encodeFile( + Backup.create(payload.toString().toByteArray(Charsets.UTF_8), String(passphrase)), + ) + } } /** Merge a backup file in. Returns how much of it was actually new. */ @@ -2655,12 +2665,40 @@ class MessengerViewModel( ?: return@withContext Result.failure(IllegalStateException("Sign in first.")) runCatching { v.mergeImportedPayload( - HistoryExport.open(bytes, passphrase), + openBackup(bytes, passphrase), nowSeconds = DeviceClock.expirySeconds(), ) } } + /** + * Open a backup of either kind, told apart by SHAPE rather than by name. + * + * A file name is whatever the person saving it chose, and the two formats + * are not distinguishable by extension once somebody has renamed one. The + * section 16 envelope announces itself with `alg` and `ct`; the older + * Android-only export announces itself with `magic`. So the shape decides, + * and somebody restoring an old backup never has to know which they have. + * + * The section 16 attempt goes first because it is what this app writes now. + * A wrong passphrase must NOT fall through to the legacy reader: it would + * report "not a Flamenet history export" for a file that is one, and send + * somebody looking for a corrupted file instead of a typo. + */ + private fun openBackup( + bytes: ByteArray, + passphrase: CharArray, + ): org.json.JSONObject { + val envelope = runCatching { Backup.decodeFile(bytes) }.getOrNull() + if (envelope != null) { + val plain = + runCatching { Backup.open(envelope, String(passphrase)) } + .getOrElse { throw HistoryExport.WrongPassphrase() } + return BackupPayload.fromShared(org.json.JSONObject(String(plain, Charsets.UTF_8))) + } + return HistoryExport.open(bytes, passphrase) + } + // -- Stories (ยง6.6) ------------------------------------------------------. fun refreshStories() = diff --git a/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt b/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt index f730f4f..61322dc 100644 --- a/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt +++ b/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt @@ -1,5 +1,6 @@ package io.flamenet.e2e +import org.json.JSONObject import java.security.SecureRandom import java.util.Base64 @@ -104,6 +105,49 @@ object Backup { ) } + /** + * The file as bytes on disk. + * + * Written with a JSON writer rather than by hand, unlike [headerBytes]: + * this is the outer document, nobody authenticates it, and a reader parses + * it. Only the header inside the AAD has to match another engine byte for + * byte. + */ + fun encodeFile(file: File): ByteArray = + JSONObject() + .put("v", file.v) + .put("alg", file.alg) + .put("salt", file.salt) + .put("iters", file.iters) + .put("nonce", file.nonce) + .put("ct", file.ct) + .toString() + .toByteArray(Charsets.UTF_8) + + /** + * Read a file back, or say it is not one. + * + * Recognising the shape is what lets an import accept both this and the + * older Android-only export without being told which it was handed, so + * nobody has to know what wrote the file they are restoring. + */ + fun decodeFile(bytes: ByteArray): File { + val o = + runCatching { JSONObject(String(bytes, Charsets.UTF_8)) }.getOrNull() + ?: throw UnsupportedBackup("not a backup file") + for (key in listOf("v", "alg", "salt", "nonce", "ct")) { + if (!o.has(key)) throw UnsupportedBackup("not a backup file") + } + return File( + v = o.optInt("v"), + alg = o.optString("alg"), + salt = o.optString("salt"), + iters = o.optInt("iters"), + nonce = o.optString("nonce"), + ct = o.optString("ct"), + ) + } + /** Open a backup, returning the payload bytes. */ fun open( file: File, diff --git a/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt b/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt new file mode 100644 index 0000000..93d81a1 --- /dev/null +++ b/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt @@ -0,0 +1,379 @@ +package io.flamenet.e2e + +import org.json.JSONArray +import org.json.JSONObject + +/** + * This device's history, translated into the shape both apps agree on. + * + * Section 16 pins the envelope: the KDF, the canonical header used as AAD and + * the AES-GCM sealing. It says nothing about what is sealed. Until now each app + * sealed its own storage verbatim and stamped the file with the platform that + * wrote it, and each then refused the other's, in as many words: "import it on + * a device of the same kind". Somebody moving between an Android phone and an + * iPhone lost every conversation they had. + * + * The iPhone half of the shared shape landed first. This is the other half. + * + * It is a translation and not a dump, which is the whole point. This app stores + * a message under the keys its wire format uses (`secs`, `fwd`, `rx`), keeps + * one-to-one and group threads in a single map under prefixed ids, and holds + * expiries in unix seconds. The shared shape spells those out, separates the + * two kinds of thread and counts in milliseconds. Exporting the storage as-is + * is exactly what made the two formats incompatible in the first place. + * + * Three rules hold it together. + * + * **Times are epoch milliseconds, as integers.** This app already keeps `at` in + * millis, so messages pass straight through; expiries do not, and are converted + * at the boundary. A backup that disagreed about the unit would restore every + * story at the wrong time without ever failing. + * + * **A key this file does not name is not carried.** Both sides build a fresh + * object rather than copying, so adding a field to storage and expecting it to + * appear in a backup does not work. `BackupPayloadTest` reads this source and + * fails if a known key stops being mentioned. + * + * **What the other app cannot represent is kept apart rather than faked.** + * Federated threads are the case that matters: this app keys them by address, + * and the iPhone's history has no room for one, so they travel under their own + * key. An iPhone ignores it and an Android phone gets them back. + */ +object BackupPayload { + const val VERSION = 1 + + /** What a section 16 backup is called on disk. Shared with the iPhone. */ + const val FILE_EXTENSION = "fnbackup" + + private const val GROUP_PREFIX = "group:" + private const val ADDRESS_PREFIX = "addr:" + + /** Federated threads, which the iPhone has nowhere to put. */ + private const val ADDRESS_THREADS = "addrThreads" + + // -- Writing -------------------------------------------------------------. + + /** Turn [internal], as [Vault.exportPayload] produces it, into the shared shape. */ + fun toShared(internal: JSONObject): JSONObject { + val root = JSONObject().put("v", VERSION) + + val threads = JSONObject() + val groups = JSONObject() + val addressed = JSONObject() + internal.optJSONObject("threads")?.let { all -> + for (tid in all.keys()) { + val messages = all.optJSONArray(tid) ?: continue + when { + tid.startsWith(GROUP_PREFIX) -> { + groups.put(tid.removePrefix(GROUP_PREFIX), mapArray(messages, ::messageToShared)) + } + + tid.startsWith(ADDRESS_PREFIX) -> { + addressed.put(tid, mapArray(messages, ::messageToShared)) + } + + else -> { + threads.put(tid, mapArray(messages, ::messageToShared)) + } + } + } + } + if (threads.length() > 0) root.put("threads", threads) + if (groups.length() > 0) root.put("groups", groups) + if (addressed.length() > 0) root.put(ADDRESS_THREADS, addressed) + + internal.optJSONArray("callLog")?.takeIf { it.length() > 0 }?.let { log -> + root.put( + "calls", + mapArray(log) { c -> + JSONObject() + .put("peer", c.optLong("peer")) + .put("at", c.optLong("at")) + .put("in", c.optBoolean("in")) + // The only rename here: this app writes `ok`. + .put("connected", c.optBoolean("ok")) + }, + ) + } + + // Stories are one flat list here and are keyed by author over there. + internal.optJSONObject("stories")?.optJSONArray("items")?.let { items -> + val byAuthor = JSONObject() + for (i in 0 until items.length()) { + val s = items.optJSONObject(i) ?: continue + val author = s.optLong("author").toString() + val list = byAuthor.optJSONArray(author) ?: JSONArray().also { byAuthor.put(author, it) } + list.put(storyToShared(s)) + } + if (byAuthor.length() > 0) root.put("stories", byAuthor) + } + + internal.optJSONObject("roster")?.let { r -> + r.optJSONArray("items")?.takeIf { it.length() > 0 }?.let { root.put("roster", it) } + if (r.has("offServer")) root.put("rosterOffServer", r.optBoolean("offServer")) + } + + internal.optJSONObject("groupNames")?.takeIf { it.length() > 0 }?.let { root.put("groupNames", it) } + + internal.optJSONObject("groupSecrets")?.takeIf { it.length() > 0 }?.let { all -> + val out = JSONObject() + for (gid in all.keys()) { + val s = all.optJSONObject(gid) ?: continue + val one = + JSONObject() + .put("tag", s.optString("tag")) + .put("key", s.optString("key")) + // The relay refuses a roster rewrite whose version does + // not increase, so a restore that reset it would lock + // the group out of its own next rekey. + .put("version", s.optInt("version")) + s.optString("owner").takeIf { it.isNotEmpty() }?.let { one.put("ownerPrivate", it) } + out.put(gid, one) + } + root.put("groupSecrets", out) + } + + return root + } + + private fun messageToShared(m: JSONObject): JSONObject { + val out = + JSONObject() + .put("mine", m.optBoolean("mine")) + .put("at", m.optLong("at")) + .put("text", m.optString("text")) + m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) } + // The author rides along on one-to-one threads too. The iPhone reads it + // only for groups and ignores it elsewhere, which costs nothing and + // keeps an Android-to-Android restore whole. + if (m.has("author")) out.put("author", m.optLong("author")) + m.optJSONObject("att")?.let { + out.put("att", attachmentToShared(it, m.optString("kind", Attachment.IMAGE))) + } + m.optJSONObject("rx")?.takeIf { it.length() > 0 }?.let { out.put("reactions", it) } + m.optLong("edited", 0L).takeIf { it > 0L }?.let { out.put("edited", it) } + // Both shapes, for the reason `Content.isForwarded` accepts both: + // `optBoolean` does not coerce a number, so a `"fwd":1` from another + // engine would read as false. + if (m.optBoolean("fwd", false) || m.optInt("fwd", 0) != 0) out.put("forwarded", true) + m.optInt("secs", 0).takeIf { it > 0 }?.let { out.put("lifetime", it) } + m.optJSONObject("re")?.let { out.put("re", it) } + m.optJSONObject("stk")?.let { out.put("stk", it) } + m.optJSONObject("lnk")?.let { out.put("lnk", it) } + m.optString("delivery").takeIf { it.isNotEmpty() }?.let { out.put("delivery", it.lowercase()) } + return out + } + + /** + * The kind lives beside `att` here and inside it over there, and the sizes + * and duration are spelled out rather than abbreviated. + */ + private fun attachmentToShared( + a: JSONObject, + kind: String, + ): JSONObject { + val out = + JSONObject() + .put("kind", kind) + .put("id", a.optString("id")) + .put("key", a.optString("key")) + .put("digest", a.optString("digest")) + .put("bytes", a.optInt("bytes")) + .put("mime", a.optString("mime")) + .put("width", a.optInt("w")) + .put("height", a.optInt("h")) + a.optDouble("dur", 0.0).takeIf { it > 0.0 }?.let { out.put("duration", it) } + // Seconds here, milliseconds there. + a.optDouble("exp", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", secondsToMillis(it)) } + return out + } + + private fun storyToShared(s: JSONObject): JSONObject { + val out = + JSONObject() + .put("id", s.opt("id")) + .put("author", s.optLong("author")) + .put("mine", s.optBoolean("mine")) + .put("caption", s.optString("caption")) + .put("at", s.optLong("at")) + s.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", secondsToMillis(it)) } + s.optJSONObject("ref")?.let { out.put("att", attachmentToShared(it, Attachment.STORY)) } + s.optString("gid").takeIf { it.isNotEmpty() }?.let { out.put("gid", it) } + s.optString("stream").takeIf { it.isNotEmpty() }?.let { out.put("stream", it) } + s.optString("key").takeIf { it.isNotEmpty() }?.let { out.put("key", it) } + s.optJSONObject("set")?.let { out.put("set", it) } + // The iPhone has no unread mark on a story, so this is ours alone. It + // travels anyway: coming back here it is the difference between a strip + // that remembers what you watched and one that shows everything as new. + if (s.optBoolean("seen", false)) out.put("seen", true) + return out + } + + // -- Reading -------------------------------------------------------------. + + class NewerBackup( + val version: Int, + ) : Exception("This backup was written by a newer version of the app.") + + /** Turn a shared payload back into what [Vault.mergeImportedPayload] expects. */ + fun fromShared(shared: JSONObject): JSONObject { + val version = shared.optInt("v", 0) + if (version > VERSION) throw NewerBackup(version) + + val threads = JSONObject() + shared.optJSONObject("threads")?.let { all -> + for (tid in all.keys()) { + all.optJSONArray(tid)?.let { threads.put(tid, mapArray(it, ::messageFromShared)) } + } + } + shared.optJSONObject("groups")?.let { all -> + for (gid in all.keys()) { + all.optJSONArray(gid)?.let { + threads.put(GROUP_PREFIX + gid, mapArray(it, ::messageFromShared)) + } + } + } + shared.optJSONObject(ADDRESS_THREADS)?.let { all -> + // Already prefixed when written, so they go back under their own id. + for (tid in all.keys()) { + all.optJSONArray(tid)?.let { threads.put(tid, mapArray(it, ::messageFromShared)) } + } + } + + val out = JSONObject().put("threads", threads) + + shared.optJSONArray("calls")?.takeIf { it.length() > 0 }?.let { calls -> + out.put( + "callLog", + mapArray(calls) { c -> + JSONObject() + .put("peer", c.optLong("peer")) + .put("at", c.optLong("at")) + .put("in", c.optBoolean("in")) + .put("ok", c.optBoolean("connected")) + }, + ) + } + + shared.optJSONObject("stories")?.let { byAuthor -> + val items = JSONArray() + for (author in byAuthor.keys()) { + val list = byAuthor.optJSONArray(author) ?: continue + for (i in 0 until list.length()) { + val s = list.optJSONObject(i) ?: continue + items.put(storyFromShared(s, author.toLongOrNull() ?: 0L)) + } + } + if (items.length() > 0) out.put("stories", JSONObject().put("items", items)) + } + + shared.optJSONArray("roster")?.takeIf { it.length() > 0 }?.let { + out.put( + "roster", + JSONObject().put("items", it).put("offServer", shared.optBoolean("rosterOffServer", false)), + ) + } + + shared.optJSONObject("groupNames")?.takeIf { it.length() > 0 }?.let { out.put("groupNames", it) } + + shared.optJSONObject("groupSecrets")?.takeIf { it.length() > 0 }?.let { all -> + val secrets = JSONObject() + for (gid in all.keys()) { + val s = all.optJSONObject(gid) ?: continue + val one = + JSONObject() + .put("tag", s.optString("tag")) + .put("key", s.optString("key")) + .put("version", s.optInt("version")) + s.optString("ownerPrivate").takeIf { it.isNotEmpty() }?.let { one.put("owner", it) } + secrets.put(gid, one) + } + out.put("groupSecrets", secrets) + } + + return out + } + + private fun messageFromShared(m: JSONObject): JSONObject { + val out = + JSONObject() + .put("mine", m.optBoolean("mine")) + .put("at", m.optLong("at")) + .put("text", m.optString("text")) + .put("author", m.optLong("author")) + m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) } + m.optJSONObject("att")?.let { att -> + out.put("kind", att.optString("kind", Attachment.IMAGE)) + out.put("att", attachmentFromShared(att)) + } + m.optJSONObject("reactions")?.takeIf { it.length() > 0 }?.let { out.put("rx", it) } + m.optLong("edited", 0L).takeIf { it > 0L }?.let { out.put("edited", it) } + if (m.optBoolean("forwarded", false) || m.optInt("forwarded", 0) != 0) out.put("fwd", true) + m.optInt("lifetime", 0).takeIf { it > 0 }?.let { out.put("secs", it) } + m.optJSONObject("re")?.let { out.put("re", it) } + m.optJSONObject("stk")?.let { out.put("stk", it) } + m.optJSONObject("lnk")?.let { out.put("lnk", it) } + // Stored upper case here, lower case on the wire and in the file. + m.optString("delivery").takeIf { it.isNotEmpty() }?.let { out.put("delivery", it.uppercase()) } + return out + } + + private fun attachmentFromShared(a: JSONObject): JSONObject { + val out = + JSONObject() + .put("id", a.optString("id")) + .put("key", a.optString("key")) + .put("digest", a.optString("digest")) + .put("bytes", a.optInt("bytes")) + .put("mime", a.optString("mime")) + .put("w", a.optInt("width")) + .put("h", a.optInt("height")) + a.optDouble("duration", 0.0).takeIf { it > 0.0 }?.let { out.put("dur", it) } + a.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("exp", millisToSeconds(it)) } + return out + } + + private fun storyFromShared( + s: JSONObject, + author: Long, + ): JSONObject { + val out = + JSONObject() + // The iPhone numbers its stories and this app names them, so an + // id that arrives as a number becomes its own digits rather + // than being dropped. `mergeImportedPayload` de-duplicates on + // this string, so it only has to be stable, not numeric. + .put("id", s.opt("id")?.toString().orEmpty()) + .put("author", if (s.has("author")) s.optLong("author") else author) + .put("mine", s.optBoolean("mine")) + .put("caption", s.optString("caption")) + .put("at", s.optLong("at")) + .put("seen", s.optBoolean("seen", false)) + s.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", millisToSeconds(it)) } + // `ref` carries the expiry as well, because that is where the strip + // reads it back from when deciding whether the bytes may still render. + s.optJSONObject("att")?.let { out.put("ref", attachmentFromShared(it)) } + s.optString("gid").takeIf { it.isNotEmpty() }?.let { out.put("gid", it) } + s.optString("stream").takeIf { it.isNotEmpty() }?.let { out.put("stream", it) } + s.optString("key").takeIf { it.isNotEmpty() }?.let { out.put("key", it) } + s.optJSONObject("set")?.let { out.put("set", it) } + return out + } + + // -- Small shared pieces -------------------------------------------------. + + private fun secondsToMillis(seconds: Double): Long = Math.round(seconds * 1000.0) + + private fun millisToSeconds(millis: Double): Double = millis / 1000.0 + + private fun mapArray( + source: JSONArray, + transform: (JSONObject) -> JSONObject, + ): JSONArray { + val out = JSONArray() + for (i in 0 until source.length()) { + source.optJSONObject(i)?.let { out.put(transform(it)) } + } + return out + } +} diff --git a/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt b/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt new file mode 100644 index 0000000..a107c88 --- /dev/null +++ b/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt @@ -0,0 +1,401 @@ +package io.flamenet.e2e + +import org.json.JSONArray +import org.json.JSONObject +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The shared backup shape, in both directions. + * + * A translation that loses something does not fail: it restores a shorter + * conversation, which looks exactly like a backup of a shorter conversation. + * So the round trip is asserted field by field rather than by eye, and the + * shared side is asserted against the key names the iPhone actually writes, + * because agreeing with ourselves is not interoperability. + */ +class BackupPayloadTest { + private fun attachment(): JSONObject = + JSONObject() + .put("id", "att-1") + .put("key", "a".repeat(43)) + .put("digest", "b".repeat(43)) + .put("bytes", 4096) + .put("mime", "image/jpeg") + .put("w", 800) + .put("h", 600) + .put("dur", 12.5) + + private fun message(): JSONObject = + JSONObject() + .put("author", 7L) + .put("mid", "mid-1") + .put("text", "still on for saturday") + .put("mine", true) + .put("at", 1_700_000_000_000L) + .put("secs", 3600) + .put("delivery", "DELIVERED") + .put("kind", Attachment.IMAGE) + .put("att", attachment()) + .put("lnk", JSONObject().put("url", "https://example.org").put("title", "T").put("desc", "D")) + .put("fwd", true) + .put("re", JSONObject().put("mid", "mid-0").put("author", 9L).put("preview", "earlier")) + .put("stk", JSONObject().put("pack", "classic").put("id", "wave")) + .put("edited", 1_700_000_060_000L) + .put("rx", JSONObject().put("fire", JSONArray(listOf(7L, 9L)))) + + private fun internalPayload(): JSONObject = + JSONObject() + .put( + "threads", + JSONObject() + .put("7", JSONArray().put(message())) + .put("group:gid-1", JSONArray().put(message())) + .put("addr:someone@elsewhere.example", JSONArray().put(message())), + ).put( + "callLog", + JSONArray().put( + JSONObject() + .put("peer", 42L) + .put("in", true) + .put("ok", true) + .put("at", 1_756_400_500_000L), + ), + ).put( + "stories", + JSONObject().put( + "items", + JSONArray().put( + JSONObject() + .put("id", "story-1") + .put("author", 9L) + .put("mine", false) + .put("caption", "look") + .put("at", 1_700_000_000_000L) + .put("seen", true) + .put("expires", 1_700_600_000.0) + .put("ref", attachment()) + .put("gid", "gid-1") + .put("stream", "https://example.org/s.m3u8") + .put("key", "story-key") + .put("set", JSONObject().put("id", "set-1").put("i", 1).put("n", 3)), + ), + ), + ).put( + "roster", + JSONObject() + .put("items", JSONArray().put(JSONObject().put("id", 9L).put("handle", "ada").put("name", "Ada"))) + .put("offServer", true), + ).put("groupNames", JSONObject().put("gid-1", "Saturday")) + .put( + "groupSecrets", + JSONObject().put( + "gid-1", + JSONObject() + .put("tag", "t") + .put("key", "k") + .put("version", 4) + .put("owner", "p"), + ), + ) + + private fun roundTrip(): JSONObject = BackupPayload.fromShared(BackupPayload.toShared(internalPayload())) + + // -- The shared side is what the iPhone reads ----------------------------. + + @Test + fun `the shared shape uses the names the iphone writes`() { + val shared = BackupPayload.toShared(internalPayload()) + assertEquals(BackupPayload.VERSION, shared.getInt("v")) + val m = shared.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + // Renamed on the way out. Writing this app's own key names is what made + // the two formats incompatible to begin with. + assertEquals(3600, m.getInt("lifetime")) + assertTrue(m.getBoolean("forwarded")) + assertEquals( + setOf("fire"), + m + .getJSONObject("reactions") + .keys() + .asSequence() + .toSet(), + ) + assertFalse(m.has("secs"), "the wire key must not survive into the file") + assertFalse(m.has("fwd")) + assertFalse(m.has("rx")) + // Carried through untouched, because both apps already agree on these. + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals("wave", m.getJSONObject("stk").getString("id")) + assertEquals("https://example.org", m.getJSONObject("lnk").getString("url")) + assertEquals("delivered", m.getString("delivery")) + // The kind moves inside the attachment, and the sizes are spelled out. + val att = m.getJSONObject("att") + assertEquals(Attachment.IMAGE, att.getString("kind")) + assertEquals(800, att.getInt("width")) + assertEquals(600, att.getInt("height")) + assertEquals(12.5, att.getDouble("duration")) + assertFalse(att.has("w")) + assertFalse(att.has("h")) + } + + @Test + fun `groups get their own map and keep no prefix`() { + val shared = BackupPayload.toShared(internalPayload()) + assertTrue(shared.getJSONObject("groups").has("gid-1")) + assertFalse(shared.getJSONObject("threads").has("group:gid-1")) + } + + @Test + fun `a federated thread travels apart rather than as a broken peer id`() { + val shared = BackupPayload.toShared(internalPayload()) + // The iPhone keys threads by a numeric user id and skips anything else, + // so an address left in `threads` would be dropped in silence. Keeping + // it separate means an iPhone ignores it and this app gets it back. + assertFalse(shared.getJSONObject("threads").has("addr:someone@elsewhere.example")) + assertTrue(shared.getJSONObject("addrThreads").has("addr:someone@elsewhere.example")) + assertTrue(roundTrip().getJSONObject("threads").has("addr:someone@elsewhere.example")) + } + + @Test + fun `stories are keyed by author and expiries are milliseconds`() { + val shared = BackupPayload.toShared(internalPayload()) + val s = shared.getJSONObject("stories").getJSONArray("9").getJSONObject(0) + assertEquals(1_700_600_000_000L, s.getLong("expires")) + assertEquals("set-1", s.getJSONObject("set").getString("id")) + assertTrue(s.getBoolean("seen")) + assertTrue(s.has("att")) + assertFalse(s.has("ref")) + } + + @Test + fun `the call log renames ok to connected`() { + val c = BackupPayload.toShared(internalPayload()).getJSONArray("calls").getJSONObject(0) + assertTrue(c.getBoolean("connected")) + assertFalse(c.has("ok")) + } + + @Test + fun `a group secret keeps its version and its owner key`() { + val shared = BackupPayload.toShared(internalPayload()) + val g = shared.getJSONObject("groupSecrets").getJSONObject("gid-1") + // The relay refuses a roster rewrite whose version does not increase, + // so a restore that reset this would lock the group out of its rekey. + assertEquals(4, g.getInt("version")) + assertEquals("p", g.getString("ownerPrivate")) + assertEquals("p", roundTrip().getJSONObject("groupSecrets").getJSONObject("gid-1").getString("owner")) + } + + // -- And back again ------------------------------------------------------. + + @Test + fun `a message survives the round trip whole`() { + val m = roundTrip().getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("still on for saturday", m.getString("text")) + assertEquals("mid-1", m.getString("mid")) + assertEquals(1_700_000_000_000L, m.getLong("at")) + assertTrue(m.getBoolean("mine")) + assertEquals(7L, m.getLong("author")) + assertEquals(3600, m.getInt("secs")) + assertTrue(m.getBoolean("fwd")) + assertEquals(1_700_000_060_000L, m.getLong("edited")) + assertEquals("DELIVERED", m.getString("delivery")) + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals("wave", m.getJSONObject("stk").getString("id")) + assertEquals("https://example.org", m.getJSONObject("lnk").getString("url")) + assertEquals( + listOf(7L, 9L), + m.getJSONObject("rx").getJSONArray("fire").let { a -> + (0 until a.length()).map { a.getLong(it) } + }, + ) + val att = m.getJSONObject("att") + assertEquals(Attachment.IMAGE, m.getString("kind")) + assertEquals(800, att.getInt("w")) + assertEquals(600, att.getInt("h")) + assertEquals(12.5, att.getDouble("dur")) + assertEquals(4096, att.getInt("bytes")) + } + + @Test + fun `a story comes back in seconds, where the strip reads it`() { + val s = roundTrip().getJSONObject("stories").getJSONArray("items").getJSONObject(0) + assertEquals("story-1", s.getString("id")) + assertEquals(9L, s.getLong("author")) + // Seconds again. `isActive` compares against unix seconds, and a value + // a thousand times too large means an expired story never drops. + assertEquals(1_700_600_000.0, s.getDouble("expires")) + assertTrue(s.getBoolean("seen")) + assertEquals("set-1", s.getJSONObject("set").getString("id")) + assertEquals("https://example.org/s.m3u8", s.getString("stream")) + } + + @Test + fun `the roster and its off-server flag come back together`() { + val r = roundTrip().getJSONObject("roster") + assertTrue(r.getBoolean("offServer")) + assertEquals("ada", r.getJSONArray("items").getJSONObject(0).getString("handle")) + } + + @Test + fun `a numeric story id from the iphone becomes a usable name`() { + // The iPhone numbers its stories; this app names them. Dropping the id + // would make every imported story look new on every import, because + // the merge de-duplicates on exactly this. + val shared = + JSONObject().put("v", 1).put( + "stories", + JSONObject().put("9", JSONArray().put(JSONObject().put("id", 1_700_000_000).put("at", 1L))), + ) + val s = + BackupPayload + .fromShared(shared) + .getJSONObject("stories") + .getJSONArray("items") + .getJSONObject(0) + assertEquals("1700000000", s.getString("id")) + assertEquals(9L, s.getLong("author")) + } + + @Test + fun `an empty history makes an empty payload rather than nothing`() { + val shared = BackupPayload.toShared(JSONObject().put("threads", JSONObject())) + assertEquals(1, shared.getInt("v")) + assertFalse(shared.has("threads")) + assertEquals(0, BackupPayload.fromShared(shared).getJSONObject("threads").length()) + } + + @Test + fun `a backup from a newer app is refused rather than half read`() { + assertFailsWith { + BackupPayload.fromShared(JSONObject().put("v", 99)) + } + } + + @Test + fun `a message with nothing optional carries nothing optional`() { + val bare = + JSONObject().put( + "threads", + JSONObject().put("7", JSONArray().put(JSONObject().put("text", "hi").put("at", 1L))), + ) + val m = + BackupPayload + .toShared(bare) + .getJSONObject("threads") + .getJSONArray("7") + .getJSONObject(0) + assertFalse(m.has("lifetime")) + assertFalse(m.has("forwarded")) + assertFalse(m.has("att")) + assertFalse(m.has("re")) + assertFalse(m.has("delivery")) + assertNull(m.opt("mid")) + } + + // -- Through the real envelope -------------------------------------------. + + @Test + fun `a sealed backup opens again and is still whole`() { + val payload = BackupPayload.toShared(internalPayload()) + val file = Backup.create(payload.toString().toByteArray(Charsets.UTF_8), "correct horse battery") + val bytes = Backup.encodeFile(file) + + val back = Backup.decodeFile(bytes) + val plain = Backup.open(back, "correct horse battery") + val restored = BackupPayload.fromShared(JSONObject(String(plain, Charsets.UTF_8))) + + val m = restored.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("still on for saturday", m.getString("text")) + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals(3600, m.getInt("secs")) + } + + @Test + fun `the older android-only export is still recognisable by shape`() { + // Import has to accept both without being told which it was handed, and + // a file name is whatever the person saving it chose. So the shape + // decides: section 16 announces itself with `alg` and `ct`, the old one + // with `magic`. + val legacy = HistoryExport.seal(JSONObject().put("threads", JSONObject()), "pw-123456".toCharArray()) + assertFailsWith { Backup.decodeFile(legacy) } + + val modern = Backup.encodeFile(Backup.create("{}".toByteArray(), "pw-123456")) + assertFailsWith { HistoryExport.open(modern, "pw-123456".toCharArray()) } + } + + @Test + fun `garbage is not mistaken for a backup`() { + assertFailsWith { Backup.decodeFile("not json".toByteArray()) } + assertFailsWith { + Backup.decodeFile(JSONObject().put("v", 1).toString().toByteArray()) + } + } + + // -- Reading what the iPhone writes --------------------------------------. + + /** + * A payload in the shape `E2EBackupPayload` produces, transcribed from that + * encoder rather than generated by it. + * + * That is the weak point of this test and it is worth stating plainly: it + * proves this app reads the shape the iPhone is BELIEVED to write. Only a + * vector both apps are built against can prove they agree, and section 16's + * `backup-v1.json` pins the envelope alone, which is how the two shapes + * drifted apart in the first place. + */ + private fun asTheIphoneWritesIt(): JSONObject = + JSONObject( + """ + {"v":1, + "threads":{"7":[{"mine":true,"at":1700000000000,"text":"hello","mid":"m-1", + "reactions":{"fire":[7,9]},"edited":1700000060000,"forwarded":true,"lifetime":3600, + "re":{"mid":"m-0","author":9,"preview":"earlier"},"reStory":true, + "stk":{"pack":"classic","id":"wave"}, + "lnk":{"url":"https://example.org","title":"T","desc":"D"}, + "delivery":"delivered", + "att":{"kind":"image","id":"a-1","key":"AAAA","digest":"BBBB","bytes":4096, + "mime":"image/jpeg","width":800,"height":600,"duration":12.5}}]}, + "groups":{"gid-1":[{"mine":false,"at":1700000000000,"text":"in the group","author":9, + "mid":"g-1","mentionsMe":true}]}, + "stories":{"9":[{"id":1700000000,"author":9,"mine":false,"caption":"look", + "at":1700000000000,"expires":1700600000000,"key":"story-key", + "set":{"id":"set-1","i":1,"n":3}}]}, + "roster":[{"id":9,"handle":"ada","name":"Ada"}], + "rosterOffServer":true, + "calls":[{"peer":42,"at":1756400500000,"in":true,"connected":true}], + "groupNames":{"gid-1":"Saturday"}, + "groupSecrets":{"gid-1":{"tag":"t","key":"k","version":4,"ownerPrivate":"p"}}} + """.trimIndent(), + ) + + @Test + fun `an iphone backup lands in this app's own storage shape`() { + val out = BackupPayload.fromShared(asTheIphoneWritesIt()) + + val m = out.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("hello", m.getString("text")) + assertEquals(3600, m.getInt("secs"), "lifetime becomes this app's secs") + assertTrue(m.getBoolean("fwd"), "forwarded becomes this app's fwd") + assertTrue(m.has("rx"), "reactions become this app's rx") + assertEquals("DELIVERED", m.getString("delivery"), "stored upper case here") + assertEquals("image", m.getString("kind"), "the kind comes back out of the attachment") + assertEquals(800, m.getJSONObject("att").getInt("w")) + assertEquals(12.5, m.getJSONObject("att").getDouble("dur")) + + // A group thread goes back under the prefix this app keys them by. + val g = out.getJSONObject("threads").getJSONArray("group:gid-1").getJSONObject(0) + assertEquals(9L, g.getLong("author")) + + val s = out.getJSONObject("stories").getJSONArray("items").getJSONObject(0) + assertEquals("1700000000", s.getString("id")) + assertEquals(1_700_600_000.0, s.getDouble("expires"), "milliseconds there, seconds here") + + assertTrue(out.getJSONObject("roster").getBoolean("offServer")) + assertEquals("t", out.getJSONObject("groupSecrets").getJSONObject("gid-1").getString("tag")) + assertTrue(out.getJSONArray("callLog").getJSONObject(0).getBoolean("ok")) + } +}