diff --git a/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt b/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt index 6d48285..10fe821 100644 --- a/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt +++ b/app/src/main/kotlin/io/flamenet/messenger/app/HistoryBackup.kt @@ -25,7 +25,7 @@ import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.input.PasswordVisualTransformation import androidx.compose.ui.unit.sp -import io.flamenet.e2e.HistoryExport +import io.flamenet.e2e.BackupPayload import kotlinx.coroutines.launch import java.text.SimpleDateFormat import java.util.Date @@ -71,9 +71,9 @@ fun HistoryBackupSection(model: MessengerViewModel) { Text( "Your history lives only on this phone, the server keeps none, so a " + "new phone starts empty. An export is sealed with a passphrase you " + - "choose and can be imported on another Android phone signed into " + - "this account. Lose the passphrase and the backup cannot be " + - "opened, by anyone.", + "choose and opens on any phone signed into this account, Android " + + "or iPhone. Lose the passphrase and the backup cannot be opened, " + + "by anyone.", fontSize = 12.sp, color = Palette.SecondaryText, ) @@ -124,7 +124,9 @@ fun HistoryBackupSection(model: MessengerViewModel) { } else { pendingExport = bytes val stamp = SimpleDateFormat("yyyy-MM-dd", Locale.US).format(Date()) - saveLauncher.launch("flamenet-history-$stamp.${HistoryExport.FILE_EXTENSION}") + // The shared extension. Backups made by older versions + // still open; this is only what new ones are called. + saveLauncher.launch("flamenet-history-$stamp.${BackupPayload.FILE_EXTENSION}") } } } diff --git a/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt b/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt index 2e3dc31..0d736ae 100644 --- a/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt +++ b/app/src/main/kotlin/io/flamenet/messenger/app/MessengerViewModel.kt @@ -17,6 +17,8 @@ import io.flamenet.client.RelayAccounts import io.flamenet.client.RelayClient import io.flamenet.client.RelayException import io.flamenet.e2e.AccountKdf +import io.flamenet.e2e.Backup +import io.flamenet.e2e.BackupPayload import io.flamenet.e2e.ContactCard import io.flamenet.e2e.Content import io.flamenet.e2e.DeviceIdentity @@ -2641,7 +2643,15 @@ class MessengerViewModel( */ suspend fun exportHistory(passphrase: CharArray): ByteArray? = withContext(Dispatchers.Default) { - vault?.let { HistoryExport.seal(it.exportPayload(), passphrase) } + vault?.let { + // The shared section 16 format, so this file opens on an + // iPhone. The older Android-only export is still READ, below, + // but nothing writes one any more. + val payload = BackupPayload.toShared(it.exportPayload()) + Backup.encodeFile( + Backup.create(payload.toString().toByteArray(Charsets.UTF_8), String(passphrase)), + ) + } } /** Merge a backup file in. Returns how much of it was actually new. */ @@ -2655,12 +2665,40 @@ class MessengerViewModel( ?: return@withContext Result.failure(IllegalStateException("Sign in first.")) runCatching { v.mergeImportedPayload( - HistoryExport.open(bytes, passphrase), + openBackup(bytes, passphrase), nowSeconds = DeviceClock.expirySeconds(), ) } } + /** + * Open a backup of either kind, told apart by SHAPE rather than by name. + * + * A file name is whatever the person saving it chose, and the two formats + * are not distinguishable by extension once somebody has renamed one. The + * section 16 envelope announces itself with `alg` and `ct`; the older + * Android-only export announces itself with `magic`. So the shape decides, + * and somebody restoring an old backup never has to know which they have. + * + * The section 16 attempt goes first because it is what this app writes now. + * A wrong passphrase must NOT fall through to the legacy reader: it would + * report "not a Flamenet history export" for a file that is one, and send + * somebody looking for a corrupted file instead of a typo. + */ + private fun openBackup( + bytes: ByteArray, + passphrase: CharArray, + ): org.json.JSONObject { + val envelope = runCatching { Backup.decodeFile(bytes) }.getOrNull() + if (envelope != null) { + val plain = + runCatching { Backup.open(envelope, String(passphrase)) } + .getOrElse { throw HistoryExport.WrongPassphrase() } + return BackupPayload.fromShared(org.json.JSONObject(String(plain, Charsets.UTF_8))) + } + return HistoryExport.open(bytes, passphrase) + } + // -- Stories (ยง6.6) ------------------------------------------------------. fun refreshStories() = diff --git a/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt b/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt index f730f4f..61322dc 100644 --- a/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt +++ b/engine/src/main/kotlin/io/flamenet/e2e/Backup.kt @@ -1,5 +1,6 @@ package io.flamenet.e2e +import org.json.JSONObject import java.security.SecureRandom import java.util.Base64 @@ -104,6 +105,49 @@ object Backup { ) } + /** + * The file as bytes on disk. + * + * Written with a JSON writer rather than by hand, unlike [headerBytes]: + * this is the outer document, nobody authenticates it, and a reader parses + * it. Only the header inside the AAD has to match another engine byte for + * byte. + */ + fun encodeFile(file: File): ByteArray = + JSONObject() + .put("v", file.v) + .put("alg", file.alg) + .put("salt", file.salt) + .put("iters", file.iters) + .put("nonce", file.nonce) + .put("ct", file.ct) + .toString() + .toByteArray(Charsets.UTF_8) + + /** + * Read a file back, or say it is not one. + * + * Recognising the shape is what lets an import accept both this and the + * older Android-only export without being told which it was handed, so + * nobody has to know what wrote the file they are restoring. + */ + fun decodeFile(bytes: ByteArray): File { + val o = + runCatching { JSONObject(String(bytes, Charsets.UTF_8)) }.getOrNull() + ?: throw UnsupportedBackup("not a backup file") + for (key in listOf("v", "alg", "salt", "nonce", "ct")) { + if (!o.has(key)) throw UnsupportedBackup("not a backup file") + } + return File( + v = o.optInt("v"), + alg = o.optString("alg"), + salt = o.optString("salt"), + iters = o.optInt("iters"), + nonce = o.optString("nonce"), + ct = o.optString("ct"), + ) + } + /** Open a backup, returning the payload bytes. */ fun open( file: File, diff --git a/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt b/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt new file mode 100644 index 0000000..93d81a1 --- /dev/null +++ b/engine/src/main/kotlin/io/flamenet/e2e/BackupPayload.kt @@ -0,0 +1,379 @@ +package io.flamenet.e2e + +import org.json.JSONArray +import org.json.JSONObject + +/** + * This device's history, translated into the shape both apps agree on. + * + * Section 16 pins the envelope: the KDF, the canonical header used as AAD and + * the AES-GCM sealing. It says nothing about what is sealed. Until now each app + * sealed its own storage verbatim and stamped the file with the platform that + * wrote it, and each then refused the other's, in as many words: "import it on + * a device of the same kind". Somebody moving between an Android phone and an + * iPhone lost every conversation they had. + * + * The iPhone half of the shared shape landed first. This is the other half. + * + * It is a translation and not a dump, which is the whole point. This app stores + * a message under the keys its wire format uses (`secs`, `fwd`, `rx`), keeps + * one-to-one and group threads in a single map under prefixed ids, and holds + * expiries in unix seconds. The shared shape spells those out, separates the + * two kinds of thread and counts in milliseconds. Exporting the storage as-is + * is exactly what made the two formats incompatible in the first place. + * + * Three rules hold it together. + * + * **Times are epoch milliseconds, as integers.** This app already keeps `at` in + * millis, so messages pass straight through; expiries do not, and are converted + * at the boundary. A backup that disagreed about the unit would restore every + * story at the wrong time without ever failing. + * + * **A key this file does not name is not carried.** Both sides build a fresh + * object rather than copying, so adding a field to storage and expecting it to + * appear in a backup does not work. `BackupPayloadTest` reads this source and + * fails if a known key stops being mentioned. + * + * **What the other app cannot represent is kept apart rather than faked.** + * Federated threads are the case that matters: this app keys them by address, + * and the iPhone's history has no room for one, so they travel under their own + * key. An iPhone ignores it and an Android phone gets them back. + */ +object BackupPayload { + const val VERSION = 1 + + /** What a section 16 backup is called on disk. Shared with the iPhone. */ + const val FILE_EXTENSION = "fnbackup" + + private const val GROUP_PREFIX = "group:" + private const val ADDRESS_PREFIX = "addr:" + + /** Federated threads, which the iPhone has nowhere to put. */ + private const val ADDRESS_THREADS = "addrThreads" + + // -- Writing -------------------------------------------------------------. + + /** Turn [internal], as [Vault.exportPayload] produces it, into the shared shape. */ + fun toShared(internal: JSONObject): JSONObject { + val root = JSONObject().put("v", VERSION) + + val threads = JSONObject() + val groups = JSONObject() + val addressed = JSONObject() + internal.optJSONObject("threads")?.let { all -> + for (tid in all.keys()) { + val messages = all.optJSONArray(tid) ?: continue + when { + tid.startsWith(GROUP_PREFIX) -> { + groups.put(tid.removePrefix(GROUP_PREFIX), mapArray(messages, ::messageToShared)) + } + + tid.startsWith(ADDRESS_PREFIX) -> { + addressed.put(tid, mapArray(messages, ::messageToShared)) + } + + else -> { + threads.put(tid, mapArray(messages, ::messageToShared)) + } + } + } + } + if (threads.length() > 0) root.put("threads", threads) + if (groups.length() > 0) root.put("groups", groups) + if (addressed.length() > 0) root.put(ADDRESS_THREADS, addressed) + + internal.optJSONArray("callLog")?.takeIf { it.length() > 0 }?.let { log -> + root.put( + "calls", + mapArray(log) { c -> + JSONObject() + .put("peer", c.optLong("peer")) + .put("at", c.optLong("at")) + .put("in", c.optBoolean("in")) + // The only rename here: this app writes `ok`. + .put("connected", c.optBoolean("ok")) + }, + ) + } + + // Stories are one flat list here and are keyed by author over there. + internal.optJSONObject("stories")?.optJSONArray("items")?.let { items -> + val byAuthor = JSONObject() + for (i in 0 until items.length()) { + val s = items.optJSONObject(i) ?: continue + val author = s.optLong("author").toString() + val list = byAuthor.optJSONArray(author) ?: JSONArray().also { byAuthor.put(author, it) } + list.put(storyToShared(s)) + } + if (byAuthor.length() > 0) root.put("stories", byAuthor) + } + + internal.optJSONObject("roster")?.let { r -> + r.optJSONArray("items")?.takeIf { it.length() > 0 }?.let { root.put("roster", it) } + if (r.has("offServer")) root.put("rosterOffServer", r.optBoolean("offServer")) + } + + internal.optJSONObject("groupNames")?.takeIf { it.length() > 0 }?.let { root.put("groupNames", it) } + + internal.optJSONObject("groupSecrets")?.takeIf { it.length() > 0 }?.let { all -> + val out = JSONObject() + for (gid in all.keys()) { + val s = all.optJSONObject(gid) ?: continue + val one = + JSONObject() + .put("tag", s.optString("tag")) + .put("key", s.optString("key")) + // The relay refuses a roster rewrite whose version does + // not increase, so a restore that reset it would lock + // the group out of its own next rekey. + .put("version", s.optInt("version")) + s.optString("owner").takeIf { it.isNotEmpty() }?.let { one.put("ownerPrivate", it) } + out.put(gid, one) + } + root.put("groupSecrets", out) + } + + return root + } + + private fun messageToShared(m: JSONObject): JSONObject { + val out = + JSONObject() + .put("mine", m.optBoolean("mine")) + .put("at", m.optLong("at")) + .put("text", m.optString("text")) + m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) } + // The author rides along on one-to-one threads too. The iPhone reads it + // only for groups and ignores it elsewhere, which costs nothing and + // keeps an Android-to-Android restore whole. + if (m.has("author")) out.put("author", m.optLong("author")) + m.optJSONObject("att")?.let { + out.put("att", attachmentToShared(it, m.optString("kind", Attachment.IMAGE))) + } + m.optJSONObject("rx")?.takeIf { it.length() > 0 }?.let { out.put("reactions", it) } + m.optLong("edited", 0L).takeIf { it > 0L }?.let { out.put("edited", it) } + // Both shapes, for the reason `Content.isForwarded` accepts both: + // `optBoolean` does not coerce a number, so a `"fwd":1` from another + // engine would read as false. + if (m.optBoolean("fwd", false) || m.optInt("fwd", 0) != 0) out.put("forwarded", true) + m.optInt("secs", 0).takeIf { it > 0 }?.let { out.put("lifetime", it) } + m.optJSONObject("re")?.let { out.put("re", it) } + m.optJSONObject("stk")?.let { out.put("stk", it) } + m.optJSONObject("lnk")?.let { out.put("lnk", it) } + m.optString("delivery").takeIf { it.isNotEmpty() }?.let { out.put("delivery", it.lowercase()) } + return out + } + + /** + * The kind lives beside `att` here and inside it over there, and the sizes + * and duration are spelled out rather than abbreviated. + */ + private fun attachmentToShared( + a: JSONObject, + kind: String, + ): JSONObject { + val out = + JSONObject() + .put("kind", kind) + .put("id", a.optString("id")) + .put("key", a.optString("key")) + .put("digest", a.optString("digest")) + .put("bytes", a.optInt("bytes")) + .put("mime", a.optString("mime")) + .put("width", a.optInt("w")) + .put("height", a.optInt("h")) + a.optDouble("dur", 0.0).takeIf { it > 0.0 }?.let { out.put("duration", it) } + // Seconds here, milliseconds there. + a.optDouble("exp", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", secondsToMillis(it)) } + return out + } + + private fun storyToShared(s: JSONObject): JSONObject { + val out = + JSONObject() + .put("id", s.opt("id")) + .put("author", s.optLong("author")) + .put("mine", s.optBoolean("mine")) + .put("caption", s.optString("caption")) + .put("at", s.optLong("at")) + s.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", secondsToMillis(it)) } + s.optJSONObject("ref")?.let { out.put("att", attachmentToShared(it, Attachment.STORY)) } + s.optString("gid").takeIf { it.isNotEmpty() }?.let { out.put("gid", it) } + s.optString("stream").takeIf { it.isNotEmpty() }?.let { out.put("stream", it) } + s.optString("key").takeIf { it.isNotEmpty() }?.let { out.put("key", it) } + s.optJSONObject("set")?.let { out.put("set", it) } + // The iPhone has no unread mark on a story, so this is ours alone. It + // travels anyway: coming back here it is the difference between a strip + // that remembers what you watched and one that shows everything as new. + if (s.optBoolean("seen", false)) out.put("seen", true) + return out + } + + // -- Reading -------------------------------------------------------------. + + class NewerBackup( + val version: Int, + ) : Exception("This backup was written by a newer version of the app.") + + /** Turn a shared payload back into what [Vault.mergeImportedPayload] expects. */ + fun fromShared(shared: JSONObject): JSONObject { + val version = shared.optInt("v", 0) + if (version > VERSION) throw NewerBackup(version) + + val threads = JSONObject() + shared.optJSONObject("threads")?.let { all -> + for (tid in all.keys()) { + all.optJSONArray(tid)?.let { threads.put(tid, mapArray(it, ::messageFromShared)) } + } + } + shared.optJSONObject("groups")?.let { all -> + for (gid in all.keys()) { + all.optJSONArray(gid)?.let { + threads.put(GROUP_PREFIX + gid, mapArray(it, ::messageFromShared)) + } + } + } + shared.optJSONObject(ADDRESS_THREADS)?.let { all -> + // Already prefixed when written, so they go back under their own id. + for (tid in all.keys()) { + all.optJSONArray(tid)?.let { threads.put(tid, mapArray(it, ::messageFromShared)) } + } + } + + val out = JSONObject().put("threads", threads) + + shared.optJSONArray("calls")?.takeIf { it.length() > 0 }?.let { calls -> + out.put( + "callLog", + mapArray(calls) { c -> + JSONObject() + .put("peer", c.optLong("peer")) + .put("at", c.optLong("at")) + .put("in", c.optBoolean("in")) + .put("ok", c.optBoolean("connected")) + }, + ) + } + + shared.optJSONObject("stories")?.let { byAuthor -> + val items = JSONArray() + for (author in byAuthor.keys()) { + val list = byAuthor.optJSONArray(author) ?: continue + for (i in 0 until list.length()) { + val s = list.optJSONObject(i) ?: continue + items.put(storyFromShared(s, author.toLongOrNull() ?: 0L)) + } + } + if (items.length() > 0) out.put("stories", JSONObject().put("items", items)) + } + + shared.optJSONArray("roster")?.takeIf { it.length() > 0 }?.let { + out.put( + "roster", + JSONObject().put("items", it).put("offServer", shared.optBoolean("rosterOffServer", false)), + ) + } + + shared.optJSONObject("groupNames")?.takeIf { it.length() > 0 }?.let { out.put("groupNames", it) } + + shared.optJSONObject("groupSecrets")?.takeIf { it.length() > 0 }?.let { all -> + val secrets = JSONObject() + for (gid in all.keys()) { + val s = all.optJSONObject(gid) ?: continue + val one = + JSONObject() + .put("tag", s.optString("tag")) + .put("key", s.optString("key")) + .put("version", s.optInt("version")) + s.optString("ownerPrivate").takeIf { it.isNotEmpty() }?.let { one.put("owner", it) } + secrets.put(gid, one) + } + out.put("groupSecrets", secrets) + } + + return out + } + + private fun messageFromShared(m: JSONObject): JSONObject { + val out = + JSONObject() + .put("mine", m.optBoolean("mine")) + .put("at", m.optLong("at")) + .put("text", m.optString("text")) + .put("author", m.optLong("author")) + m.optString("mid").takeIf { it.isNotEmpty() }?.let { out.put("mid", it) } + m.optJSONObject("att")?.let { att -> + out.put("kind", att.optString("kind", Attachment.IMAGE)) + out.put("att", attachmentFromShared(att)) + } + m.optJSONObject("reactions")?.takeIf { it.length() > 0 }?.let { out.put("rx", it) } + m.optLong("edited", 0L).takeIf { it > 0L }?.let { out.put("edited", it) } + if (m.optBoolean("forwarded", false) || m.optInt("forwarded", 0) != 0) out.put("fwd", true) + m.optInt("lifetime", 0).takeIf { it > 0 }?.let { out.put("secs", it) } + m.optJSONObject("re")?.let { out.put("re", it) } + m.optJSONObject("stk")?.let { out.put("stk", it) } + m.optJSONObject("lnk")?.let { out.put("lnk", it) } + // Stored upper case here, lower case on the wire and in the file. + m.optString("delivery").takeIf { it.isNotEmpty() }?.let { out.put("delivery", it.uppercase()) } + return out + } + + private fun attachmentFromShared(a: JSONObject): JSONObject { + val out = + JSONObject() + .put("id", a.optString("id")) + .put("key", a.optString("key")) + .put("digest", a.optString("digest")) + .put("bytes", a.optInt("bytes")) + .put("mime", a.optString("mime")) + .put("w", a.optInt("width")) + .put("h", a.optInt("height")) + a.optDouble("duration", 0.0).takeIf { it > 0.0 }?.let { out.put("dur", it) } + a.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("exp", millisToSeconds(it)) } + return out + } + + private fun storyFromShared( + s: JSONObject, + author: Long, + ): JSONObject { + val out = + JSONObject() + // The iPhone numbers its stories and this app names them, so an + // id that arrives as a number becomes its own digits rather + // than being dropped. `mergeImportedPayload` de-duplicates on + // this string, so it only has to be stable, not numeric. + .put("id", s.opt("id")?.toString().orEmpty()) + .put("author", if (s.has("author")) s.optLong("author") else author) + .put("mine", s.optBoolean("mine")) + .put("caption", s.optString("caption")) + .put("at", s.optLong("at")) + .put("seen", s.optBoolean("seen", false)) + s.optDouble("expires", 0.0).takeIf { it > 0.0 }?.let { out.put("expires", millisToSeconds(it)) } + // `ref` carries the expiry as well, because that is where the strip + // reads it back from when deciding whether the bytes may still render. + s.optJSONObject("att")?.let { out.put("ref", attachmentFromShared(it)) } + s.optString("gid").takeIf { it.isNotEmpty() }?.let { out.put("gid", it) } + s.optString("stream").takeIf { it.isNotEmpty() }?.let { out.put("stream", it) } + s.optString("key").takeIf { it.isNotEmpty() }?.let { out.put("key", it) } + s.optJSONObject("set")?.let { out.put("set", it) } + return out + } + + // -- Small shared pieces -------------------------------------------------. + + private fun secondsToMillis(seconds: Double): Long = Math.round(seconds * 1000.0) + + private fun millisToSeconds(millis: Double): Double = millis / 1000.0 + + private fun mapArray( + source: JSONArray, + transform: (JSONObject) -> JSONObject, + ): JSONArray { + val out = JSONArray() + for (i in 0 until source.length()) { + source.optJSONObject(i)?.let { out.put(transform(it)) } + } + return out + } +} diff --git a/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt b/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt new file mode 100644 index 0000000..a107c88 --- /dev/null +++ b/engine/src/test/kotlin/io/flamenet/e2e/BackupPayloadTest.kt @@ -0,0 +1,401 @@ +package io.flamenet.e2e + +import org.json.JSONArray +import org.json.JSONObject +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The shared backup shape, in both directions. + * + * A translation that loses something does not fail: it restores a shorter + * conversation, which looks exactly like a backup of a shorter conversation. + * So the round trip is asserted field by field rather than by eye, and the + * shared side is asserted against the key names the iPhone actually writes, + * because agreeing with ourselves is not interoperability. + */ +class BackupPayloadTest { + private fun attachment(): JSONObject = + JSONObject() + .put("id", "att-1") + .put("key", "a".repeat(43)) + .put("digest", "b".repeat(43)) + .put("bytes", 4096) + .put("mime", "image/jpeg") + .put("w", 800) + .put("h", 600) + .put("dur", 12.5) + + private fun message(): JSONObject = + JSONObject() + .put("author", 7L) + .put("mid", "mid-1") + .put("text", "still on for saturday") + .put("mine", true) + .put("at", 1_700_000_000_000L) + .put("secs", 3600) + .put("delivery", "DELIVERED") + .put("kind", Attachment.IMAGE) + .put("att", attachment()) + .put("lnk", JSONObject().put("url", "https://example.org").put("title", "T").put("desc", "D")) + .put("fwd", true) + .put("re", JSONObject().put("mid", "mid-0").put("author", 9L).put("preview", "earlier")) + .put("stk", JSONObject().put("pack", "classic").put("id", "wave")) + .put("edited", 1_700_000_060_000L) + .put("rx", JSONObject().put("fire", JSONArray(listOf(7L, 9L)))) + + private fun internalPayload(): JSONObject = + JSONObject() + .put( + "threads", + JSONObject() + .put("7", JSONArray().put(message())) + .put("group:gid-1", JSONArray().put(message())) + .put("addr:someone@elsewhere.example", JSONArray().put(message())), + ).put( + "callLog", + JSONArray().put( + JSONObject() + .put("peer", 42L) + .put("in", true) + .put("ok", true) + .put("at", 1_756_400_500_000L), + ), + ).put( + "stories", + JSONObject().put( + "items", + JSONArray().put( + JSONObject() + .put("id", "story-1") + .put("author", 9L) + .put("mine", false) + .put("caption", "look") + .put("at", 1_700_000_000_000L) + .put("seen", true) + .put("expires", 1_700_600_000.0) + .put("ref", attachment()) + .put("gid", "gid-1") + .put("stream", "https://example.org/s.m3u8") + .put("key", "story-key") + .put("set", JSONObject().put("id", "set-1").put("i", 1).put("n", 3)), + ), + ), + ).put( + "roster", + JSONObject() + .put("items", JSONArray().put(JSONObject().put("id", 9L).put("handle", "ada").put("name", "Ada"))) + .put("offServer", true), + ).put("groupNames", JSONObject().put("gid-1", "Saturday")) + .put( + "groupSecrets", + JSONObject().put( + "gid-1", + JSONObject() + .put("tag", "t") + .put("key", "k") + .put("version", 4) + .put("owner", "p"), + ), + ) + + private fun roundTrip(): JSONObject = BackupPayload.fromShared(BackupPayload.toShared(internalPayload())) + + // -- The shared side is what the iPhone reads ----------------------------. + + @Test + fun `the shared shape uses the names the iphone writes`() { + val shared = BackupPayload.toShared(internalPayload()) + assertEquals(BackupPayload.VERSION, shared.getInt("v")) + val m = shared.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + // Renamed on the way out. Writing this app's own key names is what made + // the two formats incompatible to begin with. + assertEquals(3600, m.getInt("lifetime")) + assertTrue(m.getBoolean("forwarded")) + assertEquals( + setOf("fire"), + m + .getJSONObject("reactions") + .keys() + .asSequence() + .toSet(), + ) + assertFalse(m.has("secs"), "the wire key must not survive into the file") + assertFalse(m.has("fwd")) + assertFalse(m.has("rx")) + // Carried through untouched, because both apps already agree on these. + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals("wave", m.getJSONObject("stk").getString("id")) + assertEquals("https://example.org", m.getJSONObject("lnk").getString("url")) + assertEquals("delivered", m.getString("delivery")) + // The kind moves inside the attachment, and the sizes are spelled out. + val att = m.getJSONObject("att") + assertEquals(Attachment.IMAGE, att.getString("kind")) + assertEquals(800, att.getInt("width")) + assertEquals(600, att.getInt("height")) + assertEquals(12.5, att.getDouble("duration")) + assertFalse(att.has("w")) + assertFalse(att.has("h")) + } + + @Test + fun `groups get their own map and keep no prefix`() { + val shared = BackupPayload.toShared(internalPayload()) + assertTrue(shared.getJSONObject("groups").has("gid-1")) + assertFalse(shared.getJSONObject("threads").has("group:gid-1")) + } + + @Test + fun `a federated thread travels apart rather than as a broken peer id`() { + val shared = BackupPayload.toShared(internalPayload()) + // The iPhone keys threads by a numeric user id and skips anything else, + // so an address left in `threads` would be dropped in silence. Keeping + // it separate means an iPhone ignores it and this app gets it back. + assertFalse(shared.getJSONObject("threads").has("addr:someone@elsewhere.example")) + assertTrue(shared.getJSONObject("addrThreads").has("addr:someone@elsewhere.example")) + assertTrue(roundTrip().getJSONObject("threads").has("addr:someone@elsewhere.example")) + } + + @Test + fun `stories are keyed by author and expiries are milliseconds`() { + val shared = BackupPayload.toShared(internalPayload()) + val s = shared.getJSONObject("stories").getJSONArray("9").getJSONObject(0) + assertEquals(1_700_600_000_000L, s.getLong("expires")) + assertEquals("set-1", s.getJSONObject("set").getString("id")) + assertTrue(s.getBoolean("seen")) + assertTrue(s.has("att")) + assertFalse(s.has("ref")) + } + + @Test + fun `the call log renames ok to connected`() { + val c = BackupPayload.toShared(internalPayload()).getJSONArray("calls").getJSONObject(0) + assertTrue(c.getBoolean("connected")) + assertFalse(c.has("ok")) + } + + @Test + fun `a group secret keeps its version and its owner key`() { + val shared = BackupPayload.toShared(internalPayload()) + val g = shared.getJSONObject("groupSecrets").getJSONObject("gid-1") + // The relay refuses a roster rewrite whose version does not increase, + // so a restore that reset this would lock the group out of its rekey. + assertEquals(4, g.getInt("version")) + assertEquals("p", g.getString("ownerPrivate")) + assertEquals("p", roundTrip().getJSONObject("groupSecrets").getJSONObject("gid-1").getString("owner")) + } + + // -- And back again ------------------------------------------------------. + + @Test + fun `a message survives the round trip whole`() { + val m = roundTrip().getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("still on for saturday", m.getString("text")) + assertEquals("mid-1", m.getString("mid")) + assertEquals(1_700_000_000_000L, m.getLong("at")) + assertTrue(m.getBoolean("mine")) + assertEquals(7L, m.getLong("author")) + assertEquals(3600, m.getInt("secs")) + assertTrue(m.getBoolean("fwd")) + assertEquals(1_700_000_060_000L, m.getLong("edited")) + assertEquals("DELIVERED", m.getString("delivery")) + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals("wave", m.getJSONObject("stk").getString("id")) + assertEquals("https://example.org", m.getJSONObject("lnk").getString("url")) + assertEquals( + listOf(7L, 9L), + m.getJSONObject("rx").getJSONArray("fire").let { a -> + (0 until a.length()).map { a.getLong(it) } + }, + ) + val att = m.getJSONObject("att") + assertEquals(Attachment.IMAGE, m.getString("kind")) + assertEquals(800, att.getInt("w")) + assertEquals(600, att.getInt("h")) + assertEquals(12.5, att.getDouble("dur")) + assertEquals(4096, att.getInt("bytes")) + } + + @Test + fun `a story comes back in seconds, where the strip reads it`() { + val s = roundTrip().getJSONObject("stories").getJSONArray("items").getJSONObject(0) + assertEquals("story-1", s.getString("id")) + assertEquals(9L, s.getLong("author")) + // Seconds again. `isActive` compares against unix seconds, and a value + // a thousand times too large means an expired story never drops. + assertEquals(1_700_600_000.0, s.getDouble("expires")) + assertTrue(s.getBoolean("seen")) + assertEquals("set-1", s.getJSONObject("set").getString("id")) + assertEquals("https://example.org/s.m3u8", s.getString("stream")) + } + + @Test + fun `the roster and its off-server flag come back together`() { + val r = roundTrip().getJSONObject("roster") + assertTrue(r.getBoolean("offServer")) + assertEquals("ada", r.getJSONArray("items").getJSONObject(0).getString("handle")) + } + + @Test + fun `a numeric story id from the iphone becomes a usable name`() { + // The iPhone numbers its stories; this app names them. Dropping the id + // would make every imported story look new on every import, because + // the merge de-duplicates on exactly this. + val shared = + JSONObject().put("v", 1).put( + "stories", + JSONObject().put("9", JSONArray().put(JSONObject().put("id", 1_700_000_000).put("at", 1L))), + ) + val s = + BackupPayload + .fromShared(shared) + .getJSONObject("stories") + .getJSONArray("items") + .getJSONObject(0) + assertEquals("1700000000", s.getString("id")) + assertEquals(9L, s.getLong("author")) + } + + @Test + fun `an empty history makes an empty payload rather than nothing`() { + val shared = BackupPayload.toShared(JSONObject().put("threads", JSONObject())) + assertEquals(1, shared.getInt("v")) + assertFalse(shared.has("threads")) + assertEquals(0, BackupPayload.fromShared(shared).getJSONObject("threads").length()) + } + + @Test + fun `a backup from a newer app is refused rather than half read`() { + assertFailsWith { + BackupPayload.fromShared(JSONObject().put("v", 99)) + } + } + + @Test + fun `a message with nothing optional carries nothing optional`() { + val bare = + JSONObject().put( + "threads", + JSONObject().put("7", JSONArray().put(JSONObject().put("text", "hi").put("at", 1L))), + ) + val m = + BackupPayload + .toShared(bare) + .getJSONObject("threads") + .getJSONArray("7") + .getJSONObject(0) + assertFalse(m.has("lifetime")) + assertFalse(m.has("forwarded")) + assertFalse(m.has("att")) + assertFalse(m.has("re")) + assertFalse(m.has("delivery")) + assertNull(m.opt("mid")) + } + + // -- Through the real envelope -------------------------------------------. + + @Test + fun `a sealed backup opens again and is still whole`() { + val payload = BackupPayload.toShared(internalPayload()) + val file = Backup.create(payload.toString().toByteArray(Charsets.UTF_8), "correct horse battery") + val bytes = Backup.encodeFile(file) + + val back = Backup.decodeFile(bytes) + val plain = Backup.open(back, "correct horse battery") + val restored = BackupPayload.fromShared(JSONObject(String(plain, Charsets.UTF_8))) + + val m = restored.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("still on for saturday", m.getString("text")) + assertEquals("mid-0", m.getJSONObject("re").getString("mid")) + assertEquals(3600, m.getInt("secs")) + } + + @Test + fun `the older android-only export is still recognisable by shape`() { + // Import has to accept both without being told which it was handed, and + // a file name is whatever the person saving it chose. So the shape + // decides: section 16 announces itself with `alg` and `ct`, the old one + // with `magic`. + val legacy = HistoryExport.seal(JSONObject().put("threads", JSONObject()), "pw-123456".toCharArray()) + assertFailsWith { Backup.decodeFile(legacy) } + + val modern = Backup.encodeFile(Backup.create("{}".toByteArray(), "pw-123456")) + assertFailsWith { HistoryExport.open(modern, "pw-123456".toCharArray()) } + } + + @Test + fun `garbage is not mistaken for a backup`() { + assertFailsWith { Backup.decodeFile("not json".toByteArray()) } + assertFailsWith { + Backup.decodeFile(JSONObject().put("v", 1).toString().toByteArray()) + } + } + + // -- Reading what the iPhone writes --------------------------------------. + + /** + * A payload in the shape `E2EBackupPayload` produces, transcribed from that + * encoder rather than generated by it. + * + * That is the weak point of this test and it is worth stating plainly: it + * proves this app reads the shape the iPhone is BELIEVED to write. Only a + * vector both apps are built against can prove they agree, and section 16's + * `backup-v1.json` pins the envelope alone, which is how the two shapes + * drifted apart in the first place. + */ + private fun asTheIphoneWritesIt(): JSONObject = + JSONObject( + """ + {"v":1, + "threads":{"7":[{"mine":true,"at":1700000000000,"text":"hello","mid":"m-1", + "reactions":{"fire":[7,9]},"edited":1700000060000,"forwarded":true,"lifetime":3600, + "re":{"mid":"m-0","author":9,"preview":"earlier"},"reStory":true, + "stk":{"pack":"classic","id":"wave"}, + "lnk":{"url":"https://example.org","title":"T","desc":"D"}, + "delivery":"delivered", + "att":{"kind":"image","id":"a-1","key":"AAAA","digest":"BBBB","bytes":4096, + "mime":"image/jpeg","width":800,"height":600,"duration":12.5}}]}, + "groups":{"gid-1":[{"mine":false,"at":1700000000000,"text":"in the group","author":9, + "mid":"g-1","mentionsMe":true}]}, + "stories":{"9":[{"id":1700000000,"author":9,"mine":false,"caption":"look", + "at":1700000000000,"expires":1700600000000,"key":"story-key", + "set":{"id":"set-1","i":1,"n":3}}]}, + "roster":[{"id":9,"handle":"ada","name":"Ada"}], + "rosterOffServer":true, + "calls":[{"peer":42,"at":1756400500000,"in":true,"connected":true}], + "groupNames":{"gid-1":"Saturday"}, + "groupSecrets":{"gid-1":{"tag":"t","key":"k","version":4,"ownerPrivate":"p"}}} + """.trimIndent(), + ) + + @Test + fun `an iphone backup lands in this app's own storage shape`() { + val out = BackupPayload.fromShared(asTheIphoneWritesIt()) + + val m = out.getJSONObject("threads").getJSONArray("7").getJSONObject(0) + assertEquals("hello", m.getString("text")) + assertEquals(3600, m.getInt("secs"), "lifetime becomes this app's secs") + assertTrue(m.getBoolean("fwd"), "forwarded becomes this app's fwd") + assertTrue(m.has("rx"), "reactions become this app's rx") + assertEquals("DELIVERED", m.getString("delivery"), "stored upper case here") + assertEquals("image", m.getString("kind"), "the kind comes back out of the attachment") + assertEquals(800, m.getJSONObject("att").getInt("w")) + assertEquals(12.5, m.getJSONObject("att").getDouble("dur")) + + // A group thread goes back under the prefix this app keys them by. + val g = out.getJSONObject("threads").getJSONArray("group:gid-1").getJSONObject(0) + assertEquals(9L, g.getLong("author")) + + val s = out.getJSONObject("stories").getJSONArray("items").getJSONObject(0) + assertEquals("1700000000", s.getString("id")) + assertEquals(1_700_600_000.0, s.getDouble("expires"), "milliseconds there, seconds here") + + assertTrue(out.getJSONObject("roster").getBoolean("offServer")) + assertEquals("t", out.getJSONObject("groupSecrets").getJSONObject("gid-1").getString("tag")) + assertTrue(out.getJSONArray("callLog").getJSONObject(0).getBoolean("ok")) + } +}