Android writes and reads the shared backup format
Section 16 pins the envelope and says nothing about what is sealed, so each app sealed its own storage verbatim and stamped the file with the platform that wrote it. Each then refused the other's, in as many words: "import it on a device of the same kind". Somebody moving between an Android phone and an iPhone lost every conversation they had. The iPhone half landed first. This is the other half. `BackupPayload` translates rather than dumps, which is the entire point. This app stores a message under the keys its wire format uses (`secs`, `fwd`, `rx`), keeps one-to-one and group threads in one map under prefixed ids, hangs the attachment kind beside `att` rather than inside it, abbreviates the sizes, and holds expiries in unix seconds. The shared shape spells all of that out and counts in milliseconds. Exporting storage as-is is what made the two formats incompatible in the first place, so a copy would have reproduced the bug with more steps. Federated threads are the one thing that does not fit. This app keys them by address and the iPhone's history is keyed by numeric user id, with a decoder that skips anything else, so an address left in `threads` would be dropped in silence. They travel under `addrThreads` instead: an iPhone ignores the key, an Android phone gets them back, and nothing disappears without somebody having decided it should. Import tells the two formats apart by SHAPE, not by file name, because a name is whatever the person saving it chose. Section 16 announces itself with `alg` and `ct`; the older export announces itself with `magic`. A wrong passphrase on a section 16 file reports a wrong passphrase and does NOT fall through to the legacy reader, which would have answered "that file is not a Flamenet history export" for a file that is one and sent somebody hunting for corruption instead of a typo. Nothing writes the old format any more, and everything still reads it. Seventeen tests: the shared side against the key names the iPhone writes, the round trip back into this app's own storage shape, the unit conversions in both directions, and a payload transcribed from the iPhone's encoder. That last one is the weak point and the test says so: it proves this app reads the shape the iPhone is BELIEVED to write. Only a vector both apps are built against proves they agree, and `backup-v1.json` pins the envelope alone, which is exactly how the two shapes drifted apart. Full suite: 1271 tests, no failures. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Paul Hitt
co-authored by
Claude Opus 5
Claude Opus 5
parent
3bcb7d5900
commit
5a5e41bfa2
@@ -0,0 +1,401 @@
|
||||
package io.flamenet.e2e
|
||||
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFailsWith
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
/**
|
||||
* The shared backup shape, in both directions.
|
||||
*
|
||||
* A translation that loses something does not fail: it restores a shorter
|
||||
* conversation, which looks exactly like a backup of a shorter conversation.
|
||||
* So the round trip is asserted field by field rather than by eye, and the
|
||||
* shared side is asserted against the key names the iPhone actually writes,
|
||||
* because agreeing with ourselves is not interoperability.
|
||||
*/
|
||||
class BackupPayloadTest {
|
||||
private fun attachment(): JSONObject =
|
||||
JSONObject()
|
||||
.put("id", "att-1")
|
||||
.put("key", "a".repeat(43))
|
||||
.put("digest", "b".repeat(43))
|
||||
.put("bytes", 4096)
|
||||
.put("mime", "image/jpeg")
|
||||
.put("w", 800)
|
||||
.put("h", 600)
|
||||
.put("dur", 12.5)
|
||||
|
||||
private fun message(): JSONObject =
|
||||
JSONObject()
|
||||
.put("author", 7L)
|
||||
.put("mid", "mid-1")
|
||||
.put("text", "still on for saturday")
|
||||
.put("mine", true)
|
||||
.put("at", 1_700_000_000_000L)
|
||||
.put("secs", 3600)
|
||||
.put("delivery", "DELIVERED")
|
||||
.put("kind", Attachment.IMAGE)
|
||||
.put("att", attachment())
|
||||
.put("lnk", JSONObject().put("url", "https://example.org").put("title", "T").put("desc", "D"))
|
||||
.put("fwd", true)
|
||||
.put("re", JSONObject().put("mid", "mid-0").put("author", 9L).put("preview", "earlier"))
|
||||
.put("stk", JSONObject().put("pack", "classic").put("id", "wave"))
|
||||
.put("edited", 1_700_000_060_000L)
|
||||
.put("rx", JSONObject().put("fire", JSONArray(listOf(7L, 9L))))
|
||||
|
||||
private fun internalPayload(): JSONObject =
|
||||
JSONObject()
|
||||
.put(
|
||||
"threads",
|
||||
JSONObject()
|
||||
.put("7", JSONArray().put(message()))
|
||||
.put("group:gid-1", JSONArray().put(message()))
|
||||
.put("addr:someone@elsewhere.example", JSONArray().put(message())),
|
||||
).put(
|
||||
"callLog",
|
||||
JSONArray().put(
|
||||
JSONObject()
|
||||
.put("peer", 42L)
|
||||
.put("in", true)
|
||||
.put("ok", true)
|
||||
.put("at", 1_756_400_500_000L),
|
||||
),
|
||||
).put(
|
||||
"stories",
|
||||
JSONObject().put(
|
||||
"items",
|
||||
JSONArray().put(
|
||||
JSONObject()
|
||||
.put("id", "story-1")
|
||||
.put("author", 9L)
|
||||
.put("mine", false)
|
||||
.put("caption", "look")
|
||||
.put("at", 1_700_000_000_000L)
|
||||
.put("seen", true)
|
||||
.put("expires", 1_700_600_000.0)
|
||||
.put("ref", attachment())
|
||||
.put("gid", "gid-1")
|
||||
.put("stream", "https://example.org/s.m3u8")
|
||||
.put("key", "story-key")
|
||||
.put("set", JSONObject().put("id", "set-1").put("i", 1).put("n", 3)),
|
||||
),
|
||||
),
|
||||
).put(
|
||||
"roster",
|
||||
JSONObject()
|
||||
.put("items", JSONArray().put(JSONObject().put("id", 9L).put("handle", "ada").put("name", "Ada")))
|
||||
.put("offServer", true),
|
||||
).put("groupNames", JSONObject().put("gid-1", "Saturday"))
|
||||
.put(
|
||||
"groupSecrets",
|
||||
JSONObject().put(
|
||||
"gid-1",
|
||||
JSONObject()
|
||||
.put("tag", "t")
|
||||
.put("key", "k")
|
||||
.put("version", 4)
|
||||
.put("owner", "p"),
|
||||
),
|
||||
)
|
||||
|
||||
private fun roundTrip(): JSONObject = BackupPayload.fromShared(BackupPayload.toShared(internalPayload()))
|
||||
|
||||
// -- The shared side is what the iPhone reads ----------------------------.
|
||||
|
||||
@Test
|
||||
fun `the shared shape uses the names the iphone writes`() {
|
||||
val shared = BackupPayload.toShared(internalPayload())
|
||||
assertEquals(BackupPayload.VERSION, shared.getInt("v"))
|
||||
val m = shared.getJSONObject("threads").getJSONArray("7").getJSONObject(0)
|
||||
// Renamed on the way out. Writing this app's own key names is what made
|
||||
// the two formats incompatible to begin with.
|
||||
assertEquals(3600, m.getInt("lifetime"))
|
||||
assertTrue(m.getBoolean("forwarded"))
|
||||
assertEquals(
|
||||
setOf("fire"),
|
||||
m
|
||||
.getJSONObject("reactions")
|
||||
.keys()
|
||||
.asSequence()
|
||||
.toSet(),
|
||||
)
|
||||
assertFalse(m.has("secs"), "the wire key must not survive into the file")
|
||||
assertFalse(m.has("fwd"))
|
||||
assertFalse(m.has("rx"))
|
||||
// Carried through untouched, because both apps already agree on these.
|
||||
assertEquals("mid-0", m.getJSONObject("re").getString("mid"))
|
||||
assertEquals("wave", m.getJSONObject("stk").getString("id"))
|
||||
assertEquals("https://example.org", m.getJSONObject("lnk").getString("url"))
|
||||
assertEquals("delivered", m.getString("delivery"))
|
||||
// The kind moves inside the attachment, and the sizes are spelled out.
|
||||
val att = m.getJSONObject("att")
|
||||
assertEquals(Attachment.IMAGE, att.getString("kind"))
|
||||
assertEquals(800, att.getInt("width"))
|
||||
assertEquals(600, att.getInt("height"))
|
||||
assertEquals(12.5, att.getDouble("duration"))
|
||||
assertFalse(att.has("w"))
|
||||
assertFalse(att.has("h"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `groups get their own map and keep no prefix`() {
|
||||
val shared = BackupPayload.toShared(internalPayload())
|
||||
assertTrue(shared.getJSONObject("groups").has("gid-1"))
|
||||
assertFalse(shared.getJSONObject("threads").has("group:gid-1"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a federated thread travels apart rather than as a broken peer id`() {
|
||||
val shared = BackupPayload.toShared(internalPayload())
|
||||
// The iPhone keys threads by a numeric user id and skips anything else,
|
||||
// so an address left in `threads` would be dropped in silence. Keeping
|
||||
// it separate means an iPhone ignores it and this app gets it back.
|
||||
assertFalse(shared.getJSONObject("threads").has("addr:someone@elsewhere.example"))
|
||||
assertTrue(shared.getJSONObject("addrThreads").has("addr:someone@elsewhere.example"))
|
||||
assertTrue(roundTrip().getJSONObject("threads").has("addr:someone@elsewhere.example"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `stories are keyed by author and expiries are milliseconds`() {
|
||||
val shared = BackupPayload.toShared(internalPayload())
|
||||
val s = shared.getJSONObject("stories").getJSONArray("9").getJSONObject(0)
|
||||
assertEquals(1_700_600_000_000L, s.getLong("expires"))
|
||||
assertEquals("set-1", s.getJSONObject("set").getString("id"))
|
||||
assertTrue(s.getBoolean("seen"))
|
||||
assertTrue(s.has("att"))
|
||||
assertFalse(s.has("ref"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the call log renames ok to connected`() {
|
||||
val c = BackupPayload.toShared(internalPayload()).getJSONArray("calls").getJSONObject(0)
|
||||
assertTrue(c.getBoolean("connected"))
|
||||
assertFalse(c.has("ok"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a group secret keeps its version and its owner key`() {
|
||||
val shared = BackupPayload.toShared(internalPayload())
|
||||
val g = shared.getJSONObject("groupSecrets").getJSONObject("gid-1")
|
||||
// The relay refuses a roster rewrite whose version does not increase,
|
||||
// so a restore that reset this would lock the group out of its rekey.
|
||||
assertEquals(4, g.getInt("version"))
|
||||
assertEquals("p", g.getString("ownerPrivate"))
|
||||
assertEquals("p", roundTrip().getJSONObject("groupSecrets").getJSONObject("gid-1").getString("owner"))
|
||||
}
|
||||
|
||||
// -- And back again ------------------------------------------------------.
|
||||
|
||||
@Test
|
||||
fun `a message survives the round trip whole`() {
|
||||
val m = roundTrip().getJSONObject("threads").getJSONArray("7").getJSONObject(0)
|
||||
assertEquals("still on for saturday", m.getString("text"))
|
||||
assertEquals("mid-1", m.getString("mid"))
|
||||
assertEquals(1_700_000_000_000L, m.getLong("at"))
|
||||
assertTrue(m.getBoolean("mine"))
|
||||
assertEquals(7L, m.getLong("author"))
|
||||
assertEquals(3600, m.getInt("secs"))
|
||||
assertTrue(m.getBoolean("fwd"))
|
||||
assertEquals(1_700_000_060_000L, m.getLong("edited"))
|
||||
assertEquals("DELIVERED", m.getString("delivery"))
|
||||
assertEquals("mid-0", m.getJSONObject("re").getString("mid"))
|
||||
assertEquals("wave", m.getJSONObject("stk").getString("id"))
|
||||
assertEquals("https://example.org", m.getJSONObject("lnk").getString("url"))
|
||||
assertEquals(
|
||||
listOf(7L, 9L),
|
||||
m.getJSONObject("rx").getJSONArray("fire").let { a ->
|
||||
(0 until a.length()).map { a.getLong(it) }
|
||||
},
|
||||
)
|
||||
val att = m.getJSONObject("att")
|
||||
assertEquals(Attachment.IMAGE, m.getString("kind"))
|
||||
assertEquals(800, att.getInt("w"))
|
||||
assertEquals(600, att.getInt("h"))
|
||||
assertEquals(12.5, att.getDouble("dur"))
|
||||
assertEquals(4096, att.getInt("bytes"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a story comes back in seconds, where the strip reads it`() {
|
||||
val s = roundTrip().getJSONObject("stories").getJSONArray("items").getJSONObject(0)
|
||||
assertEquals("story-1", s.getString("id"))
|
||||
assertEquals(9L, s.getLong("author"))
|
||||
// Seconds again. `isActive` compares against unix seconds, and a value
|
||||
// a thousand times too large means an expired story never drops.
|
||||
assertEquals(1_700_600_000.0, s.getDouble("expires"))
|
||||
assertTrue(s.getBoolean("seen"))
|
||||
assertEquals("set-1", s.getJSONObject("set").getString("id"))
|
||||
assertEquals("https://example.org/s.m3u8", s.getString("stream"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the roster and its off-server flag come back together`() {
|
||||
val r = roundTrip().getJSONObject("roster")
|
||||
assertTrue(r.getBoolean("offServer"))
|
||||
assertEquals("ada", r.getJSONArray("items").getJSONObject(0).getString("handle"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a numeric story id from the iphone becomes a usable name`() {
|
||||
// The iPhone numbers its stories; this app names them. Dropping the id
|
||||
// would make every imported story look new on every import, because
|
||||
// the merge de-duplicates on exactly this.
|
||||
val shared =
|
||||
JSONObject().put("v", 1).put(
|
||||
"stories",
|
||||
JSONObject().put("9", JSONArray().put(JSONObject().put("id", 1_700_000_000).put("at", 1L))),
|
||||
)
|
||||
val s =
|
||||
BackupPayload
|
||||
.fromShared(shared)
|
||||
.getJSONObject("stories")
|
||||
.getJSONArray("items")
|
||||
.getJSONObject(0)
|
||||
assertEquals("1700000000", s.getString("id"))
|
||||
assertEquals(9L, s.getLong("author"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an empty history makes an empty payload rather than nothing`() {
|
||||
val shared = BackupPayload.toShared(JSONObject().put("threads", JSONObject()))
|
||||
assertEquals(1, shared.getInt("v"))
|
||||
assertFalse(shared.has("threads"))
|
||||
assertEquals(0, BackupPayload.fromShared(shared).getJSONObject("threads").length())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a backup from a newer app is refused rather than half read`() {
|
||||
assertFailsWith<BackupPayload.NewerBackup> {
|
||||
BackupPayload.fromShared(JSONObject().put("v", 99))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a message with nothing optional carries nothing optional`() {
|
||||
val bare =
|
||||
JSONObject().put(
|
||||
"threads",
|
||||
JSONObject().put("7", JSONArray().put(JSONObject().put("text", "hi").put("at", 1L))),
|
||||
)
|
||||
val m =
|
||||
BackupPayload
|
||||
.toShared(bare)
|
||||
.getJSONObject("threads")
|
||||
.getJSONArray("7")
|
||||
.getJSONObject(0)
|
||||
assertFalse(m.has("lifetime"))
|
||||
assertFalse(m.has("forwarded"))
|
||||
assertFalse(m.has("att"))
|
||||
assertFalse(m.has("re"))
|
||||
assertFalse(m.has("delivery"))
|
||||
assertNull(m.opt("mid"))
|
||||
}
|
||||
|
||||
// -- Through the real envelope -------------------------------------------.
|
||||
|
||||
@Test
|
||||
fun `a sealed backup opens again and is still whole`() {
|
||||
val payload = BackupPayload.toShared(internalPayload())
|
||||
val file = Backup.create(payload.toString().toByteArray(Charsets.UTF_8), "correct horse battery")
|
||||
val bytes = Backup.encodeFile(file)
|
||||
|
||||
val back = Backup.decodeFile(bytes)
|
||||
val plain = Backup.open(back, "correct horse battery")
|
||||
val restored = BackupPayload.fromShared(JSONObject(String(plain, Charsets.UTF_8)))
|
||||
|
||||
val m = restored.getJSONObject("threads").getJSONArray("7").getJSONObject(0)
|
||||
assertEquals("still on for saturday", m.getString("text"))
|
||||
assertEquals("mid-0", m.getJSONObject("re").getString("mid"))
|
||||
assertEquals(3600, m.getInt("secs"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the older android-only export is still recognisable by shape`() {
|
||||
// Import has to accept both without being told which it was handed, and
|
||||
// a file name is whatever the person saving it chose. So the shape
|
||||
// decides: section 16 announces itself with `alg` and `ct`, the old one
|
||||
// with `magic`.
|
||||
val legacy = HistoryExport.seal(JSONObject().put("threads", JSONObject()), "pw-123456".toCharArray())
|
||||
assertFailsWith<Backup.UnsupportedBackup> { Backup.decodeFile(legacy) }
|
||||
|
||||
val modern = Backup.encodeFile(Backup.create("{}".toByteArray(), "pw-123456"))
|
||||
assertFailsWith<HistoryExport.NotAnExport> { HistoryExport.open(modern, "pw-123456".toCharArray()) }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `garbage is not mistaken for a backup`() {
|
||||
assertFailsWith<Backup.UnsupportedBackup> { Backup.decodeFile("not json".toByteArray()) }
|
||||
assertFailsWith<Backup.UnsupportedBackup> {
|
||||
Backup.decodeFile(JSONObject().put("v", 1).toString().toByteArray())
|
||||
}
|
||||
}
|
||||
|
||||
// -- Reading what the iPhone writes --------------------------------------.
|
||||
|
||||
/**
|
||||
* A payload in the shape `E2EBackupPayload` produces, transcribed from that
|
||||
* encoder rather than generated by it.
|
||||
*
|
||||
* That is the weak point of this test and it is worth stating plainly: it
|
||||
* proves this app reads the shape the iPhone is BELIEVED to write. Only a
|
||||
* vector both apps are built against can prove they agree, and section 16's
|
||||
* `backup-v1.json` pins the envelope alone, which is how the two shapes
|
||||
* drifted apart in the first place.
|
||||
*/
|
||||
private fun asTheIphoneWritesIt(): JSONObject =
|
||||
JSONObject(
|
||||
"""
|
||||
{"v":1,
|
||||
"threads":{"7":[{"mine":true,"at":1700000000000,"text":"hello","mid":"m-1",
|
||||
"reactions":{"fire":[7,9]},"edited":1700000060000,"forwarded":true,"lifetime":3600,
|
||||
"re":{"mid":"m-0","author":9,"preview":"earlier"},"reStory":true,
|
||||
"stk":{"pack":"classic","id":"wave"},
|
||||
"lnk":{"url":"https://example.org","title":"T","desc":"D"},
|
||||
"delivery":"delivered",
|
||||
"att":{"kind":"image","id":"a-1","key":"AAAA","digest":"BBBB","bytes":4096,
|
||||
"mime":"image/jpeg","width":800,"height":600,"duration":12.5}}]},
|
||||
"groups":{"gid-1":[{"mine":false,"at":1700000000000,"text":"in the group","author":9,
|
||||
"mid":"g-1","mentionsMe":true}]},
|
||||
"stories":{"9":[{"id":1700000000,"author":9,"mine":false,"caption":"look",
|
||||
"at":1700000000000,"expires":1700600000000,"key":"story-key",
|
||||
"set":{"id":"set-1","i":1,"n":3}}]},
|
||||
"roster":[{"id":9,"handle":"ada","name":"Ada"}],
|
||||
"rosterOffServer":true,
|
||||
"calls":[{"peer":42,"at":1756400500000,"in":true,"connected":true}],
|
||||
"groupNames":{"gid-1":"Saturday"},
|
||||
"groupSecrets":{"gid-1":{"tag":"t","key":"k","version":4,"ownerPrivate":"p"}}}
|
||||
""".trimIndent(),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `an iphone backup lands in this app's own storage shape`() {
|
||||
val out = BackupPayload.fromShared(asTheIphoneWritesIt())
|
||||
|
||||
val m = out.getJSONObject("threads").getJSONArray("7").getJSONObject(0)
|
||||
assertEquals("hello", m.getString("text"))
|
||||
assertEquals(3600, m.getInt("secs"), "lifetime becomes this app's secs")
|
||||
assertTrue(m.getBoolean("fwd"), "forwarded becomes this app's fwd")
|
||||
assertTrue(m.has("rx"), "reactions become this app's rx")
|
||||
assertEquals("DELIVERED", m.getString("delivery"), "stored upper case here")
|
||||
assertEquals("image", m.getString("kind"), "the kind comes back out of the attachment")
|
||||
assertEquals(800, m.getJSONObject("att").getInt("w"))
|
||||
assertEquals(12.5, m.getJSONObject("att").getDouble("dur"))
|
||||
|
||||
// A group thread goes back under the prefix this app keys them by.
|
||||
val g = out.getJSONObject("threads").getJSONArray("group:gid-1").getJSONObject(0)
|
||||
assertEquals(9L, g.getLong("author"))
|
||||
|
||||
val s = out.getJSONObject("stories").getJSONArray("items").getJSONObject(0)
|
||||
assertEquals("1700000000", s.getString("id"))
|
||||
assertEquals(1_700_600_000.0, s.getDouble("expires"), "milliseconds there, seconds here")
|
||||
|
||||
assertTrue(out.getJSONObject("roster").getBoolean("offServer"))
|
||||
assertEquals("t", out.getJSONObject("groupSecrets").getJSONObject("gid-1").getString("tag"))
|
||||
assertTrue(out.getJSONArray("callLog").getJSONObject(0).getBoolean("ok"))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user