video calls: capture/render/signal + video buttons; story card sizing; Alerts off the nav; avatar live refresh; sealed-key eviction

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Paul Hitt
2026-08-28 15:11:52 -04:00
co-authored by Claude Opus 4.8
parent e7a3b60b32
commit cf3f43fdbf
9 changed files with 359 additions and 34 deletions
+4
View File
@@ -29,6 +29,10 @@
without it fails with a RuntimeException carrying no useful message.
-->
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<!-- Video calls (§8.10). Camera is optional hardware: a phone without one
still installs, and a video call degrades to voice. -->
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" android:required="false" />
<!-- The app is usable without a microphone; it simply cannot record. -->
<uses-feature android:name="android.hardware.microphone" android:required="false" />
@@ -5,6 +5,14 @@ import io.flamenet.e2e.Content
import org.json.JSONObject
import org.webrtc.AudioSource
import org.webrtc.AudioTrack
import org.webrtc.Camera2Enumerator
import org.webrtc.DefaultVideoDecoderFactory
import org.webrtc.DefaultVideoEncoderFactory
import org.webrtc.EglBase
import org.webrtc.SurfaceTextureHelper
import org.webrtc.VideoCapturer
import org.webrtc.VideoSource
import org.webrtc.VideoTrack
import org.webrtc.DataChannel
import org.webrtc.IceCandidate
import org.webrtc.MediaConstraints
@@ -46,7 +54,8 @@ class CallManager(
enum class State { IDLE, DIALLING, RINGING, CONNECTING, ACTIVE, ENDED }
data class Active(val callId: String, val peerId: Long, val incoming: Boolean)
data class Active(val callId: String, val peerId: Long, val incoming: Boolean,
val video: Boolean = false)
var current: Active? = null
private set
@@ -62,6 +71,20 @@ class CallManager(
private var audioSource: AudioSource? = null
private var audioTrack: AudioTrack? = null
// -- Video (§6.8v) -------------------------------------------------------
// One EGL context for the factory's codecs, the capturer and every
// renderer — WebRTC textures only travel between things that share it.
val eglBase: EglBase by lazy { EglBase.create() }
private var videoCapturer: VideoCapturer? = null
private var videoHelper: SurfaceTextureHelper? = null
private var videoSource: VideoSource? = null
var localVideoTrack: VideoTrack? = null
private set
var remoteVideoTrack: VideoTrack? = null
private set
/** Fired on the main thread when a remote video track appears or goes. */
var onVideoTracksChanged: (() -> Unit)? = null
private val main = android.os.Handler(android.os.Looper.getMainLooper())
/**
@@ -147,11 +170,18 @@ class CallManager(
.createInitializationOptions()
)
return PeerConnectionFactory.builder()
// Video codecs even for a voice call: the factory is built once
// per process, and a voice-only factory would make the first
// video call render nothing while looking connected.
.setVideoEncoderFactory(
DefaultVideoEncoderFactory(eglBase.eglBaseContext, true, true))
.setVideoDecoderFactory(DefaultVideoDecoderFactory(eglBase.eglBaseContext))
.createPeerConnectionFactory()
.also { factory = it }
}
private fun open(turn: JSONObject?, peerId: Long, callId: String): PeerConnection? {
private fun open(turn: JSONObject?, peerId: Long, callId: String,
video: Boolean): PeerConnection? {
val config = PeerConnection.RTCConfiguration(iceServers(turn)).apply {
// ALL rather than RELAY: direct paths are tried, and TURN is the
// fallback. RELAY would force every call through the server, which
@@ -203,7 +233,17 @@ class CallManager(
override fun onRemoveStream(stream: MediaStream?) = Unit
override fun onDataChannel(channel: DataChannel?) = Unit
override fun onRenegotiationNeeded() = Unit
override fun onAddTrack(receiver: RtpReceiver?, streams: Array<out MediaStream>?) = Unit
override fun onAddTrack(receiver: RtpReceiver?, streams: Array<out MediaStream>?) {
// The other side's camera. Same JNI rules as above: store the
// reference, hop to main, and let the UI attach a renderer.
runCatching {
val track = receiver?.track()
if (track is VideoTrack) {
remoteVideoTrack = track
main.post { runCatching { onVideoTracksChanged?.invoke() } }
}
}
}
}
val pc = ensureFactory().createPeerConnection(config, observer) ?: return null
@@ -212,13 +252,44 @@ class CallManager(
pc.addTrack(track, listOf("fnstream"))
audioSource = source
audioTrack = track
if (video) {
// ⚠️ Failure here downgrades to voice rather than killing the
// call — an emulator without a camera, or a denied permission
// mid-flow, should still connect and say something.
runCatching {
val cams = Camera2Enumerator(context)
val name = cams.deviceNames.firstOrNull { cams.isFrontFacing(it) }
?: cams.deviceNames.firstOrNull()
if (name != null) {
val cap = cams.createCapturer(name, null)
val helper = SurfaceTextureHelper.create("fncap", eglBase.eglBaseContext)
val vs = ensureFactory().createVideoSource(cap.isScreencast)
cap.initialize(helper, context, vs.capturerObserver)
cap.startCapture(960, 540, 24)
val vt = ensureFactory().createVideoTrack("fnvideo", vs)
val sender = pc.addTrack(vt, listOf("fnstream"))
// A cap, so video through the shared TURN cannot saturate
// the relay host. ~800 kbps is comfortable 540p.
runCatching {
val params = sender.parameters
params.encodings.forEach { it.maxBitrateBps = 800_000 }
sender.parameters = params
}
videoCapturer = cap
videoHelper = helper
videoSource = vs
localVideoTrack = vt
main.post { runCatching { onVideoTracksChanged?.invoke() } }
}
}
}
connection = pc
current = Active(callId, peerId, incoming = false)
current = Active(callId, peerId, incoming = false, video = video)
return pc
}
/** Place a call. The offer is gathered, then sent as one encrypted envelope. */
fun call(peerId: Long, turn: JSONObject?) {
fun call(peerId: Long, turn: JSONObject?, video: Boolean = false) {
if (state != State.IDLE && state != State.ENDED) return
val callId = UUID.randomUUID().toString()
// 🔴 `current` BEFORE the state change. The state setter notifies the
@@ -226,9 +297,9 @@ class CallManager(
// is a call to draw — DIALLING with a null `current` cleared the
// banner the UI had just put up, and the caller's screen showed
// nothing until a second tap redrew it (while this guard ate the tap).
current = Active(callId, peerId, incoming = false)
current = Active(callId, peerId, incoming = false, video = video)
state = State.DIALLING
val pc = open(turn, peerId, callId) ?: run {
val pc = open(turn, peerId, callId, video) ?: run {
current = null
state = State.IDLE
return
@@ -241,7 +312,8 @@ class CallManager(
// arrives, so the caller's screen says what is actually true.
main.post {
sendWhenGathered { full ->
signal(peerId, Content.CallSignal(Content.CallSignal.OFFER, callId, full))
signal(peerId, Content.CallSignal(
Content.CallSignal.OFFER, callId, full, video = video))
}
}
}
@@ -288,7 +360,8 @@ class CallManager(
return
}
pendingOffer = fromUser to incoming
current = Active(incoming.callId, fromUser, incoming = true)
current = Active(incoming.callId, fromUser, incoming = true,
video = incoming.video)
state = State.RINGING
}
Content.CallSignal.ANSWER -> {
@@ -317,11 +390,11 @@ class CallManager(
pendingOffer = null
val sdp = offer.sdp ?: return
state = State.CONNECTING
val pc = open(turn, fromUser, offer.callId) ?: run {
val pc = open(turn, fromUser, offer.callId, offer.video) ?: run {
state = State.IDLE
return
}
current = Active(offer.callId, fromUser, incoming = true)
current = Active(offer.callId, fromUser, incoming = true, video = offer.video)
pc.setRemoteDescription(
SimpleSdpObserver(),
SessionDescription(SessionDescription.Type.OFFER, sdp),
@@ -334,7 +407,8 @@ class CallManager(
main.post {
sendWhenGathered { full ->
signal(fromUser, Content.CallSignal(
Content.CallSignal.ANSWER, offer.callId, full))
Content.CallSignal.ANSWER, offer.callId, full,
video = offer.video))
}
}
}
@@ -400,6 +474,18 @@ class CallManager(
audioTrack = null
audioSource?.let { runCatching { it.dispose() } }
audioSource = null
// The camera, released in capture order. Same rule as the mic: the
// capturer holds the hardware, not the connection, and a missed
// dispose leaves the camera indicator on with no call attached.
videoCapturer?.let { runCatching { it.stopCapture() }; runCatching { it.dispose() } }
videoCapturer = null
videoHelper?.let { runCatching { it.dispose() } }
videoHelper = null
videoSource?.let { runCatching { it.dispose() } }
videoSource = null
localVideoTrack = null
remoteVideoTrack = null
main.post { runCatching { onVideoTracksChanged?.invoke() } }
pendingOffer = null
// 🔴 ENDED fires while `current` still says who the call was with —
// the observer files the call log entry from it, and nulling `current`
@@ -18,6 +18,7 @@ import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.key
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
@@ -217,14 +218,149 @@ fun LiveCallScreen(model: MessengerViewModel, phase: MessengerViewModel.Phase.Si
)
}
CallScreenBody(
peerName = call.peerName,
status = status,
actions = actions,
warning = if (call.relayless && call.state != CallManager.State.ACTIVE) {
"This server has no call relay, so the call only connects if a " +
"direct path exists between the two networks."
} else null,
)
// A video call with anything to show renders surfaces instead of the
// ring; the same action row overlays the bottom. Voice calls and video
// calls still connecting keep the kit's ring screen.
val manager = model.activeCallManager()
val showVideo = call.video && call.state == CallManager.State.ACTIVE && manager != null
if (showVideo) {
VideoCallBody(
manager = manager!!,
videoTick = call.videoTick,
peerName = call.peerName,
actions = actions,
)
} else {
CallScreenBody(
peerName = call.peerName,
status = status,
actions = actions,
warning = if (call.relayless && call.state != CallManager.State.ACTIVE) {
"This server has no call relay, so the call only connects if a " +
"direct path exists between the two networks."
} else null,
)
}
return true
}
/**
* The video layout: their camera full-screen, yours in a corner card, the
* usual actions along the bottom — the kit's 08 Video Call artboard.
*
* ⚠️ `videoTick` exists to re-run this when tracks appear: the track objects
* live on the manager, not in Compose state, so nothing else would recompose.
*/
@Composable
private fun VideoCallBody(
manager: CallManager,
videoTick: Int,
peerName: String,
actions: List<CallAction>,
) {
Box(modifier = Modifier.fillMaxSize().background(Color.Black).testTag("video-call")) {
val remote = manager.remoteVideoTrack
val local = manager.localVideoTrack
if (remote != null) {
key(remote, videoTick) {
VideoSurface(manager, remote, mirror = false,
modifier = Modifier.fillMaxSize())
}
} else {
Text(
"Waiting for their camera…",
color = Color.White,
modifier = Modifier.align(Alignment.Center),
)
}
if (local != null) {
key(local, videoTick) {
VideoSurface(
manager, local, mirror = true,
modifier = Modifier
.align(Alignment.TopEnd)
.padding(Kit.margin)
.size(width = 108.dp, height = 144.dp)
.clip(androidx.compose.foundation.shape.RoundedCornerShape(12.dp)),
)
}
}
Column(
modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = Space.xl),
horizontalAlignment = Alignment.CenterHorizontally,
) {
Text(peerName, color = Color.White, fontSize = 16.sp,
fontWeight = FontWeight.Medium)
Spacer(Modifier.height(Space.md))
Row(horizontalArrangement = Arrangement.spacedBy(Space.lg)) {
actions.forEach { CallButton(it, diameter = if (it.isPrimary) 72.dp else 48.dp) }
}
}
}
}
/**
* One WebRTC renderer. Init and release travel with the composable — a
* SurfaceViewRenderer that outlives its sink leaks the EGL surface, and one
* released while still sinked crashes in native code.
*/
@Composable
private fun VideoSurface(
manager: CallManager,
track: org.webrtc.VideoTrack,
mirror: Boolean,
modifier: Modifier = Modifier,
) {
androidx.compose.ui.viewinterop.AndroidView(
modifier = modifier,
factory = { ctx ->
org.webrtc.SurfaceViewRenderer(ctx).apply {
init(manager.eglBase.eglBaseContext, null)
setMirror(mirror)
setEnableHardwareScaler(true)
runCatching { track.addSink(this) }
}
},
onRelease = { view ->
runCatching { track.removeSink(view) }
runCatching { view.release() }
},
)
}
/**
* A "start a video call" action that asks for the camera first.
*
* Returns a function of the peer id. The permission launcher must be created
* during composition, so this cannot be a plain method on the model — and
* asking at the moment of the tap is the only time the question makes sense
* to the person being asked.
*/
@Composable
fun rememberVideoCall(model: MessengerViewModel): (Long) -> Unit {
val context = androidx.compose.ui.platform.LocalContext.current
val pending = androidx.compose.runtime.remember {
androidx.compose.runtime.mutableStateOf<Long?>(null)
}
val ask = androidx.activity.compose.rememberLauncherForActivityResult(
androidx.activity.result.contract.ActivityResultContracts.RequestPermission()
) { granted ->
val peer = pending.value
pending.value = null
// Denied still calls — as voice-grade video (their picture, no ours is
// wrong; the manager simply has no camera to add, so it degrades).
if (peer != null && granted) model.requestCall(peer, video = true)
}
return { peer ->
val has = androidx.core.content.ContextCompat.checkSelfPermission(
context, android.Manifest.permission.CAMERA
) == android.content.pm.PackageManager.PERMISSION_GRANTED
if (has) {
model.requestCall(peer, video = true)
} else {
pending.value = peer
ask.launch(android.Manifest.permission.CAMERA)
}
}
}
@@ -49,6 +49,7 @@ fun ColumnScope.CallsScreen(
model: MessengerViewModel,
phase: MessengerViewModel.Phase.SignedIn,
) {
val videoCall = rememberVideoCall(model)
val showNewCall = androidx.compose.runtime.remember {
androidx.compose.runtime.mutableStateOf(false)
}
@@ -105,6 +106,19 @@ fun ColumnScope.CallsScreen(
)
Icon(KitIcon.Call, contentDescription = null,
tint = Palette.Accent, modifier = Modifier.size(20.dp))
Spacer(Modifier.width(Space.md))
Icon(
KitIcon.Video,
contentDescription = "Video call",
tint = Palette.Accent,
modifier = Modifier
.size(22.dp)
.clickable {
showNewCall.value = false
videoCall(buddy.userId)
}
.testTag("new-video-call-${buddy.userId}"),
)
}
}
}
@@ -199,7 +213,24 @@ fun ColumnScope.CallsScreen(
)
}
}
// Ring them back, which is the only thing this row is for.
// Ring them back — voice, or camera-first.
Box(
modifier = Modifier
.size(Kit.icon + Space.md)
.clip(CircleShape)
.background(Palette.AccentSoft)
.clickable { videoCall(entry.peerId) }
.testTag("video-back-${entry.peerId}"),
contentAlignment = Alignment.Center,
) {
Icon(
KitIcon.Video,
contentDescription = "Video call $name",
tint = MaterialTheme.colorScheme.primary,
modifier = Modifier.size(Kit.icon),
)
}
Spacer(Modifier.width(Space.sm))
Box(
modifier = Modifier
.size(Kit.icon + Space.md)
@@ -1293,6 +1293,13 @@ fun ColumnScope.ChatScreen(
// different key handling, and an enabled-looking button that cannot
// work is worse than an absent one.
if (chat.peerId != null) {
val videoCall = rememberVideoCall(model)
IconAction(
icon = KitIcon.Video,
label = "Video call",
tag = "start-video-call",
onClick = { videoCall(chat.peerId) },
)
// ⚠️ **The kit's bar has a video icon here and this does not**, and
// that is deliberate. Neither CallManager has any video path at
// all — no capturer, no video track, nothing — so the button would
@@ -537,10 +537,11 @@ private fun BottomNav(
phase.calls.count { !it.connected && it.incoming }, "nav-calls")
NavButton(model, phase, MessengerViewModel.Phase.Screen.STORIES,
KitIcon.Stories, KitIcon.Stories, "Stories", 0, "nav-stories")
NavButton(model, phase, MessengerViewModel.Phase.Screen.NOTIFICATIONS,
KitIcon.Alerts, KitIcon.Alerts, "Alerts", unreadNotes, "nav-alerts")
// Alerts left the bar 2026-08-28 — five tabs plus it was crowded, and
// iOS never had one (its unread count badges Account). Same here now;
// the screen itself survives for the badge tap-through from Account.
NavButton(model, phase, MessengerViewModel.Phase.Screen.ACCOUNT,
KitIcon.Account, KitIcon.Account, "Account", 0, "nav-account")
KitIcon.Account, KitIcon.Account, "Account", unreadNotes, "nav-account")
}
}
@@ -159,6 +159,10 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
val state: CallManager.State,
val peerId: Long,
val peerName: String,
/** A camera call — the live screen renders video surfaces. */
val video: Boolean = false,
/** Bumped when video tracks appear/vanish so Compose re-reads them. */
val videoTick: Int = 0,
val incoming: Boolean,
val muted: Boolean = false,
/**
@@ -986,7 +990,7 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
peerId = active.peerId,
peerName = nameFor(phase, active.peerId),
incoming = active.incoming,
)).copy(state = state),
)).copy(state = state, video = active.video),
)
}
}
@@ -1000,6 +1004,9 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
refreshCallLog()
}
}
manager.onVideoTracksChanged = {
update { it.copy(call = it.call?.copy(videoTick = it.call.videoTick + 1)) }
}
calls = manager
installCallSignalHandler()
return manager
@@ -1099,8 +1106,12 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
* find out afterwards. Asked once, not every time — a prompt people see
* repeatedly is a prompt they stop reading.
*/
fun requestCall(peerId: Long) {
if (callPrefs().getBoolean(CALL_PRIVACY_ACKNOWLEDGED, false)) placeCall(peerId)
/** Remembered across the privacy prompt, so "Understood, call" keeps it. */
private var pendingCallVideo = false
fun requestCall(peerId: Long, video: Boolean = false) {
pendingCallVideo = video
if (callPrefs().getBoolean(CALL_PRIVACY_ACKNOWLEDGED, false)) placeCall(peerId, video)
else update { it.copy(callPrivacyPrompt = peerId) }
}
@@ -1108,10 +1119,10 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
callPrefs().edit().putBoolean(CALL_PRIVACY_ACKNOWLEDGED, true).apply()
val peerId = signedIn()?.callPrivacyPrompt
update { it.copy(callPrivacyPrompt = null) }
if (andCall && peerId != null) placeCall(peerId)
if (andCall && peerId != null) placeCall(peerId, pendingCallVideo)
}
private fun placeCall(peerId: Long) = viewModelScope.launch {
private fun placeCall(peerId: Long, video: Boolean = false) = viewModelScope.launch {
callWasConnected = false
val current = session ?: return@launch
val manager = callManager()
@@ -1123,17 +1134,18 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
// and that banner is never shown. A call to somebody with no registered
// device looked identical to a call that rang.
android.util.Log.i("fnmsg-call",
"placing call to $peerId, turn=${if (turn == null) "none" else "yes"}")
"placing call to $peerId, video=$video, turn=${if (turn == null) "none" else "yes"}")
update {
it.copy(call = Phase.CallState(
state = CallManager.State.DIALLING,
peerId = peerId,
peerName = nameFor(it, peerId),
incoming = false,
video = video,
relayless = turn == null,
))
}
withContext(Dispatchers.IO) { manager.call(peerId, turn) }
withContext(Dispatchers.IO) { manager.call(peerId, turn, video) }
}
fun acceptCall() = viewModelScope.launch {
@@ -1152,6 +1164,13 @@ class MessengerViewModel(app: Application) : AndroidViewModel(app) {
update { it.copy(call = null) }
}
/**
* The live manager, for the call screen's video renderers only. Views
* otherwise take plain values; a video surface is the one thing that
* genuinely needs the track objects and the shared EGL context.
*/
fun activeCallManager(): CallManager? = calls
fun setCallMuted(muted: Boolean) {
calls?.setMuted(muted)
update { it.copy(call = it.call?.copy(muted = muted)) }
@@ -9,6 +9,7 @@ import androidx.compose.foundation.layout.ColumnScope
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
@@ -173,12 +174,15 @@ private fun StoryCard(
Box(
modifier = Modifier
.fillMaxWidth()
.aspectRatio(375f / 320f)
.height(240.dp)
.clip(RoundedCornerShape(Radius.small))
.background(Palette.SurfaceAlt),
contentAlignment = Alignment.Center,
) {
AttachedImage(story.ref, model)
// 🔴 The card's own image, cropped to FILL the box — not the chat
// bubble's 260dp thumbnail, which floated small on a full-width
// backboard and read as a broken layout.
StoryImage(story.ref, model)
}
if (story.caption.isNotBlank()) {
@@ -290,3 +294,35 @@ fun ColumnScope.StoriesCards(
Spacer(Modifier.height(Space.xl))
}
}
/**
* A story's image at card size: fills the box, cropped. The chat bubble's
* `AttachedImage` deliberately caps at 260dp and letterboxes — right for a
* bubble, wrong for the kit's full-width post card.
*/
@Composable
private fun StoryImage(ref: io.flamenet.e2e.Attachment.Ref, model: MessengerViewModel) {
val bitmap = remember(ref.id) {
androidx.compose.runtime.mutableStateOf<android.graphics.Bitmap?>(null)
}
val failed = remember(ref.id) { androidx.compose.runtime.mutableStateOf(false) }
LaunchedEffect(ref.id) {
val bytes = model.attachmentBytes(ref)
if (bytes == null) { failed.value = true; return@LaunchedEffect }
bitmap.value = android.graphics.BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
if (bitmap.value == null) failed.value = true
}
val loaded = bitmap.value
when {
loaded != null -> androidx.compose.foundation.Image(
bitmap = loaded.asImageBitmap(),
contentDescription = "Story image",
contentScale = androidx.compose.ui.layout.ContentScale.Crop,
modifier = Modifier.fillMaxSize(),
)
failed.value -> Text("Image unavailable", fontSize = 14.sp,
color = Palette.SecondaryText)
else -> Text("Loading…", fontSize = 14.sp, color = Palette.SecondaryText)
}
}
@@ -108,6 +108,9 @@ data class Content(
val callId: String,
val sdp: String? = null,
val reason: String? = null,
/** Offer/answer: this call carries video. Absent on the wire = false,
* so voice-only clients interoperate unchanged. */
val video: Boolean = false,
) {
companion object {
const val OFFER = "call-offer"
@@ -235,6 +238,7 @@ data class Content(
val call = JSONObject().put("id", signal.callId)
signal.sdp?.let { call.put("sdp", it) }
signal.reason?.let { call.put("reason", it) }
if (signal.video) call.put("video", true)
return """{"fnc":1,"kind":${quote(signal.action)},"call":$call}"""
}
@@ -324,6 +328,7 @@ data class Content(
callId = call.optString("id"),
sdp = call.optString("sdp").takeIf { it.isNotEmpty() },
reason = call.optString("reason").takeIf { it.isNotEmpty() },
video = call.optBoolean("video", false),
))
}