(feat): improve mobile
Some checks failed
/ release (push) Has been cancelled
/ build-web (push) Has been cancelled
/ build-desktop (linux) (push) Has been cancelled
/ build-mobile (push) Has been cancelled

(feat): add camera sharing
(fix): vite tauri connection
(fix): methanium/ui theme not applied to call popout
This commit is contained in:
Alois 2026-08-01 01:25:41 +02:00
commit 62aaa7c01d
Signed by: alois
SSH key fingerprint: SHA256:GBzT2DXvAuGV9XIV5W3WrzVpjU54FThmxHXdbz95J24
31 changed files with 1915 additions and 859 deletions

View file

@ -95,7 +95,11 @@
"target": [
"dmg"
],
"icon": "build/icons/icon.icns"
"icon": "build/icons/icon.icns",
"extendInfo": {
"NSCameraUsageDescription": "Tensamin uses your camera when you choose to share it in a call.",
"NSMicrophoneUsageDescription": "Tensamin uses your microphone for calls and shared audio."
}
},
"publish": null
}

View file

@ -180,6 +180,29 @@ function registerDisplayMediaHandler() {
);
}
function registerMediaPermissionHandler() {
const isTrustedRenderer = (url: string) => {
try {
const parsed = new URL(url);
return parsed.protocol === "file:";
} catch {
return false;
}
};
session.defaultSession.setPermissionCheckHandler(
(_webContents, permission, requestingOrigin) =>
permission === "media" && isTrustedRenderer(requestingOrigin),
);
session.defaultSession.setPermissionRequestHandler(
(_webContents, permission, callback, details) => {
callback(
permission === "media" && isTrustedRenderer(details.requestingUrl),
);
},
);
}
function registerIpc() {
verboseLog("registering ipc handlers");
@ -360,6 +383,7 @@ async function start() {
verboseLog("app ready");
registerIpc();
registerDisplayMediaHandler();
registerMediaPermissionHandler();
initTray(() => mainWindow);
await createWindow();
}

View file

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<!-- AndroidTV support -->
<uses-feature android:name="android.software.leanback" android:required="false" />
@ -39,6 +45,11 @@
<!-- DEEP LINK PLUGIN. AUTO-GENERATED. DO NOT REMOVE. -->
</activity>
<service
android:name=".MediaProjectionService"
android:exported="false"
android:foregroundServiceType="mediaProjection" />
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"

View file

@ -1,24 +1,65 @@
package net.tensamin.client
import android.Manifest
import android.app.Activity
import android.content.pm.PackageManager
import android.graphics.Rect
import android.media.projection.MediaProjectionManager
import android.os.Bundle
import android.view.ViewGroup
import android.view.ViewTreeObserver
import android.view.WindowManager
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.widget.FrameLayout
import androidx.activity.result.contract.ActivityResultContracts
import androidx.core.content.ContextCompat
import androidx.core.view.ViewCompat
import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat
import org.json.JSONObject
class MainActivity : TauriActivity() {
private var contentRoot: FrameLayout? = null
private var contentChild: android.view.View? = null
private var previousUsableHeight = 0
private var attachLayoutListener: ViewTreeObserver.OnGlobalLayoutListener? = null
private var mediaWebView: WebView? = null
private var pendingScreenAudio: Boolean? = null
private val screenCaptureLauncher = registerForActivityResult(
ActivityResultContracts.StartActivityForResult(),
) { result ->
val includeAudio = pendingScreenAudio
pendingScreenAudio = null
if (includeAudio == null) return@registerForActivityResult
val data = result.data
if (result.resultCode != Activity.RESULT_OK || data == null) {
MobileMediaEvents.emitError("Screen capture permission was denied")
return@registerForActivityResult
}
MediaProjectionService.start(this, result.resultCode, data, includeAudio)
}
private val cameraPermissionLauncher = registerForActivityResult(
ActivityResultContracts.RequestMultiplePermissions(),
) {
emitCameraPermission()
}
private val screenAudioPermissionLauncher = registerForActivityResult(
ActivityResultContracts.RequestPermission(),
) {
launchScreenCaptureIntent()
}
override fun onWebViewCreate(webView: WebView) {
webView.setInitialScale(300)
mediaWebView = webView
MobileMediaEvents.attach(webView)
webView.addJavascriptInterface(MobileMediaJavascriptInterface(), "tensaminMobileMedia")
}
override fun onCreate(savedInstanceState: Bundle?) {
@ -35,9 +76,84 @@ class MainActivity : TauriActivity() {
attachLayoutListener = null
contentRoot = null
contentChild = null
mediaWebView?.removeJavascriptInterface("tensaminMobileMedia")
mediaWebView = null
MobileMediaEvents.detach()
super.onDestroy()
}
private fun startScreenShare(includeAudio: Boolean) {
runOnUiThread {
if (pendingScreenAudio != null) {
MobileMediaEvents.emitError("Screen capture permission is already pending")
return@runOnUiThread
}
pendingScreenAudio = includeAudio
if (
includeAudio &&
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) !=
PackageManager.PERMISSION_GRANTED
) {
screenAudioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
} else {
launchScreenCaptureIntent()
}
}
}
private fun launchScreenCaptureIntent() {
val manager = getSystemService(MediaProjectionManager::class.java)
screenCaptureLauncher.launch(manager.createScreenCaptureIntent())
}
private fun stopScreenShare() {
runOnUiThread {
pendingScreenAudio = null
MediaProjectionService.stop(this)
}
}
private fun requestCameraPermission() {
runOnUiThread {
cameraPermissionLauncher.launch(
arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO),
)
}
}
private fun emitCameraPermission() {
val detail = JSONObject()
.put(
"camera",
ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED,
)
.put(
"microphone",
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED,
)
MobileMediaEvents.emit("tensamin-mobile-camera-permission", detail)
}
private inner class MobileMediaJavascriptInterface {
@JavascriptInterface
fun startScreenShare(includeAudio: Boolean) {
this@MainActivity.startScreenShare(includeAudio)
}
@JavascriptInterface
fun stopScreenShare() {
this@MainActivity.stopScreenShare()
}
@JavascriptInterface
fun requestCameraPermission() {
this@MainActivity.requestCameraPermission()
}
}
private fun installKeyboardResizeWorkaround() {
val content = window.decorView.findViewById<FrameLayout>(android.R.id.content)
contentRoot = content

View file

@ -0,0 +1,348 @@
package net.tensamin.client
import android.Manifest
import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.PendingIntent
import android.app.Service
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.graphics.Bitmap
import android.graphics.PixelFormat
import android.hardware.display.DisplayManager
import android.hardware.display.VirtualDisplay
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioPlaybackCaptureConfiguration
import android.media.AudioRecord
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.media.ImageReader
import android.os.Build
import android.os.Handler
import android.os.HandlerThread
import android.os.IBinder
import android.util.Base64
import android.util.DisplayMetrics
import androidx.core.content.ContextCompat
import java.io.ByteArrayOutputStream
import java.util.concurrent.atomic.AtomicBoolean
import org.json.JSONObject
class MediaProjectionService : Service() {
private var projection: MediaProjection? = null
private var virtualDisplay: VirtualDisplay? = null
private var imageReader: ImageReader? = null
private var captureThread: HandlerThread? = null
private var audioRecord: AudioRecord? = null
private var audioThread: Thread? = null
private val captureActive = AtomicBoolean(false)
private var lastFrameAt = 0L
private val projectionCallback = object : MediaProjection.Callback() {
override fun onStop() {
stopCapture(stopProjection = false, emitStopped = true)
stopSelf()
}
}
override fun onBind(intent: Intent?): IBinder? = null
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
if (intent?.action == ACTION_STOP) {
stopCapture(stopProjection = true, emitStopped = true)
stopSelf()
return START_NOT_STICKY
}
val permissionData = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
intent?.getParcelableExtra(EXTRA_PERMISSION_DATA, Intent::class.java)
} else {
@Suppress("DEPRECATION")
intent?.getParcelableExtra(EXTRA_PERMISSION_DATA)
}
val resultCode = intent?.getIntExtra(EXTRA_RESULT_CODE, Int.MIN_VALUE) ?: Int.MIN_VALUE
if (permissionData == null || resultCode == Int.MIN_VALUE) {
MobileMediaEvents.emitError("Screen capture permission data is missing")
stopSelf()
return START_NOT_STICKY
}
val includeAudio = intent?.getBooleanExtra(EXTRA_INCLUDE_AUDIO, false) ?: false
try {
startForegroundNotification()
startCapture(resultCode, permissionData, includeAudio)
} catch (error: Throwable) {
stopCapture(stopProjection = true, emitStopped = false)
stopForeground(STOP_FOREGROUND_REMOVE)
stopSelf()
MobileMediaEvents.emitError(error.message ?: "Unable to start screen capture")
}
return START_NOT_STICKY
}
override fun onDestroy() {
stopCapture(stopProjection = true, emitStopped = true)
super.onDestroy()
}
private fun startForegroundNotification() {
val notificationManager = getSystemService(NotificationManager::class.java)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
notificationManager.createNotificationChannel(
NotificationChannel(
NOTIFICATION_CHANNEL_ID,
"Screen sharing",
NotificationManager.IMPORTANCE_LOW,
),
)
}
val stopIntent = Intent(this, MediaProjectionService::class.java).setAction(ACTION_STOP)
val stopPendingIntent = PendingIntent.getService(
this,
0,
stopIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
)
val notification = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
android.app.Notification.Builder(this, NOTIFICATION_CHANNEL_ID)
} else {
android.app.Notification.Builder(this)
}
.setSmallIcon(android.R.drawable.ic_menu_share)
.setContentTitle("Tensamin is sharing your screen")
.setContentText("Tap Stop to end screen sharing")
.setOngoing(true)
.setCategory(android.app.Notification.CATEGORY_SERVICE)
.addAction(android.R.drawable.ic_media_pause, "Stop", stopPendingIntent)
.build()
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(
NOTIFICATION_ID,
notification,
ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION,
)
} else {
startForeground(NOTIFICATION_ID, notification)
}
}
@Suppress("DEPRECATION")
private fun startCapture(resultCode: Int, permissionData: Intent, includeAudio: Boolean) {
stopCapture(stopProjection = true, emitStopped = false)
val manager = getSystemService(MediaProjectionManager::class.java)
val newProjection = manager.getMediaProjection(resultCode, permissionData)
?: error("Android did not provide a screen capture session")
projection = newProjection
val thread = HandlerThread("tensamin-screen-capture").also { it.start() }
captureThread = thread
val handler = Handler(thread.looper)
newProjection.registerCallback(projectionCallback, handler)
val metrics = DisplayMetrics()
getSystemService(android.view.WindowManager::class.java).defaultDisplay.getRealMetrics(metrics)
val displayWidth = metrics.widthPixels
val displayHeight = metrics.heightPixels
val width = minOf(displayWidth, MAX_FRAME_WIDTH)
val height = (displayHeight.toLong() * width / displayWidth).toInt()
val density = metrics.densityDpi
val reader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2)
imageReader = reader
reader.setOnImageAvailableListener({ source -> captureFrame(source, width, height) }, handler)
virtualDisplay = newProjection.createVirtualDisplay(
"Tensamin screen sharing",
width,
height,
density,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
reader.surface,
null,
handler,
)
captureActive.set(true)
val audioStarted = includeAudio && startAudioCapture(newProjection)
MobileMediaEvents.emit(
"tensamin-mobile-screen-started",
JSONObject().put("includeAudio", audioStarted),
)
}
private fun captureFrame(source: ImageReader, width: Int, height: Int) {
val image = source.acquireLatestImage() ?: return
try {
val now = System.currentTimeMillis()
if (!captureActive.get() || now - lastFrameAt < FRAME_INTERVAL_MS) return
lastFrameAt = now
val plane = image.planes[0]
val paddedWidth = plane.rowStride / plane.pixelStride
val paddedBitmap = Bitmap.createBitmap(paddedWidth, height, Bitmap.Config.ARGB_8888)
paddedBitmap.copyPixelsFromBuffer(plane.buffer)
val croppedBitmap = if (paddedWidth == width) {
paddedBitmap
} else {
Bitmap.createBitmap(paddedBitmap, 0, 0, width, height).also { paddedBitmap.recycle() }
}
val outputBitmap = croppedBitmap
val bytes = ByteArrayOutputStream()
outputBitmap.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, bytes)
val outputWidth = outputBitmap.width
val outputHeight = outputBitmap.height
outputBitmap.recycle()
MobileMediaEvents.emit(
"tensamin-mobile-screen-frame",
JSONObject()
.put("data", Base64.encodeToString(bytes.toByteArray(), Base64.NO_WRAP))
.put("mimeType", "image/jpeg")
.put("width", outputWidth)
.put("height", outputHeight),
)
} catch (error: Throwable) {
if (captureActive.get()) {
MobileMediaEvents.emitError(error.message ?: "Unable to read a screen frame")
}
} finally {
image.close()
}
}
private fun startAudioCapture(activeProjection: MediaProjection): Boolean {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
MobileMediaEvents.emitError("System audio capture requires Android 10 or newer")
return false
}
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) !=
PackageManager.PERMISSION_GRANTED
) {
MobileMediaEvents.emitError("Microphone permission is required for system audio capture")
return false
}
return try {
val format = AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(AUDIO_SAMPLE_RATE)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
val configuration = AudioPlaybackCaptureConfiguration.Builder(activeProjection)
.addMatchingUsage(AudioAttributes.USAGE_MEDIA)
.addMatchingUsage(AudioAttributes.USAGE_GAME)
.build()
val minimumBuffer = AudioRecord.getMinBufferSize(
AUDIO_SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
)
check(minimumBuffer > 0) { "Android could not allocate a system audio buffer" }
val bufferSize = maxOf(minimumBuffer * 2, AUDIO_BATCH_BYTES)
val record = AudioRecord.Builder()
.setAudioFormat(format)
.setAudioPlaybackCaptureConfig(configuration)
.setBufferSizeInBytes(bufferSize)
.build()
check(record.state == AudioRecord.STATE_INITIALIZED) {
"Android could not initialize system audio capture"
}
audioRecord = record
record.startRecording()
audioThread = Thread({ readAudio(record) }, "tensamin-audio-capture").also { it.start() }
true
} catch (error: Throwable) {
audioRecord?.release()
audioRecord = null
MobileMediaEvents.emitError(error.message ?: "Unable to capture system audio")
false
}
}
private fun readAudio(record: AudioRecord) {
val buffer = ByteArray(AUDIO_BATCH_BYTES)
while (captureActive.get() && !Thread.currentThread().isInterrupted) {
val read = record.read(buffer, 0, buffer.size, AudioRecord.READ_BLOCKING)
if (read > 0 && captureActive.get()) {
MobileMediaEvents.emit(
"tensamin-mobile-screen-audio",
JSONObject()
.put("data", Base64.encodeToString(buffer, 0, read, Base64.NO_WRAP))
.put("sampleRate", AUDIO_SAMPLE_RATE)
.put("channelCount", 1)
.put("encoding", "pcm16le"),
)
} else if (read < 0 && captureActive.get()) {
MobileMediaEvents.emitError("System audio capture stopped with code $read")
return
}
}
}
@Synchronized
private fun stopCapture(stopProjection: Boolean, emitStopped: Boolean) {
val wasActive = captureActive.getAndSet(false)
val record = audioRecord
audioRecord = null
try {
record?.stop()
} catch (_: IllegalStateException) {
}
record?.release()
audioThread?.interrupt()
audioThread = null
imageReader?.setOnImageAvailableListener(null, null)
virtualDisplay?.release()
virtualDisplay = null
imageReader?.close()
imageReader = null
val oldProjection = projection
projection = null
oldProjection?.unregisterCallback(projectionCallback)
if (stopProjection) oldProjection?.stop()
captureThread?.quitSafely()
captureThread = null
lastFrameAt = 0L
if (wasActive && emitStopped) {
MobileMediaEvents.emit("tensamin-mobile-screen-stopped")
}
}
companion object {
private const val ACTION_STOP = "net.tensamin.client.STOP_SCREEN_SHARE"
private const val EXTRA_RESULT_CODE = "resultCode"
private const val EXTRA_PERMISSION_DATA = "permissionData"
private const val EXTRA_INCLUDE_AUDIO = "includeAudio"
private const val NOTIFICATION_CHANNEL_ID = "screen-sharing"
private const val NOTIFICATION_ID = 7314
private const val FRAME_INTERVAL_MS = 75L
private const val MAX_FRAME_WIDTH = 1280
private const val JPEG_QUALITY = 72
private const val AUDIO_SAMPLE_RATE = 48_000
private const val AUDIO_BATCH_BYTES = 9_600
fun start(context: Context, resultCode: Int, data: Intent, includeAudio: Boolean) {
val intent = Intent(context, MediaProjectionService::class.java)
.putExtra(EXTRA_RESULT_CODE, resultCode)
.putExtra(EXTRA_PERMISSION_DATA, data)
.putExtra(EXTRA_INCLUDE_AUDIO, includeAudio)
ContextCompat.startForegroundService(context, intent)
}
fun stop(context: Context) {
context.startService(
Intent(context, MediaProjectionService::class.java).setAction(ACTION_STOP),
)
}
}
}

View file

@ -0,0 +1,29 @@
package net.tensamin.client
import android.webkit.WebView
import java.lang.ref.WeakReference
import org.json.JSONObject
object MobileMediaEvents {
private var webView = WeakReference<WebView>(null)
fun attach(value: WebView) {
webView = WeakReference(value)
}
fun detach() {
webView.clear()
}
fun emitError(message: String) {
emit("tensamin-mobile-screen-error", JSONObject().put("message", message))
}
fun emit(name: String, detail: JSONObject = JSONObject()) {
val view = webView.get() ?: return
val script = "window.dispatchEvent(new CustomEvent(" +
JSONObject.quote(name) +
", { detail: " + detail.toString() + " }));"
view.post { view.evaluateJavascript(script, null) }
}
}

View file

@ -5,7 +5,14 @@ import {
PopoverTrigger,
useIsMobile,
} from "@methanium/ui";
import { ArrowLeft, House, Phone, Settings, User } from "lucide-react";
import {
ArrowLeft,
EllipsisVertical,
House,
Phone,
Settings,
User,
} from "lucide-react";
import { useLocation, useNavigate, useSearch } from "@tanstack/react-router";
import { joinCall, useCall } from "@tensamin/call/store";
import Wrapper from "@tensamin/user/wrapper";
@ -40,6 +47,8 @@ export default function Navbar({ forMobile }: { forMobile: boolean }) {
const [userInfoOpen, setUserInfoOpen] = useState(false);
const { callId } = useCall();
return (
<div
data-tauri-drag-region
@ -76,6 +85,9 @@ export default function Navbar({ forMobile }: { forMobile: boolean }) {
</Button>
</>
)}
{isMobile && pathname === "/call" && callId && (
<p className="font-medium text-[1.07rem]">{displayCallId(callId)}</p>
)}
{pathname === "/chat" && id && (
<Wrapper
userId={id}
@ -172,6 +184,11 @@ export default function Navbar({ forMobile }: { forMobile: boolean }) {
)}
</>
)}
{isMobile && pathname === "/call" && callId && (
<Button variant="ghost">
<EllipsisVertical />
</Button>
)}
<Controls className="mr-2" />
</div>
</div>

View file

@ -106,7 +106,7 @@ export default defineConfig({
? {
protocol: "ws",
host,
port: 1421,
clientPort: 3000,
}
: undefined,
watch: {