(feat): improve mobile

(feat): add camera sharing
(fix): vite tauri connection
(fix): methanium/ui theme not applied to call popout

packages/call/src/mediaShare/browser.ts

@@ -0,0 +1,84 @@
+import type {
+  MediaShareAdapter,
+  MediaShareCapabilities,
+  MediaShareKind,
+  MediaShareRequest,
+  MediaShareSession,
+  MediaShareSource,
+} from "./types";
+
+export async function listCameraSources(): Promise<MediaShareSource[]> {
+  const permissionStream = await navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: true,
+  });
+  let devices: MediaDeviceInfo[];
+  try {
+    devices = await navigator.mediaDevices.enumerateDevices();
+  } finally {
+    permissionStream.getTracks().forEach((track) => track.stop());
+  }
+  let cameraIndex = 0;
+
+  return devices
+    .filter((device) => device.kind === "videoinput")
+    .map((device) => ({
+      id: device.deviceId,
+      kind: "camera" as const,
+      name: device.label || `Camera ${++cameraIndex}`,
+    }));
+}
+
+export async function startCamera(
+  sourceId?: string,
+): Promise<MediaShareSession> {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: sourceId ? { deviceId: { exact: sourceId } } : true,
+  });
+
+  return streamSession(stream);
+}
+
+function streamSession(stream: MediaStream): MediaShareSession {
+  return {
+    tracks: stream.getTracks(),
+    stop: async () => {
+      stream.getTracks().forEach((track) => track.stop());
+    },
+  };
+}
+
+export class BrowserMediaShareAdapter implements MediaShareAdapter {
+  async getCapabilities(): Promise<MediaShareCapabilities> {
+    return {
+      runtime: "browser",
+      screenPicker: "native",
+      canShareScreenAudio: true,
+      canSelectScreenAudioOutput: false,
+    };
+  }
+
+  async listSources(kind: MediaShareKind): Promise<MediaShareSource[]> {
+    return kind === "camera" ? listCameraSources() : [];
+  }
+
+  async start(request: MediaShareRequest): Promise<MediaShareSession> {
+    if (request.kind === "camera") {
+      return startCamera(request.sourceId);
+    }
+
+    const options: DisplayMediaStreamOptions & {
+      systemAudio: "include" | "exclude";
+      surfaceSwitching: "include" | "exclude";
+    } = {
+      audio: request.includeAudio ?? true,
+      video: true,
+      systemAudio: request.includeAudio === false ? "exclude" : "include",
+      surfaceSwitching: "include",
+    };
+    const stream = await navigator.mediaDevices.getDisplayMedia(options);
+
+    return streamSession(stream);
+  }
+}

packages/call/src/mediaShare/controller.ts

@@ -0,0 +1,154 @@
+import { log } from "@tensamin/shared/log";
+import { type LocalTrack, Room, Track } from "livekit-client";
+import {
+  getMediaShareAdapter,
+  type MediaShareKind,
+  type MediaShareRequest,
+  type MediaShareSession,
+} from ".";
+
+export type LocalMediaShareSession = {
+  tracks: Array<LocalTrack | MediaStreamTrack>;
+  capture: MediaShareSession;
+};
+
+type MediaShareStoreState = {
+  screenShareSession: LocalMediaShareSession | null;
+  cameraSession: LocalMediaShareSession | null;
+};
+
+type MediaShareStoreSetState = (
+  updater:
+    | Partial<MediaShareStoreState>
+    | ((state: MediaShareStoreState) => Partial<MediaShareStoreState>),
+) => void;
+
+type MediaShareControllerOptions = {
+  room: Room;
+  getState: () => MediaShareStoreState;
+  setState: MediaShareStoreSetState;
+  getLocalParticipantId: () => number | null;
+  startWatching: (participantId: number) => void;
+  stopWatching: (participantId: number) => void;
+  syncParticipantState: () => void;
+};
+
+export function createMediaShareController({
+  room,
+  getState,
+  setState,
+  getLocalParticipantId,
+  startWatching,
+  stopWatching,
+  syncParticipantState,
+}: MediaShareControllerOptions) {
+  function getSession(kind: MediaShareKind) {
+    return kind === "screen"
+      ? getState().screenShareSession
+      : getState().cameraSession;
+  }
+
+  function setSession(
+    kind: MediaShareKind,
+    session: LocalMediaShareSession | null,
+  ) {
+    setState(
+      kind === "screen"
+        ? { screenShareSession: session }
+        : { cameraSession: session },
+    );
+  }
+
+  async function clearPublishedShare(kind: MediaShareKind) {
+    const session = getSession(kind);
+    if (!session) return;
+
+    setSession(kind, null);
+    await Promise.all(
+      session.tracks.map((track) =>
+        room.localParticipant.unpublishTrack(track, true).catch((error) => {
+          log(1, "call", "red", `Failed to unpublish ${kind} track`, error);
+        }),
+      ),
+    );
+    await session.capture.stop().catch((error) => {
+      log(1, "call", "red", `Failed to stop ${kind} capture`, error);
+    });
+
+    if (kind === "screen") {
+      const localParticipantId = getLocalParticipantId();
+      if (localParticipantId != null) stopWatching(localParticipantId);
+    }
+  }
+
+  async function publishShare(
+    kind: MediaShareKind,
+    capture: MediaShareSession,
+  ) {
+    if (capture.tracks.length === 0) {
+      await capture.stop();
+      throw new Error(`No ${kind} tracks were created.`);
+    }
+
+    const published: MediaStreamTrack[] = [];
+    try {
+      for (const track of capture.tracks) {
+        await room.localParticipant.publishTrack(track, {
+          source:
+            track.kind === Track.Kind.Audio
+              ? Track.Source.ScreenShareAudio
+              : kind === "screen"
+                ? Track.Source.ScreenShare
+                : Track.Source.Camera,
+        });
+        published.push(track);
+      }
+    } catch (error) {
+      await Promise.all(
+        published.map((track) =>
+          room.localParticipant.unpublishTrack(track, true),
+        ),
+      );
+      await capture.stop();
+      throw error;
+    }
+
+    for (const track of capture.tracks) {
+      track.addEventListener(
+        "ended",
+        () => {
+          void stop(kind);
+        },
+        { once: true },
+      );
+    }
+
+    setSession(kind, { tracks: capture.tracks, capture });
+    syncParticipantState();
+
+    if (kind === "screen") {
+      const localParticipantId = getLocalParticipantId();
+      if (localParticipantId != null) startWatching(localParticipantId);
+    }
+  }
+
+  async function start(request: MediaShareRequest) {
+    await clearPublishedShare(request.kind);
+    const capture = await getMediaShareAdapter().start(request);
+    await publishShare(request.kind, capture);
+  }
+
+  async function stop(kind: MediaShareKind) {
+    await clearPublishedShare(kind);
+    syncParticipantState();
+  }
+
+  async function clearAll() {
+    await Promise.all([
+      clearPublishedShare("screen"),
+      clearPublishedShare("camera"),
+    ]);
+  }
+
+  return { clearAll, start, stop };
+}

packages/call/src/mediaShare/electron.ts

@@ -0,0 +1,54 @@
+import type {} from "@tensamin/shared/desktopMedia";
+import { BrowserMediaShareAdapter, listCameraSources } from "./browser";
+import type {
+  MediaShareCapabilities,
+  MediaShareKind,
+  MediaShareRequest,
+  MediaShareSession,
+  MediaShareSource,
+} from "./types";
+
+export class ElectronMediaShareAdapter extends BrowserMediaShareAdapter {
+  override async getCapabilities(): Promise<MediaShareCapabilities> {
+    const capabilities =
+      await window.tensaminDesktop?.media?.getScreenShareCapabilities?.();
+
+    return {
+      runtime: "electron",
+      screenPicker: "sources",
+      canShareScreenAudio: capabilities?.hasReliableSystemAudio ?? false,
+      canSelectScreenAudioOutput:
+        capabilities?.showAudioOutputSelector ?? false,
+    };
+  }
+
+  override async listSources(
+    kind: MediaShareKind,
+  ): Promise<MediaShareSource[]> {
+    if (kind === "camera") {
+      return listCameraSources();
+    }
+
+    return (
+      (await window.tensaminDesktop?.media?.listScreenShareSources?.()) ?? []
+    );
+  }
+
+  override async start(request: MediaShareRequest): Promise<MediaShareSession> {
+    if (request.kind === "camera") {
+      return super.start(request);
+    }
+
+    if (!request.sourceId) {
+      throw new Error("Choose a screen or window to share.");
+    }
+
+    const select = window.tensaminDesktop?.media?.selectScreenShareSource;
+    if (!select) {
+      throw new Error("Electron screen capture is unavailable.");
+    }
+
+    await select(request.sourceId);
+    return super.start({ ...request, sourceId: undefined });
+  }
+}

packages/call/src/mediaShare/index.ts

@@ -0,0 +1,27 @@
+import { BrowserMediaShareAdapter } from "./browser";
+import { ElectronMediaShareAdapter } from "./electron";
+import { TauriMediaShareAdapter } from "./tauri";
+import type { MediaShareAdapter } from "./types";
+
+let adapter: MediaShareAdapter | null = null;
+
+export function getMediaShareAdapter(): MediaShareAdapter {
+  if (!adapter) {
+    adapter = window.tensaminMobileMedia
+      ? new TauriMediaShareAdapter()
+      : window.tensaminDesktop?.media
+        ? new ElectronMediaShareAdapter()
+        : new BrowserMediaShareAdapter();
+  }
+
+  return adapter;
+}
+
+export type {
+  MediaShareAdapter,
+  MediaShareCapabilities,
+  MediaShareKind,
+  MediaShareRequest,
+  MediaShareSession,
+  MediaShareSource,
+} from "./types";

packages/call/src/mediaShare/tauri.ts

@@ -0,0 +1,270 @@
+import { listCameraSources, startCamera } from "./browser";
+import type {
+  MediaShareAdapter,
+  MediaShareCapabilities,
+  MediaShareKind,
+  MediaShareRequest,
+  MediaShareSession,
+  MediaShareSource,
+} from "./types";
+
+type MobileMediaApi = {
+  startScreenShare: (includeAudio: boolean) => void;
+  stopScreenShare: () => void;
+  requestCameraPermission: () => void;
+};
+
+declare global {
+  interface Window {
+    tensaminMobileMedia?: MobileMediaApi;
+  }
+}
+
+type FrameDetail = {
+  data: string;
+  mimeType: string;
+  width: number;
+  height: number;
+};
+
+type AudioDetail = {
+  data: string;
+  sampleRate: number;
+  channelCount: number;
+  encoding: "pcm16le";
+};
+
+function eventDetail<T>(event: Event): T {
+  return (event as CustomEvent<T>).detail;
+}
+
+function decodeBase64(value: string): Uint8Array {
+  const decoded = atob(value);
+  const bytes = new Uint8Array(decoded.length);
+  for (let index = 0; index < decoded.length; index += 1) {
+    bytes[index] = decoded.charCodeAt(index);
+  }
+  return bytes;
+}
+
+async function requestCameraPermission() {
+  const bridge = window.tensaminMobileMedia;
+  if (!bridge) throw new Error("Tauri mobile media bridge is unavailable.");
+
+  await new Promise<void>((resolve, reject) => {
+    const timeout = window.setTimeout(() => {
+      cleanup();
+      reject(new Error("Timed out waiting for camera permission."));
+    }, 30_000);
+    const onPermission = (event: Event) => {
+      cleanup();
+      const permission = eventDetail<{ camera: boolean }>(event);
+      if (permission.camera) {
+        resolve();
+      } else {
+        reject(new Error("Camera permission was denied."));
+      }
+    };
+    const cleanup = () => {
+      window.clearTimeout(timeout);
+      window.removeEventListener(
+        "tensamin-mobile-camera-permission",
+        onPermission,
+      );
+    };
+
+    window.addEventListener("tensamin-mobile-camera-permission", onPermission);
+    bridge.requestCameraPermission();
+  });
+}
+
+async function startMobileScreen(
+  includeAudio: boolean,
+): Promise<MediaShareSession> {
+  const bridge = window.tensaminMobileMedia;
+  if (!bridge) throw new Error("Tauri mobile media bridge is unavailable.");
+
+  return new Promise<MediaShareSession>((resolve, reject) => {
+    const canvas = document.createElement("canvas");
+    const context = canvas.getContext("2d");
+    if (!context) {
+      reject(new Error("Unable to create the mobile capture canvas."));
+      return;
+    }
+
+    const stream = canvas.captureStream(15);
+    let audioContext: AudioContext | null = null;
+    let audioNode: ScriptProcessorNode | null = null;
+    let audioDestination: MediaStreamAudioDestinationNode | null = null;
+    const audioQueue: Float32Array[] = [];
+    let audioQueueOffset = 0;
+    let queuedAudioSamples = 0;
+    let started = false;
+    let resolved = false;
+    let firstFrame = false;
+    let lastError: Error | null = null;
+    let errorTimer = 0;
+
+    const timeout = window.setTimeout(() => {
+      cleanup();
+      bridge.stopScreenShare();
+      reject(
+        lastError ?? new Error("Timed out starting mobile screen sharing."),
+      );
+    }, 60_000);
+
+    const complete = () => {
+      if (!started || !firstFrame || resolved) return;
+      resolved = true;
+      window.clearTimeout(timeout);
+      resolve({
+        tracks: stream.getTracks(),
+        stop: async () => {
+          bridge.stopScreenShare();
+          cleanup();
+        },
+      });
+    };
+
+    const onStarted = (event: Event) => {
+      started = true;
+      const detail = eventDetail<{ includeAudio: boolean }>(event);
+      if (detail.includeAudio) {
+        audioContext = new AudioContext({ sampleRate: 48_000 });
+        audioNode = audioContext.createScriptProcessor(2048, 0, 1);
+        audioDestination = audioContext.createMediaStreamDestination();
+        audioNode.onaudioprocess = ({ outputBuffer }) => {
+          const output = outputBuffer.getChannelData(0);
+          output.fill(0);
+          let outputOffset = 0;
+          while (outputOffset < output.length && audioQueue.length > 0) {
+            const chunk = audioQueue[0];
+            const available = chunk.length - audioQueueOffset;
+            const count = Math.min(available, output.length - outputOffset);
+            output.set(
+              chunk.subarray(audioQueueOffset, audioQueueOffset + count),
+              outputOffset,
+            );
+            outputOffset += count;
+            audioQueueOffset += count;
+            queuedAudioSamples -= count;
+            if (audioQueueOffset === chunk.length) {
+              audioQueue.shift();
+              audioQueueOffset = 0;
+            }
+          }
+        };
+        audioNode.connect(audioDestination);
+        void audioContext.resume();
+        for (const track of audioDestination.stream.getAudioTracks()) {
+          stream.addTrack(track);
+        }
+      }
+      complete();
+    };
+
+    const onFrame = (event: Event) => {
+      const detail = eventDetail<FrameDetail>(event);
+      const image = new Image();
+      image.onload = () => {
+        if (canvas.width !== detail.width || canvas.height !== detail.height) {
+          canvas.width = detail.width;
+          canvas.height = detail.height;
+        }
+        context.drawImage(image, 0, 0, canvas.width, canvas.height);
+        firstFrame = true;
+        complete();
+      };
+      image.src = `data:${detail.mimeType};base64,${detail.data}`;
+    };
+
+    const onAudio = (event: Event) => {
+      const bytes = decodeBase64(eventDetail<AudioDetail>(event).data);
+      const samples = new Int16Array(
+        bytes.buffer,
+        bytes.byteOffset,
+        Math.floor(bytes.byteLength / 2),
+      );
+      const chunk = new Float32Array(samples.length);
+      for (let index = 0; index < samples.length; index += 1) {
+        chunk[index] = samples[index] / 32768;
+      }
+      audioQueue.push(chunk);
+      queuedAudioSamples += chunk.length;
+      const maximumQueuedSamples = 48_000 * 2;
+      while (queuedAudioSamples > maximumQueuedSamples && audioQueue.length) {
+        const dropped = audioQueue.shift();
+        if (!dropped) break;
+        queuedAudioSamples -= dropped.length - audioQueueOffset;
+        audioQueueOffset = 0;
+      }
+    };
+
+    const onStopped = () => {
+      cleanup();
+      if (!resolved) reject(new Error("Mobile screen sharing was stopped."));
+    };
+
+    const onError = (event: Event) => {
+      lastError = new Error(eventDetail<{ message: string }>(event).message);
+      window.clearTimeout(errorTimer);
+      errorTimer = window.setTimeout(() => {
+        if (!started && !resolved) {
+          cleanup();
+          reject(lastError ?? new Error("Mobile screen sharing failed."));
+        }
+      }, 300);
+    };
+
+    const listeners: Array<[string, EventListener]> = [
+      ["tensamin-mobile-screen-started", onStarted],
+      ["tensamin-mobile-screen-frame", onFrame],
+      ["tensamin-mobile-screen-audio", onAudio],
+      ["tensamin-mobile-screen-stopped", onStopped],
+      ["tensamin-mobile-screen-error", onError],
+    ];
+    const cleanup = () => {
+      window.clearTimeout(timeout);
+      window.clearTimeout(errorTimer);
+      listeners.forEach(([name, listener]) =>
+        window.removeEventListener(name, listener),
+      );
+      stream.getTracks().forEach((track) => track.stop());
+      audioNode?.disconnect();
+      audioNode = null;
+      void audioContext?.close();
+      audioContext = null;
+    };
+
+    listeners.forEach(([name, listener]) =>
+      window.addEventListener(name, listener),
+    );
+    bridge.startScreenShare(includeAudio);
+  });
+}
+
+export class TauriMediaShareAdapter implements MediaShareAdapter {
+  async getCapabilities(): Promise<MediaShareCapabilities> {
+    return {
+      runtime: "tauri",
+      screenPicker: "system",
+      canShareScreenAudio: true,
+      canSelectScreenAudioOutput: false,
+    };
+  }
+
+  async listSources(kind: MediaShareKind): Promise<MediaShareSource[]> {
+    if (kind !== "camera") return [];
+    await requestCameraPermission();
+    return listCameraSources();
+  }
+
+  async start(request: MediaShareRequest): Promise<MediaShareSession> {
+    if (request.kind === "screen") {
+      return startMobileScreen(request.includeAudio ?? true);
+    }
+
+    await requestCameraPermission();
+    return startCamera(request.sourceId);
+  }
+}

packages/call/src/mediaShare/types.ts

@@ -0,0 +1,33 @@
+export type MediaShareKind = "screen" | "camera";
+
+export type MediaShareSource = {
+  id: string;
+  kind: "screen" | "window" | "camera";
+  name: string;
+  subtitle?: string | null;
+  thumbnail?: string | null;
+};
+
+export type MediaShareCapabilities = {
+  runtime: "browser" | "electron" | "tauri";
+  screenPicker: "native" | "sources" | "system";
+  canShareScreenAudio: boolean;
+  canSelectScreenAudioOutput: boolean;
+};
+
+export type MediaShareRequest = {
+  kind: MediaShareKind;
+  sourceId?: string;
+  includeAudio?: boolean;
+};
+
+export type MediaShareSession = {
+  tracks: MediaStreamTrack[];
+  stop: () => Promise<void>;
+};
+
+export interface MediaShareAdapter {
+  getCapabilities(): Promise<MediaShareCapabilities>;
+  listSources(kind: MediaShareKind): Promise<MediaShareSource[]>;
+  start(request: MediaShareRequest): Promise<MediaShareSession>;
+}