(feat): add basic call ui

(feat): add screensharing (incl. broken desktop picker)
(qol): add todo
This commit is contained in:
Alois 2026-04-30 22:41:49 +02:00
commit fbd8ef4fa0
21 changed files with 1167 additions and 380 deletions

View file

@ -21,12 +21,9 @@ struct ScreenShareAudioOutput {
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct ScreenShareCapabilities { struct ScreenShareCapabilities {
platform: String, platform: String,
#[serde(rename = "usesPipeWireAudioPicker")] show_audio_output_selector: bool,
uses_pipewire_audio_picker: bool, show_audio_switch: bool,
#[serde(rename = "supportsSystemAudioSwitch")] has_reliable_system_audio: bool,
supports_system_audio_switch: bool,
#[serde(rename = "supportsReliableSystemAudio")]
supports_reliable_system_audio: bool,
} }
#[tauri::command] #[tauri::command]
@ -100,13 +97,14 @@ fn list_screen_share_sources() -> Result<Vec<ScreenShareSource>, String> {
} }
#[tauri::command] #[tauri::command]
fn list_screen_share_audio_outputs() -> Result<Vec<ScreenShareAudioOutput>, String> { fn list_audio_outputs() -> Result<Vec<ScreenShareAudioOutput>, String> {
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
{ {
use serde_json::Value;
use std::process::Command; use std::process::Command;
let output = Command::new("pactl") let output = Command::new("pactl")
.args(["list", "short", "sinks"]) .args(["--format=json", "list", "sinks"])
.output() .output()
.map_err(|error| error.to_string())?; .map_err(|error| error.to_string())?;
@ -114,28 +112,51 @@ fn list_screen_share_audio_outputs() -> Result<Vec<ScreenShareAudioOutput>, Stri
return Err(String::from_utf8_lossy(&output.stderr).trim().to_string()); return Err(String::from_utf8_lossy(&output.stderr).trim().to_string());
} }
let default_sink = Command::new("pactl")
.arg("get-default-sink")
.output()
.ok()
.filter(|result| result.status.success())
.map(|result| String::from_utf8_lossy(&result.stdout).trim().to_string());
let sinks: Value = serde_json::from_slice(&output.stdout).map_err(|error| error.to_string())?;
let sink_entries = sinks
.as_array()
.ok_or_else(|| "Unexpected pactl sink response".to_string())?;
let mut outputs = Vec::new(); let mut outputs = Vec::new();
for line in String::from_utf8_lossy(&output.stdout).lines() { for sink in sink_entries {
let mut parts = line.split('\t'); let Some(index) = sink.get("index").and_then(Value::as_i64) else {
let Some(id) = parts.next() else {
continue; continue;
}; };
let Some(name) = parts.next() else { let Some(name) = sink.get("name").and_then(Value::as_str) else {
continue; continue;
}; };
let description = parts.next_back().unwrap_or(name).to_string(); let description = sink
.get("description")
.and_then(Value::as_str)
.or_else(|| {
sink.get("properties")
.and_then(|properties| properties.get("device.description"))
.and_then(Value::as_str)
})
.unwrap_or(name)
.to_string();
let is_default = default_sink.as_deref() == Some(name);
outputs.push(ScreenShareAudioOutput { outputs.push(ScreenShareAudioOutput {
id: id.to_string(), id: index.to_string(),
name: description, name: description,
is_default: false, is_default,
}); });
} }
outputs.sort_by_key(|output| !output.is_default);
return Ok(outputs); return Ok(outputs);
} }
@ -143,57 +164,6 @@ fn list_screen_share_audio_outputs() -> Result<Vec<ScreenShareAudioOutput>, Stri
Ok(Vec::new()) Ok(Vec::new())
} }
#[tauri::command]
fn capture_screen_share_frame(source_id: &str) -> Result<String, String> {
#[cfg(any(target_os = "linux", target_os = "windows", target_os = "macos"))]
{
use base64::Engine;
use image::{codecs::jpeg::JpegEncoder, DynamicImage};
use std::io::Cursor;
use xcap::{Monitor, Window};
let Some((kind, index)) = source_id.split_once(':') else {
return Err("Invalid source id".to_string());
};
let index = index
.parse::<usize>()
.map_err(|_| "Invalid source index".to_string())?;
let frame = match kind {
"screen" => Monitor::all()
.map_err(|error| error.to_string())?
.into_iter()
.nth(index)
.ok_or_else(|| "Screen source not found".to_string())?
.capture_image()
.map_err(|error| error.to_string())?,
"window" => Window::all()
.map_err(|error| error.to_string())?
.into_iter()
.filter(|window| !window.is_minimized().unwrap_or(false))
.nth(index)
.ok_or_else(|| "Window source not found".to_string())?
.capture_image()
.map_err(|error| error.to_string())?,
_ => return Err("Unsupported source kind".to_string()),
};
let mut buffer = Cursor::new(Vec::new());
let image = DynamicImage::ImageRgba8(frame);
JpegEncoder::new_with_quality(&mut buffer, 70)
.encode_image(&image)
.map_err(|error| error.to_string())?;
let encoded = base64::engine::general_purpose::STANDARD.encode(buffer.into_inner());
return Ok(format!("data:image/jpeg;base64,{encoded}"));
}
#[allow(unreachable_code)]
Err("Screen capture is not supported on this platform".to_string())
}
#[tauri::command] #[tauri::command]
fn get_screen_share_capabilities() -> ScreenShareCapabilities { fn get_screen_share_capabilities() -> ScreenShareCapabilities {
ScreenShareCapabilities { ScreenShareCapabilities {
@ -207,9 +177,9 @@ fn get_screen_share_capabilities() -> ScreenShareCapabilities {
"other" "other"
} }
.to_string(), .to_string(),
uses_pipewire_audio_picker: cfg!(target_os = "linux"), show_audio_output_selector: cfg!(target_os = "linux"),
supports_system_audio_switch: cfg!(any(target_os = "windows", target_os = "macos")), show_audio_switch: cfg!(any(target_os = "windows", target_os = "macos")),
supports_reliable_system_audio: cfg!(target_os = "windows"), has_reliable_system_audio: cfg!(target_os = "windows"),
} }
} }
@ -254,9 +224,8 @@ pub fn run() {
}) })
.invoke_handler(tauri::generate_handler![ .invoke_handler(tauri::generate_handler![
list_screen_share_sources, list_screen_share_sources,
list_screen_share_audio_outputs, list_audio_outputs,
get_screen_share_capabilities, get_screen_share_capabilities
capture_screen_share_frame
]) ])
.run(tauri::generate_context!()) .run(tauri::generate_context!())
.expect("error while running tauri application"); .expect("error while running tauri application");

View file

@ -1,9 +1,4 @@
import { import { createContext, useContext, useMemo, type ReactNode } from "react";
createContext,
useContext,
useMemo,
type ReactNode,
} from "react";
import { invoke, isTauri } from "@tauri-apps/api/core"; import { invoke, isTauri } from "@tauri-apps/api/core";
export type DesktopScreenShareSource = { export type DesktopScreenShareSource = {
@ -21,13 +16,12 @@ export type DesktopScreenShareAudioOutput = {
export type DesktopScreenShareCapabilities = { export type DesktopScreenShareCapabilities = {
platform: "linux" | "macos" | "windows" | "other"; platform: "linux" | "macos" | "windows" | "other";
usesPipeWireAudioPicker: boolean; showAudioOutputSelector: boolean;
supportsSystemAudioSwitch: boolean; showAudioSwitch: boolean;
supportsReliableSystemAudio: boolean; hasReliableSystemAudio: boolean;
}; };
type DesktopMediaContextValue = { type DesktopMediaContextValue = {
isDesktopTauri: boolean;
getScreenShareCapabilities: () => Promise<DesktopScreenShareCapabilities>; getScreenShareCapabilities: () => Promise<DesktopScreenShareCapabilities>;
listScreenShareSources: () => Promise<DesktopScreenShareSource[]>; listScreenShareSources: () => Promise<DesktopScreenShareSource[]>;
listScreenShareAudioOutputs: () => Promise<DesktopScreenShareAudioOutput[]>; listScreenShareAudioOutputs: () => Promise<DesktopScreenShareAudioOutput[]>;
@ -35,9 +29,9 @@ type DesktopMediaContextValue = {
const defaultCapabilities: DesktopScreenShareCapabilities = { const defaultCapabilities: DesktopScreenShareCapabilities = {
platform: "other", platform: "other",
usesPipeWireAudioPicker: false, showAudioOutputSelector: false,
supportsSystemAudioSwitch: false, showAudioSwitch: false,
supportsReliableSystemAudio: false, hasReliableSystemAudio: false,
}; };
const desktopMediaContext = createContext<DesktopMediaContextValue | undefined>( const desktopMediaContext = createContext<DesktopMediaContextValue | undefined>(
@ -69,7 +63,9 @@ async function getScreenShareCapabilities(): Promise<DesktopScreenShareCapabilit
return defaultCapabilities; return defaultCapabilities;
} }
return invoke<DesktopScreenShareCapabilities>("get_screen_share_capabilities"); return invoke<DesktopScreenShareCapabilities>(
"get_screen_share_capabilities",
);
} }
export function useDesktopMedia() { export function useDesktopMedia() {
@ -85,7 +81,6 @@ export function useDesktopMedia() {
export default function Provider({ children }: { children: ReactNode }) { export default function Provider({ children }: { children: ReactNode }) {
const value = useMemo<DesktopMediaContextValue>( const value = useMemo<DesktopMediaContextValue>(
() => ({ () => ({
isDesktopTauri: isTauri(),
getScreenShareCapabilities, getScreenShareCapabilities,
listScreenShareSources, listScreenShareSources,
listScreenShareAudioOutputs, listScreenShareAudioOutputs,

View file

@ -1,15 +1,26 @@
import { Card, CardContent } from "@tensamin/ui"; import { Card, CardContent, Button } from "@tensamin/ui";
import MuteButton from "./buttons/mute"; import MuteButton from "./buttons/mute";
import DeafButton from "./buttons/deaf"; import DeafButton from "./buttons/deaf";
import ScreenshareButton from "./buttons/screenshare"; import ScreenshareButton from "./buttons/screenshare";
import LeaveButton from "./buttons/leave"; import LeaveButton from "./buttons/leave";
import { stopWatchingFocusedStream, useCall } from "../store";
import InviteButton from "./buttons/invite";
export default function Actions() { export default function Actions() {
const sharedClasses = "w-14 h-10"; const sharedClasses = "w-14 h-10";
const sharedIconSize = 15; const sharedIconSize = 15;
const view = useCall((state) => state.view);
const focusedParticipantId = useCall((state) => state.focusedParticipantId);
const watchedStreamParticipantIds = useCall(
(state) => state.watchedStreamParticipantIds,
);
const isWatchingFocusedStream =
view === "focused" &&
focusedParticipantId != null &&
watchedStreamParticipantIds.includes(focusedParticipantId);
return ( return (
<div className="w-full flex justify-center py-3"> <div className="w-full flex justify-center">
<Card className="p-1.75"> <Card className="p-1.75">
<CardContent className="p-0! flex gap-1.75"> <CardContent className="p-0! flex gap-1.75">
<MuteButton className={sharedClasses} iconSize={sharedIconSize} /> <MuteButton className={sharedClasses} iconSize={sharedIconSize} />
@ -18,7 +29,18 @@ export default function Actions() {
className={sharedClasses} className={sharedClasses}
iconSize={sharedIconSize} iconSize={sharedIconSize}
/> />
<LeaveButton className={sharedClasses} iconSize={sharedIconSize} /> <InviteButton className={sharedClasses} iconSize={sharedIconSize} />
{isWatchingFocusedStream ? (
<Button
className="h-10"
variant="destructive"
onClick={() => stopWatchingFocusedStream()}
>
Stop watching
</Button>
) : (
<LeaveButton className={sharedClasses} iconSize={sharedIconSize} />
)}
</CardContent> </CardContent>
</Card> </Card>
</div> </div>

View file

@ -0,0 +1,59 @@
import { useTTP } from "@tensamin/ttp";
import {
Button,
Popover,
PopoverContent,
PopoverTrigger,
} from "@tensamin/ui";
import Wrapper from "@tensamin/user/wrapper";
import { Mail } from "lucide-react";
import { sendCallInvite } from "../../store";
import { log, toast } from "@tensamin/shared/log";
export default function InviteButton({
className,
iconSize,
}: {
className?: string;
iconSize?: number;
}) {
const { contacts } = useTTP();
return (
<Popover>
<PopoverTrigger
render={
<Button className={className}>
<Mail style={{ scale: (iconSize ? iconSize + 100 : 100) + "%" }} />
</Button>
}
/>
<PopoverContent className="flex w-40 flex-col gap-0.5 p-1!">
{contacts.map((contact) => (
<Wrapper
key={contact.user_id}
userId={contact.user_id}
loading={<div>Loading...</div>}
component={(user) => (
<Button
className="w-full justify-start"
variant="ghost"
onClick={() => {
void sendCallInvite(contact.user_id).catch((err) => {
toast(
"error",
"Failed to send call invite. Check console for details.",
);
log(1, "call", "red", "Failed to send call invite", err);
});
}}
>
{user.display}
</Button>
)}
/>
))}
</PopoverContent>
</Popover>
);
}

View file

@ -2,9 +2,9 @@ import { useState } from "react";
import { Button, Popover, PopoverContent, PopoverTrigger } from "@tensamin/ui"; import { Button, Popover, PopoverContent, PopoverTrigger } from "@tensamin/ui";
import { MonitorDot, ScreenShare } from "lucide-react"; import { MonitorDot, ScreenShare } from "lucide-react";
import { toast } from "@tensamin/shared/log"; import { toast } from "@tensamin/shared/log";
import { useDesktopMedia } from "@tensamin/tauri/context"; import { isTauri } from "@tauri-apps/api/core";
import { setScreenShareEnabled, useCall } from "../../store"; import { setScreenShareEnabled, useCall } from "../../store";
import ScreenShareDialog from "./screenshareDialog"; import ScreenShareDialog from "../screenshareDialog";
export default function ScreenshareButton({ export default function ScreenshareButton({
className, className,
@ -16,7 +16,6 @@ export default function ScreenshareButton({
const isScreensharing = useCall((state) => state.screenShareEnabled); const isScreensharing = useCall((state) => state.screenShareEnabled);
const [dialogOpen, setDialogOpen] = useState(false); const [dialogOpen, setDialogOpen] = useState(false);
const [menuOpen, setMenuOpen] = useState(false); const [menuOpen, setMenuOpen] = useState(false);
const { isDesktopTauri } = useDesktopMedia();
async function startWebShare() { async function startWebShare() {
try { try {
@ -67,13 +66,13 @@ export default function ScreenshareButton({
</Button> </Button>
} }
/> />
<PopoverContent className="flex w-52 flex-col gap-2"> <PopoverContent className="flex w-40 flex-col gap-2">
<Button <Button
disabled={isScreensharing} disabled={isScreensharing}
onClick={() => { onClick={() => {
setMenuOpen(false); setMenuOpen(false);
if (isDesktopTauri) { if (isTauri()) {
setDialogOpen(true); setDialogOpen(true);
return; return;
} }
@ -81,7 +80,7 @@ export default function ScreenshareButton({
void startWebShare(); void startWebShare();
}} }}
> >
Start sharing Start screenshare
</Button> </Button>
<Button <Button
variant="destructive" variant="destructive"
@ -91,15 +90,13 @@ export default function ScreenshareButton({
void stopShare(); void stopShare();
}} }}
> >
Stop sharing Stop screenshare
</Button>
<Button disabled variant="outline">
Stream quality soon
</Button> </Button>
<Button variant="outline">Change quality</Button>
</PopoverContent> </PopoverContent>
</Popover> </Popover>
{isDesktopTauri && ( {isTauri() && (
<ScreenShareDialog <ScreenShareDialog
open={dialogOpen} open={dialogOpen}
onOpenChange={setDialogOpen} onOpenChange={setDialogOpen}

View file

@ -0,0 +1,165 @@
// show speaking ring
import {
ContextMenu,
ContextMenuContent,
ContextMenuItem,
ContextMenuTrigger,
} from "@tensamin/ui";
import { focusParticipant, setCallView, useCall } from "../../store";
import { type Participant } from "livekit-client";
import { useEffect, useState } from "react";
import { useUser, type User } from "@tensamin/user/context";
import VideoViewer from "../videoViewer";
import { Loader2, MicOff, Monitor } from "lucide-react";
function TransparentButton({ children }: { children: React.ReactNode }) {
return (
<div className="h-6 px-1.5 bg-black/80 rounded-sm flex items-center justify-center">
{children}
</div>
);
}
function Overlay({
type,
user,
participant,
}: {
type: "user" | "stream";
user: User;
participant: Participant;
}) {
return (
<div className="w-full h-auto flex items-center gap-2">
{type === "user" && (
<>
{!participant.isMicrophoneEnabled && (
<TransparentButton>
<MicOff className="size-3.5" />
</TransparentButton>
)}
</>
)}
{type === "stream" && (
<TransparentButton>
<Monitor className="size-3.5" />
</TransparentButton>
)}
<TransparentButton>
<p className="text-sm">{user.display}</p>
</TransparentButton>
</div>
);
}
export default function Base({
participant,
type,
flush = false,
}: {
participant: Participant | undefined;
type: "user" | "stream";
flush?: boolean;
}) {
const { get } = useUser();
const focusedParticipantId = useCall((state) => state.focusedParticipantId);
const view = useCall((state) => state.view);
const [user, setUser] = useState<User | null>(null);
useEffect(() => {
const participantId = Number(participant?.identity);
if (
!participant ||
!Number.isInteger(participantId) ||
participantId <= 0
) {
// eslint-disable-next-line
setUser(null);
return;
}
let active = true;
void get(participantId).then((nextUser) => {
if (active) {
setUser(nextUser);
}
});
return () => {
active = false;
};
}, [participant, get]);
if (!participant || !user) {
return (
<div
className={`aspect-video w-full animate-pulse bg-muted ${
flush ? "rounded-none" : "rounded-md"
}`}
/>
);
}
const onClick = () => {
if (view === "grid") {
focusParticipant(user.user_id);
} else {
if (user.user_id === focusedParticipantId) {
setCallView("grid");
} else {
focusParticipant(user.user_id);
}
}
};
return (
<ContextMenu>
<ContextMenuTrigger
render={
<div
onClick={onClick}
className={`relative aspect-video w-full border-2 ${
flush ? "rounded-none border-x-0" : "rounded-md"
}`}
>
<div className="z-20 absolute bottom-0 left-0 w-full h-full flex justify-start items-end p-2">
{view === "grid" && (
<Overlay type={type} user={user} participant={participant} />
)}
</div>
<div
className={`z-10 bg-card absolute top-0 left-0 w-full h-full flex items-center justify-center ${
flush ? "rounded-none" : "rounded-sm"
}`}
>
{/* Detect video / user and place here */}
{type === "stream" &&
Array.from(participant.videoTrackPublications.values()).map(
(publication) =>
publication.isSubscribed && publication.track ? (
<VideoViewer
flush={flush}
participantId={participant.identity}
publication={publication}
/>
) : (
<Loader2 className="animate-spin" />
),
)}
{type === "user" && <p>{user.display}</p>}
</div>
</div>
}
/>
<ContextMenuContent>
<ContextMenuItem>Stop Watching</ContextMenuItem>
</ContextMenuContent>
</ContextMenu>
);
}

View file

@ -1,3 +0,0 @@
export default function SmallUser() {
return <div>Small User</div>;
}

View file

@ -1,3 +0,0 @@
export default function SmallVideo() {
return <div>Small Video</div>;
}

View file

@ -1,3 +0,0 @@
export default function BigUser() {
return <div>Big User</div>;
}

View file

@ -1,3 +0,0 @@
export default function BigVideo() {
return <div>Big Video</div>;
}

View file

@ -24,10 +24,7 @@ import {
import { toast } from "@tensamin/shared/log"; import { toast } from "@tensamin/shared/log";
import { AppWindow, Loader2, MonitorUp } from "lucide-react"; import { AppWindow, Loader2, MonitorUp } from "lucide-react";
import type { ScreenShareCaptureOptions } from "livekit-client"; import type { ScreenShareCaptureOptions } from "livekit-client";
import { import { setScreenShareEnabled, startLinuxDesktopScreenShare } from "../store";
setScreenShareEnabled,
startLinuxDesktopScreenShare,
} from "../../store";
const NONE_AUDIO_OUTPUT = "__none__"; const NONE_AUDIO_OUTPUT = "__none__";
@ -37,9 +34,9 @@ function buildScreenShareOptions(
selectedAudioOutputId: string, selectedAudioOutputId: string,
shareAudio: boolean, shareAudio: boolean,
): ScreenShareCaptureOptions { ): ScreenShareCaptureOptions {
const wantsAudio = capabilities.usesPipeWireAudioPicker const wantsAudio = capabilities.showAudioOutputSelector
? selectedAudioOutputId !== NONE_AUDIO_OUTPUT ? selectedAudioOutputId !== NONE_AUDIO_OUTPUT
: capabilities.supportsReliableSystemAudio && shareAudio; : capabilities.hasReliableSystemAudio && shareAudio;
return { return {
audio: wantsAudio audio: wantsAudio
@ -76,9 +73,9 @@ export default function ScreenShareDialog({
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [sources, setSources] = useState<DesktopScreenShareSource[]>([]); const [sources, setSources] = useState<DesktopScreenShareSource[]>([]);
const [audioOutputs, setAudioOutputs] = useState<DesktopScreenShareAudioOutput[]>( const [audioOutputs, setAudioOutputs] = useState<
[], DesktopScreenShareAudioOutput[]
); >([]);
const [capabilities, setCapabilities] = const [capabilities, setCapabilities] =
useState<DesktopScreenShareCapabilities | null>(null); useState<DesktopScreenShareCapabilities | null>(null);
const [selectedSourceId, setSelectedSourceId] = useState<string | null>(null); const [selectedSourceId, setSelectedSourceId] = useState<string | null>(null);
@ -107,7 +104,7 @@ export default function ScreenShareDialog({
setSources(nextSources); setSources(nextSources);
setCapabilities(nextCapabilities); setCapabilities(nextCapabilities);
if (nextCapabilities.usesPipeWireAudioPicker) { if (nextCapabilities.showAudioOutputSelector) {
const nextOutputs = await listScreenShareAudioOutputs(); const nextOutputs = await listScreenShareAudioOutputs();
if (!active) { if (!active) {
@ -197,7 +194,7 @@ export default function ScreenShareDialog({
<DialogHeader className="p-4 pb-3"> <DialogHeader className="p-4 pb-3">
<DialogTitle>Share your screen</DialogTitle> <DialogTitle>Share your screen</DialogTitle>
<DialogDescription> <DialogDescription>
Pick the window or display you want to share with the call. Choose a window or display you want to share.
</DialogDescription> </DialogDescription>
</DialogHeader> </DialogHeader>
@ -238,11 +235,11 @@ export default function ScreenShareDialog({
{!loading && sources.length === 0 && ( {!loading && sources.length === 0 && (
<p className="rounded-lg border border-dashed p-4 text-sm text-muted-foreground"> <p className="rounded-lg border border-dashed p-4 text-sm text-muted-foreground">
No windows or displays are currently available for capture. No windows or displays found.
</p> </p>
)} )}
{capabilities?.usesPipeWireAudioPicker ? ( {capabilities?.showAudioOutputSelector ? (
<div className="flex flex-col gap-2"> <div className="flex flex-col gap-2">
<Label>Share audio output</Label> <Label>Share audio output</Label>
<Select <Select
@ -252,43 +249,46 @@ export default function ScreenShareDialog({
} }
> >
<SelectTrigger className="w-full"> <SelectTrigger className="w-full">
<SelectValue placeholder="None" /> <SelectValue>
{selectedAudioOutputId === NONE_AUDIO_OUTPUT
? "None"
: (audioOutputs.find(
(output) => output.id === selectedAudioOutputId,
)?.name ?? "None")}
</SelectValue>
</SelectTrigger> </SelectTrigger>
<SelectContent> <SelectContent>
<SelectItem value={NONE_AUDIO_OUTPUT}>None</SelectItem> <SelectItem value={NONE_AUDIO_OUTPUT}>None</SelectItem>
{audioOutputs.map((output) => ( {audioOutputs.map((output) => (
<SelectItem key={output.id} value={output.id}> <SelectItem key={output.id} value={output.id}>
{output.name} {output.isDefault
? `${output.name} (Default)`
: output.name}
</SelectItem> </SelectItem>
))} ))}
</SelectContent> </SelectContent>
</Select> </Select>
<p className="text-xs text-muted-foreground">
Select a PipeWire output to try sharing system audio with your
screen.
</p>
</div> </div>
) : capabilities?.supportsSystemAudioSwitch ? ( ) : capabilities?.showAudioSwitch ? (
<div className="flex flex-col gap-2 rounded-xl border p-4"> <div className="flex flex-col gap-2 rounded-xl border p-4">
<div className="flex items-center justify-between gap-4"> <div className="flex items-center justify-between gap-4">
<div className="space-y-1"> <div className="space-y-1">
<Label htmlFor="share-system-audio">Share audio</Label> <Label htmlFor="share-system-audio">Share audio</Label>
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
Share system audio alongside your screen when the runtime can Share system audio alongside your screen when the runtime
provide it. can provide it.
</p> </p>
</div> </div>
<Switch <Switch
id="share-system-audio" id="share-system-audio"
checked={shareAudio} checked={shareAudio}
disabled={!capabilities.supportsReliableSystemAudio} disabled={!capabilities.hasReliableSystemAudio}
onCheckedChange={setShareAudio} onCheckedChange={setShareAudio}
/> />
</div> </div>
{!capabilities.supportsReliableSystemAudio && ( {!capabilities.hasReliableSystemAudio && (
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
System audio sharing is not available on this platform/runtime System audio sharing is not available on this platform.
yet.
</p> </p>
)} )}
</div> </div>
@ -297,17 +297,21 @@ export default function ScreenShareDialog({
{loading && ( {loading && (
<div className="flex items-center gap-2 text-sm text-muted-foreground"> <div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="size-4 animate-spin" /> <Loader2 className="size-4 animate-spin" />
Loading available share targets... Loading sources...
</div> </div>
)} )}
</div> </div>
<DialogFooter> <DialogFooter className="m-0! p-2!">
<Button variant="outline" onClick={() => onOpenChange(false)}> <Button variant="outline" onClick={() => onOpenChange(false)}>
Cancel Cancel
</Button> </Button>
{isScreensharing && ( {isScreensharing && (
<Button variant="destructive" disabled={loading} onClick={stopSharing}> <Button
variant="destructive"
disabled={loading}
onClick={stopSharing}
>
Stop sharing Stop sharing
</Button> </Button>
)} )}

View file

@ -5,24 +5,23 @@ import { useEffect, useState } from "react";
export default function TopBar() { export default function TopBar() {
const { get } = useUser(); const { get } = useUser();
const room = useCall((state) => state.room); const room = useCall((state) => state.room);
const view = useCall((state) => state.view);
const userIds = Array.from(room.remoteParticipants.values(), (participant) => const userIds = Array.from(room.remoteParticipants.values(), (participant) => {
Number(participant.identity), const participantId = Number(participant.identity);
); return Number.isInteger(participantId) && participantId > 0
? participantId
: null;
}).filter((participantId): participantId is number => participantId != null);
const userIdsKey = userIds.join(","); const userIdsKey = userIds.join(",");
const [users, setUsers] = useState<User[]>([]); const [users, setUsers] = useState<User[]>([]);
useEffect(() => { useEffect(() => {
let active = true; let active = true;
const ids = userIdsKey === "" ? [] : userIdsKey.split(",").map(Number);
if (userIds.length === 0) { void Promise.all(ids.map((id) => get(id)))
return () => {
active = false;
};
}
void Promise.all(userIds.map((id) => get(id)))
.then((users) => { .then((users) => {
if (!active) { if (!active) {
return; return;
@ -41,7 +40,7 @@ export default function TopBar() {
return () => { return () => {
active = false; active = false;
}; };
}, [get, userIdsKey, userIds]); }, [get, userIdsKey]);
return ( return (
<div className="w-full flex justify-between h-12"> <div className="w-full flex justify-between h-12">
@ -50,6 +49,7 @@ export default function TopBar() {
<div key={user.user_id}>{user.display}</div> <div key={user.user_id}>{user.display}</div>
))} ))}
</div> </div>
{view}
</div> </div>
); );
} }

View file

@ -0,0 +1,38 @@
import { VideoTrack, useParticipantTracks } from "@livekit/components-react";
import { TrackPublication } from "livekit-client";
import { useCall } from "../store";
import { cn } from "@tensamin/ui";
export default function VideoViewer({
className,
flush = false,
publication,
participantId,
}: {
className?: string;
flush?: boolean;
publication: TrackPublication;
participantId: string;
}) {
const room = useCall((state) => state.room);
const tracks = useParticipantTracks([publication.source], {
participantIdentity: participantId,
room,
});
const trackRef = tracks[0];
if (!trackRef) {
return null;
}
return (
<VideoTrack
trackRef={trackRef}
className={cn(
"w-full h-full aspect-video bg-black",
flush ? "rounded-none" : "rounded-md",
className,
)}
/>
);
}

View file

@ -0,0 +1,232 @@
import { invoke } from "@tauri-apps/api/core";
import { log } from "@tensamin/shared/log";
import {
type LocalTrack,
Room,
type ScreenShareCaptureOptions,
Track,
} from "livekit-client";
export type ScreenShareSession = {
tracks: Array<LocalTrack | MediaStreamTrack>;
cleanup?: () => void;
};
type ScreenShareStoreState = {
screenShareSession: ScreenShareSession | null;
};
type ScreenShareStoreSetState = (
updater:
| Partial<ScreenShareStoreState>
| ((state: ScreenShareStoreState) => Partial<ScreenShareStoreState>),
) => void;
type ScreenShareControllerOptions = {
room: Room;
getState: () => ScreenShareStoreState;
setState: ScreenShareStoreSetState;
getLocalParticipantId: () => number | null;
startWatching: (participantId: number, options?: { focus?: boolean }) => void;
stopWatching: (participantId: number) => void;
syncParticipantState: () => void;
};
export function createScreenShareController({
room,
getState,
setState,
getLocalParticipantId,
startWatching,
stopWatching,
syncParticipantState,
}: ScreenShareControllerOptions) {
async function clearPublishedScreenShare() {
const screenShareSession = getState().screenShareSession;
if (!screenShareSession) {
return;
}
await Promise.all(
screenShareSession.tracks.map((track) =>
room.localParticipant.unpublishTrack(track, true).catch((error) => {
log(1, "call", "red", "Failed to unpublish screen share track", error);
}),
),
);
screenShareSession.cleanup?.();
const localParticipantId = getLocalParticipantId();
if (localParticipantId != null) {
stopWatching(localParticipantId);
}
setState({ screenShareSession: null });
}
async function publishScreenShareTracks(
tracks: Array<LocalTrack | MediaStreamTrack>,
cleanup?: () => void,
) {
if (tracks.length === 0) {
throw new Error("No screen share tracks were created.");
}
await Promise.all(
tracks.map((track) =>
room.localParticipant.publishTrack(track, {
source:
track.kind === Track.Kind.Video
? Track.Source.ScreenShare
: Track.Source.ScreenShareAudio,
}),
),
);
for (const track of tracks) {
const mediaStreamTrack =
track instanceof MediaStreamTrack ? track : track.mediaStreamTrack;
mediaStreamTrack.addEventListener(
"ended",
() => {
void stopScreenShare();
},
{ once: true },
);
}
setState({ screenShareSession: { tracks, cleanup } });
syncParticipantState();
const localParticipantId = getLocalParticipantId();
if (localParticipantId != null) {
startWatching(localParticipantId, { focus: true });
}
}
async function startScreenShare(options?: ScreenShareCaptureOptions) {
await clearPublishedScreenShare();
const tracks = await room.localParticipant.createScreenTracks(options);
await publishScreenShareTracks(tracks, () => {
tracks.forEach((track) => track.stop());
});
}
async function startLinuxDesktopScreenShare(sourceId: string) {
await clearPublishedScreenShare();
const canvas = document.createElement("canvas");
canvas.width = 1280;
canvas.height = 720;
canvas.style.display = "none";
document.body.appendChild(canvas);
const context = canvas.getContext("2d");
if (!context) {
canvas.remove();
throw new Error("Failed to initialize the screen share canvas.");
}
const stream = canvas.captureStream(8);
const videoTrack = stream.getVideoTracks()[0];
if (!videoTrack) {
canvas.remove();
throw new Error("Failed to create a video track for screen sharing.");
}
const image = new Image();
let stopped = false;
let frameRequestInFlight = false;
const renderFrame = async () => {
if (stopped || frameRequestInFlight) {
return;
}
frameRequestInFlight = true;
try {
const dataUrl = await invoke<string>("capture_screen_share_frame", {
sourceId,
});
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () =>
reject(new Error("Failed to decode screen share frame."));
image.src = dataUrl;
});
if (
canvas.width !== image.naturalWidth ||
canvas.height !== image.naturalHeight
) {
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
}
context.drawImage(image, 0, 0, canvas.width, canvas.height);
} finally {
frameRequestInFlight = false;
}
};
await renderFrame();
const interval = window.setInterval(() => {
void renderFrame().catch((error) => {
log(
1,
"call",
"red",
"Failed to capture Linux screen share frame",
error,
);
});
}, 125);
await publishScreenShareTracks(
[videoTrack],
() => {
stopped = true;
window.clearInterval(interval);
stream.getTracks().forEach((track) => track.stop());
canvas.remove();
},
);
}
async function stopScreenShare() {
await clearPublishedScreenShare();
syncParticipantState();
}
async function setScreenShareEnabled(
enabled: boolean,
options?: ScreenShareCaptureOptions,
) {
if (enabled) {
await startScreenShare(options);
return;
}
await stopScreenShare();
}
return {
clearPublishedScreenShare,
startLinuxDesktopScreenShare,
startScreenShare,
stopScreenShare,
setScreenShareEnabled,
};
}

View file

@ -1,7 +1,6 @@
import { useCallback, useEffect, useMemo, useRef } from "react"; import { useCallback, useEffect, useMemo, useRef } from "react";
import { create } from "zustand"; import { create } from "zustand";
import { useLocation, useNavigate } from "@tanstack/react-router"; import { useLocation, useNavigate } from "@tanstack/react-router";
import { invoke } from "@tauri-apps/api/core";
import { useTTP } from "@tensamin/ttp"; import { useTTP } from "@tensamin/ttp";
import { log, toast } from "@tensamin/shared/log"; import { log, toast } from "@tensamin/shared/log";
import { ttp } from "@tensamin/shared/data"; import { ttp } from "@tensamin/shared/data";
@ -12,7 +11,7 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter";
import { import {
ExternalE2EEKeyProvider, ExternalE2EEKeyProvider,
LocalAudioTrack, LocalAudioTrack,
type LocalTrack, type Participant,
Room, Room,
RoomEvent, RoomEvent,
type RemoteTrack, type RemoteTrack,
@ -23,6 +22,10 @@ import {
} from "livekit-client"; } from "livekit-client";
import z from "zod"; import z from "zod";
import { toast as sonnerToast } from "sonner"; import { toast as sonnerToast } from "sonner";
import {
createScreenShareController,
type ScreenShareSession,
} from "./screenshare";
// logging // logging
setLogExtension( setLogExtension(
@ -53,6 +56,7 @@ type GetSharedSecretFn = (
remotePublicKey: string, remotePublicKey: string,
) => Promise<string>; ) => Promise<string>;
type DecryptTextFn = (sharedSecret: string, text: string) => Promise<string>; type DecryptTextFn = (sharedSecret: string, text: string) => Promise<string>;
type EncryptTextFn = (sharedSecret: string, text: string) => Promise<string>;
type LoadFn = (key: string) => Promise<unknown>; type LoadFn = (key: string) => Promise<unknown>;
type GetUserFn = (userId: number) => Promise<{ public_key: string }>; type GetUserFn = (userId: number) => Promise<{ public_key: string }>;
@ -61,15 +65,11 @@ type Runtime = {
send: SendFn; send: SendFn;
getSharedSecret: GetSharedSecretFn; getSharedSecret: GetSharedSecretFn;
decryptText: DecryptTextFn; decryptText: DecryptTextFn;
encryptText: EncryptTextFn;
load: LoadFn; load: LoadFn;
getUser: GetUserFn; getUser: GetUserFn;
}; };
type ScreenShareSession = {
tracks: Array<LocalTrack | MediaStreamTrack>;
cleanup?: () => void;
};
type CallStore = { type CallStore = {
state: CallState; state: CallState;
view: CallView; view: CallView;
@ -82,6 +82,10 @@ type CallStore = {
micEnabled: boolean; micEnabled: boolean;
screenShareEnabled: boolean; screenShareEnabled: boolean;
screenShareSession: ScreenShareSession | null; screenShareSession: ScreenShareSession | null;
focusedParticipantId: number | null;
watchedStreamParticipantIds: number[];
pendingWatchedParticipantIds: number[];
activeScreenShareParticipantIds: number[];
isEncrypted: boolean; isEncrypted: boolean;
room: Room; room: Room;
keyProvider: ExternalE2EEKeyProvider; keyProvider: ExternalE2EEKeyProvider;
@ -135,6 +139,75 @@ function clearRemoteAudio() {
} }
} }
function getParticipantId(identity: string | undefined): number | null {
if (!identity) {
return null;
}
const parsed = Number(identity);
return Number.isFinite(parsed) ? parsed : null;
}
function getAllParticipants(): Participant[] {
return [...room.remoteParticipants.values(), room.localParticipant];
}
function getActiveScreenShareParticipantIds(): number[] {
return getAllParticipants()
.map((participant) => ({
participantId: getParticipantId(participant.identity),
hasScreenShare:
participant.getTrackPublication(Track.Source.ScreenShare) != null,
}))
.filter(
(entry): entry is { participantId: number; hasScreenShare: true } =>
entry.participantId != null && entry.hasScreenShare,
)
.map((entry) => entry.participantId);
}
function getScreenShareTrackForParticipant(participantId: number) {
return getAllParticipants()
.find(
(participant) => getParticipantId(participant.identity) === participantId,
)
?.getTrackPublication(Track.Source.ScreenShare)?.track;
}
function hasParticipant(participantId: number) {
return getAllParticipants().some(
(participant) => getParticipantId(participant.identity) === participantId,
);
}
function syncScreenShareParticipants() {
const activeScreenShareParticipantIds = getActiveScreenShareParticipantIds();
const state = useCall.getState();
const watchedStreamParticipantIds = state.watchedStreamParticipantIds.filter(
(participantId) => activeScreenShareParticipantIds.includes(participantId),
);
const pendingWatchedParticipantIds = watchedStreamParticipantIds.filter(
(participantId) => getScreenShareTrackForParticipant(participantId) == null,
);
const focusedParticipantId =
state.focusedParticipantId != null &&
(watchedStreamParticipantIds.includes(state.focusedParticipantId) ||
hasParticipant(state.focusedParticipantId))
? state.focusedParticipantId
: null;
useCall.setState({
activeScreenShareParticipantIds,
watchedStreamParticipantIds,
pendingWatchedParticipantIds,
focusedParticipantId,
view:
state.view === "focused" && focusedParticipantId == null
? "grid"
: state.view,
});
}
// utils // utils
function requireRuntime(runtime: Runtime | null): Runtime { function requireRuntime(runtime: Runtime | null): Runtime {
if (!runtime) { if (!runtime) {
@ -144,6 +217,7 @@ function requireRuntime(runtime: Runtime | null): Runtime {
return runtime; return runtime;
} }
// Sync local participant flags and screen-share derived state for the active call UI.
export function syncParticipantState() { export function syncParticipantState() {
const { room, screenShareSession } = useCall.getState(); const { room, screenShareSession } = useCall.getState();
@ -154,25 +228,27 @@ export function syncParticipantState() {
isEncrypted: isEncrypted:
room.localParticipant.isE2EEEnabled && room.localParticipant.isEncrypted, room.localParticipant.isE2EEEnabled && room.localParticipant.isEncrypted,
}); });
syncScreenShareParticipants();
} }
// set state functions // set state functions
// Store runtime dependencies from hooks so the call store can use them outside React.
export function setCallRuntime(runtime: Runtime) { export function setCallRuntime(runtime: Runtime) {
useCall.setState({ runtime }); useCall.setState({ runtime });
} }
export function setCallState(state: CallState) { // Switch between preview, grid, and focused call layouts.
useCall.setState({ state });
}
export function setCallView(view: CallView) { export function setCallView(view: CallView) {
useCall.setState({ view }); useCall.setState({ view });
} }
// Keep the current call id in sync with navigation and connection flow.
export function setCallId(callId: string | null) { export function setCallId(callId: string | null) {
useCall.setState({ callId }); useCall.setState({ callId });
} }
// Cache server call metadata used by the preview screen.
export function setCurrentCallData( export function setCurrentCallData(
currentCallData: CurrentCallData & { exists: boolean }, currentCallData: CurrentCallData & { exists: boolean },
) { ) {
@ -180,6 +256,7 @@ export function setCurrentCallData(
} }
// more utils // more utils
// Navigate the app into the dedicated call route for an active call.
export async function openCallPage(callId: string) { export async function openCallPage(callId: string) {
await requireRuntime(useCall.getState().runtime).navigate({ await requireRuntime(useCall.getState().runtime).navigate({
to: "/call", to: "/call",
@ -187,6 +264,7 @@ export async function openCallPage(callId: string) {
}); });
} }
// Request the LiveKit token that authorizes this client to join a call.
export async function getCallToken(callId: string): Promise<string> { export async function getCallToken(callId: string): Promise<string> {
const response = await requireRuntime(useCall.getState().runtime) const response = await requireRuntime(useCall.getState().runtime)
.send("call_token", { .send("call_token", {
@ -201,6 +279,128 @@ export async function getCallToken(callId: string): Promise<string> {
return data.call_token; return data.call_token;
} }
// Encrypt the active call secret for a recipient and send the call invite.
export async function sendCallInvite(userId: number) {
const runtime = requireRuntime(useCall.getState().runtime);
const { callId, callSecret } = useCall.getState();
if (!callId || !callSecret) {
throw new Error("Cannot send call invite without an active call.");
}
const ownUserId = (await runtime.load("user_id")) as number;
const privateKey = await runtime.load("private_key");
const ownPublicKey = await runtime.getUser(ownUserId).then(
(data) => data.public_key,
);
const remotePublicKey = await runtime.getUser(userId).then(
(data) => data.public_key,
);
const sharedSecret = await runtime.getSharedSecret(
privateKey,
ownPublicKey,
remotePublicKey,
);
const encryptedCallSecret = await runtime.encryptText(sharedSecret, callSecret);
await runtime.send("call_invite", {
receiver_id: userId,
call_id: callId,
call_secret: encryptedCallSecret,
});
}
// Start tracking a participant's shared screen in the call UI.
export function startWatchingStream(
participantId: number,
options?: { focus?: boolean },
) {
const focus = options?.focus ?? false;
const trackReady = getScreenShareTrackForParticipant(participantId) != null;
useCall.setState((state) => ({
watchedStreamParticipantIds: state.watchedStreamParticipantIds.includes(
participantId,
)
? state.watchedStreamParticipantIds
: [...state.watchedStreamParticipantIds, participantId],
pendingWatchedParticipantIds: trackReady
? state.pendingWatchedParticipantIds.filter((id) => id !== participantId)
: state.pendingWatchedParticipantIds.includes(participantId)
? state.pendingWatchedParticipantIds
: [...state.pendingWatchedParticipantIds, participantId],
focusedParticipantId: focus ? participantId : state.focusedParticipantId,
view: focus ? "focused" : state.view,
}));
}
// Focus a participant in the main call view even when they are not sharing a screen.
export function focusParticipant(participantId: number) {
useCall.setState({
focusedParticipantId: participantId,
view: "focused",
});
}
// Stop tracking a participant's shared screen and clean up related UI state.
export function stopWatchingStream(participantId: number) {
useCall.setState((state) => ({
watchedStreamParticipantIds: state.watchedStreamParticipantIds.filter(
(id) => id !== participantId,
),
pendingWatchedParticipantIds: state.pendingWatchedParticipantIds.filter(
(id) => id !== participantId,
),
focusedParticipantId:
state.focusedParticipantId === participantId
? null
: state.focusedParticipantId,
view:
state.view === "focused" && state.focusedParticipantId === participantId
? "grid"
: state.view,
}));
}
// Exit focused screen-share mode for the currently highlighted participant.
export function stopWatchingFocusedStream() {
const focusedParticipantId = useCall.getState().focusedParticipantId;
if (focusedParticipantId == null) {
return;
}
stopWatchingStream(focusedParticipantId);
}
let screenShareController: ReturnType<typeof createScreenShareController> | null =
null;
function getScreenShareController() {
if (!screenShareController) {
screenShareController = createScreenShareController({
room,
getState: () => ({
screenShareSession: useCall.getState().screenShareSession,
}),
setState: (updater) => {
useCall.setState((state) =>
typeof updater === "function"
? updater({ screenShareSession: state.screenShareSession })
: updater,
);
},
getLocalParticipantId: () => getParticipantId(room.localParticipant.identity),
startWatching: startWatchingStream,
stopWatching: stopWatchingStream,
syncParticipantState,
});
}
return screenShareController;
}
// Connect to LiveKit, enable the microphone, and move the UI into the live call.
export async function connect(callId: string) { export async function connect(callId: string) {
const token = await getCallToken(callId); const token = await getCallToken(callId);
@ -234,8 +434,13 @@ export async function connect(callId: string) {
syncParticipantState(); syncParticipantState();
} }
export function disconnect() { // Tear down the active call session and return the store to a closed state.
void clearPublishedScreenShare(); export async function disconnect() {
try {
await getScreenShareController().clearPublishedScreenShare();
} catch (error) {
log(1, "call", "red", "Failed to clear screen share during disconnect", error);
}
useCall.setState({ useCall.setState({
state: "closing", state: "closing",
@ -247,17 +452,27 @@ export function disconnect() {
deaf: false, deaf: false,
view: "preview", view: "preview",
screenShareSession: null, screenShareSession: null,
focusedParticipantId: null,
watchedStreamParticipantIds: [],
pendingWatchedParticipantIds: [],
activeScreenShareParticipantIds: [],
}); });
room.remoteParticipants.forEach((participant) => { room.remoteParticipants.forEach((participant) => {
participant.setVolume(100); participant.setVolume(1);
}); });
room.disconnect(); try {
useCall.setState({ state: "closed" }); room.disconnect();
syncParticipantState(); } catch (error) {
log(1, "call", "red", "Failed to disconnect from room", error);
} finally {
useCall.setState({ state: "closed" });
syncParticipantState();
}
} }
// Prepare encryption and join or create a call with another user.
export async function joinCall( export async function joinCall(
userId: number, userId: number,
callSecret?: string, callSecret?: string,
@ -313,11 +528,12 @@ export async function joinCall(
} }
} }
// Mute or restore incoming call audio for every remote participant.
export function toggleDeaf() { export function toggleDeaf() {
const nextDeaf = !useCall.getState().deaf; const nextDeaf = !useCall.getState().deaf;
room.remoteParticipants.forEach((participant) => { room.remoteParticipants.forEach((participant) => {
participant.setVolume(nextDeaf ? 0 : 100); participant.setVolume(nextDeaf ? 0 : 1);
}); });
if (nextDeaf && room.localParticipant.isMicrophoneEnabled) { if (nextDeaf && room.localParticipant.isMicrophoneEnabled) {
@ -327,6 +543,7 @@ export function toggleDeaf() {
useCall.setState({ deaf: nextDeaf }); useCall.setState({ deaf: nextDeaf });
} }
// Toggle the local microphone while keeping deaf/mute state consistent.
export async function toggleMute() { export async function toggleMute() {
const micEnabled = useCall.getState().micEnabled; const micEnabled = useCall.getState().micEnabled;
@ -338,163 +555,30 @@ export async function toggleMute() {
syncParticipantState(); syncParticipantState();
} }
async function clearPublishedScreenShare() { // Start browser-native screen sharing for the current participant.
const screenShareSession = useCall.getState().screenShareSession;
if (!screenShareSession) {
return;
}
await Promise.all(
screenShareSession.tracks.map((track) =>
room.localParticipant.unpublishTrack(track, true).catch((error) => {
log(1, "call", "red", "Failed to unpublish screen share track", error);
}),
),
);
screenShareSession.cleanup?.();
useCall.setState({ screenShareSession: null });
}
async function publishScreenShareTracks(
tracks: Array<LocalTrack | MediaStreamTrack>,
cleanup?: () => void,
) {
if (tracks.length === 0) {
throw new Error("No screen share tracks were created.");
}
await Promise.all(
tracks.map((track) =>
room.localParticipant.publishTrack(track, {
source:
track.kind === Track.Kind.Video
? Track.Source.ScreenShare
: Track.Source.ScreenShareAudio,
}),
),
);
for (const track of tracks) {
const mediaStreamTrack =
track instanceof MediaStreamTrack ? track : track.mediaStreamTrack;
mediaStreamTrack.addEventListener(
"ended",
() => {
void stopScreenShare();
},
{ once: true },
);
}
useCall.setState({ screenShareSession: { tracks, cleanup } });
syncParticipantState();
}
export async function startScreenShare(options?: ScreenShareCaptureOptions) { export async function startScreenShare(options?: ScreenShareCaptureOptions) {
await clearPublishedScreenShare(); await getScreenShareController().startScreenShare(options);
const tracks = await room.localParticipant.createScreenTracks(options);
await publishScreenShareTracks(
tracks,
() => tracks.forEach((track) => track.stop()),
);
} }
// Start the Linux desktop capture path that renders frames through Tauri.
export async function startLinuxDesktopScreenShare(sourceId: string) { export async function startLinuxDesktopScreenShare(sourceId: string) {
await clearPublishedScreenShare(); await getScreenShareController().startLinuxDesktopScreenShare(sourceId);
const canvas = document.createElement("canvas");
canvas.width = 1280;
canvas.height = 720;
canvas.style.display = "none";
document.body.appendChild(canvas);
const context = canvas.getContext("2d");
if (!context) {
canvas.remove();
throw new Error("Failed to initialize the screen share canvas.");
}
const stream = canvas.captureStream(8);
const videoTrack = stream.getVideoTracks()[0];
if (!videoTrack) {
canvas.remove();
throw new Error("Failed to create a video track for screen sharing.");
}
const image = new Image();
let stopped = false;
let frameRequestInFlight = false;
const renderFrame = async () => {
if (stopped || frameRequestInFlight) {
return;
}
frameRequestInFlight = true;
try {
const dataUrl = await invoke<string>("capture_screen_share_frame", {
sourceId,
});
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () => reject(new Error("Failed to decode screen share frame."));
image.src = dataUrl;
});
if (canvas.width !== image.naturalWidth || canvas.height !== image.naturalHeight) {
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
}
context.drawImage(image, 0, 0, canvas.width, canvas.height);
} finally {
frameRequestInFlight = false;
}
};
await renderFrame();
const interval = window.setInterval(() => {
void renderFrame().catch((error) => {
log(1, "call", "red", "Failed to capture Linux screen share frame", error);
});
}, 125);
await publishScreenShareTracks([videoTrack], () => {
stopped = true;
window.clearInterval(interval);
stream.getTracks().forEach((track) => track.stop());
canvas.remove();
});
} }
// Stop the local participant's active screen share and related previews.
export async function stopScreenShare() { export async function stopScreenShare() {
await clearPublishedScreenShare(); await getScreenShareController().stopScreenShare();
syncParticipantState();
} }
// Toggle screen sharing on or off from UI controls.
export async function setScreenShareEnabled( export async function setScreenShareEnabled(
enabled: boolean, enabled: boolean,
options?: ScreenShareCaptureOptions, options?: ScreenShareCaptureOptions,
) { ) {
if (enabled) { await getScreenShareController().setScreenShareEnabled(enabled, options);
await startScreenShare(options);
return;
}
await stopScreenShare();
} }
// Reset the in-memory call store when leaving the call experience entirely.
export function resetCallState() { export function resetCallState() {
useCall.setState({ useCall.setState({
state: "closed", state: "closed",
@ -506,11 +590,16 @@ export function resetCallState() {
currentCallData: null, currentCallData: null,
deaf: false, deaf: false,
screenShareSession: null, screenShareSession: null,
focusedParticipantId: null,
watchedStreamParticipantIds: [],
pendingWatchedParticipantIds: [],
activeScreenShareParticipantIds: [],
}); });
syncParticipantState(); syncParticipantState();
} }
// Attach the deep noise filter to the local microphone track when available.
async function ensureNoiseFilter( async function ensureNoiseFilter(
noiseFilter: DeepFilterNoiseFilterProcessor, noiseFilter: DeepFilterNoiseFilterProcessor,
): Promise<void> { ): Promise<void> {
@ -542,6 +631,10 @@ export const useCall = create<CallStore>(() => ({
micEnabled: room.localParticipant.isMicrophoneEnabled, micEnabled: room.localParticipant.isMicrophoneEnabled,
screenShareEnabled: room.localParticipant.isScreenShareEnabled, screenShareEnabled: room.localParticipant.isScreenShareEnabled,
screenShareSession: null, screenShareSession: null,
focusedParticipantId: null,
watchedStreamParticipantIds: [],
pendingWatchedParticipantIds: [],
activeScreenShareParticipantIds: [],
isEncrypted: isEncrypted:
room.localParticipant.isE2EEEnabled && room.localParticipant.isEncrypted, room.localParticipant.isE2EEEnabled && room.localParticipant.isEncrypted,
room, room,
@ -550,6 +643,7 @@ export const useCall = create<CallStore>(() => ({
runtime: null, runtime: null,
})); }));
// Register app-level call listeners and wire React dependencies into the store.
export function useInitializeCall() { export function useInitializeCall() {
const navigate = useNavigate(); const navigate = useNavigate();
const location = useLocation(); const location = useLocation();
@ -582,10 +676,11 @@ export function useInitializeCall() {
send: send as SendFn, send: send as SendFn,
getSharedSecret: getSharedSecret as GetSharedSecretFn, getSharedSecret: getSharedSecret as GetSharedSecretFn,
decryptText: decryptText as DecryptTextFn, decryptText: decryptText as DecryptTextFn,
encryptText: encryptText as EncryptTextFn,
load: load as LoadFn, load: load as LoadFn,
getUser: get as GetUserFn, getUser: get as GetUserFn,
}); });
}, [decryptText, get, getSharedSecret, load, navigate, send]); }, [decryptText, encryptText, get, getSharedSecret, load, navigate, send]);
const showCallingScreen = useCallback( const showCallingScreen = useCallback(
async (callId: string, callSecret: string, senderId: number) => { async (callId: string, callSecret: string, senderId: number) => {
@ -659,21 +754,7 @@ export function useInitializeCall() {
if (invitedUserId != null) { if (invitedUserId != null) {
setTimeout(async () => { setTimeout(async () => {
void requireRuntime(useCall.getState().runtime) void sendCallInvite(invitedUserId)
.send("call_invite", {
receiver_id: invitedUserId,
call_id: useCall.getState().callId!,
call_secret: await encryptText(
await getSharedSecret(
await load("private_key"),
await get(await load("user_id")).then(
(data) => data.public_key,
),
await get(invitedUserId).then((data) => data.public_key),
),
useCall.getState().callSecret!,
),
})
.catch((error) => { .catch((error) => {
toast("error", "Failed to send call invite."); toast("error", "Failed to send call invite.");
log(1, "call", "red", "Failed to send call invite", error); log(1, "call", "red", "Failed to send call invite", error);
@ -696,6 +777,20 @@ export function useInitializeCall() {
}); });
}; };
const onParticipantConnected = () => {
syncParticipantState();
};
const onParticipantDisconnected = (participant: Participant) => {
const participantId = getParticipantId(participant.identity);
if (participantId != null) {
stopWatchingStream(participantId);
}
syncParticipantState();
};
const onMediaDeviceFailure = (error: Error, kind?: MediaDeviceKind) => { const onMediaDeviceFailure = (error: Error, kind?: MediaDeviceKind) => {
log(1, "call", "red", "Media device failure", { error, kind }); log(1, "call", "red", "Media device failure", { error, kind });
toast("error", "Media device failure. See console for details."); toast("error", "Media device failure. See console for details.");
@ -712,20 +807,33 @@ export function useInitializeCall() {
}; };
const onTrackSubscribed = (track: RemoteTrack) => { const onTrackSubscribed = (track: RemoteTrack) => {
if (track.kind !== "audio" || !track.sid) { if (track.kind === "audio" && track.sid) {
return; attachRemoteAudio(track.sid, track.attach());
} }
attachRemoteAudio(track.sid, track.attach()); syncParticipantState();
}; };
const onTrackUnsubscribed = (track: RemoteTrack) => { const onTrackUnsubscribed = (
if (track.kind !== "audio" || !track.sid) { track: RemoteTrack,
return; _publication: unknown,
participant: Participant,
) => {
if (track.kind === "audio" && track.sid) {
track.detach();
detachRemoteAudio(track.sid);
} }
track.detach(); const participantId = getParticipantId(participant.identity);
detachRemoteAudio(track.sid);
if (
participantId != null &&
participant.getTrackPublication(Track.Source.ScreenShare)?.track == null
) {
stopWatchingStream(participantId);
}
syncParticipantState();
}; };
room.on(RoomEvent.Connected, onConnected); room.on(RoomEvent.Connected, onConnected);
@ -733,6 +841,8 @@ export function useInitializeCall() {
room.on(RoomEvent.Disconnected, onDisconnected); room.on(RoomEvent.Disconnected, onDisconnected);
room.on(RoomEvent.TrackSubscribed, onTrackSubscribed); room.on(RoomEvent.TrackSubscribed, onTrackSubscribed);
room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed); room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
room.on(RoomEvent.ParticipantConnected, onParticipantConnected);
room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
room.on(RoomEvent.TrackMuted, onParticipantStateChange); room.on(RoomEvent.TrackMuted, onParticipantStateChange);
room.on(RoomEvent.TrackUnmuted, onParticipantStateChange); room.on(RoomEvent.TrackUnmuted, onParticipantStateChange);
room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange); room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange);
@ -750,6 +860,8 @@ export function useInitializeCall() {
room.off(RoomEvent.Disconnected, onDisconnected); room.off(RoomEvent.Disconnected, onDisconnected);
room.off(RoomEvent.TrackSubscribed, onTrackSubscribed); room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);
room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed); room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
room.off(RoomEvent.ParticipantConnected, onParticipantConnected);
room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
room.off(RoomEvent.TrackMuted, onParticipantStateChange); room.off(RoomEvent.TrackMuted, onParticipantStateChange);
room.off(RoomEvent.TrackUnmuted, onParticipantStateChange); room.off(RoomEvent.TrackUnmuted, onParticipantStateChange);
room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange); room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange);
@ -762,7 +874,7 @@ export function useInitializeCall() {
room.disconnect(); room.disconnect();
e2eeWorker.terminate(); e2eeWorker.terminate();
}; };
}, [noiseFilter, encryptText, get, getSharedSecret, load]); }, [noiseFilter]);
// fetch call data for preview page // fetch call data for preview page
useEffect(() => { useEffect(() => {

View file

@ -1,3 +1,147 @@
import { useLayoutEffect, useMemo, useRef, useState } from "react";
import { useCall } from "../../store";
import Base from "../../components/modals/base";
const TILE_ASPECT_RATIO = 16 / 9;
const SECONDARY_ROW_HEIGHT_PX = 180;
const STACK_GAP_PX = 12;
export default function View() { export default function View() {
return <div>Focused</div>; const room = useCall((state) => state.room);
const focusedParticipantId = useCall((state) => state.focusedParticipantId);
const activeScreenShareParticipantIds = useCall(
(state) => state.activeScreenShareParticipantIds,
);
const containerRef = useRef<HTMLDivElement | null>(null);
const [focusedTileSize, setFocusedTileSize] = useState({ width: 0, height: 0 });
const [isFocusedTileFlush, setIsFocusedTileFlush] = useState(false);
const users = useMemo(() => {
const participants = [
...room.remoteParticipants.values(),
room.localParticipant,
];
return participants.filter((participant) => {
const participantId = Number(participant.identity);
return Number.isInteger(participantId) && participantId > 0;
});
}, [room]);
const userIds = useMemo(
() => users.map((participant) => Number(participant.identity)),
[users],
);
const tiles = useMemo(
() => [
...activeScreenShareParticipantIds
.filter((id) => id !== focusedParticipantId)
.map((participantId) => ({
key: `stream:${participantId}`,
kind: "stream" as const,
participantId,
})),
...userIds
.filter((id) => id !== focusedParticipantId)
.filter((id) => !activeScreenShareParticipantIds.includes(id))
.map((participantId) => ({
key: `user:${participantId}`,
kind: "user" as const,
participantId,
})),
],
[activeScreenShareParticipantIds, userIds, focusedParticipantId],
);
useLayoutEffect(() => {
const container = containerRef.current;
if (!container) {
return;
}
const syncTileLayout = () => {
const rect = container.getBoundingClientRect();
const width = Math.max(0, Math.floor(window.innerWidth + 1 - rect.left));
const height = Math.max(0, Math.floor(container.clientHeight));
const reservedHeight =
tiles.length > 0 ? SECONDARY_ROW_HEIGHT_PX + STACK_GAP_PX : 0;
const availableHeight = Math.max(0, height - reservedHeight);
const nextWidth = Math.max(
0,
Math.min(width, availableHeight * TILE_ASPECT_RATIO),
);
const nextHeight = Math.max(0, nextWidth / TILE_ASPECT_RATIO);
const widthDelta = Math.abs(width - nextWidth);
setFocusedTileSize((current) =>
current.width === nextWidth && current.height === nextHeight
? current
: { width: nextWidth, height: nextHeight },
);
setIsFocusedTileFlush(widthDelta <= 1);
};
syncTileLayout();
const observer = new ResizeObserver(() => {
requestAnimationFrame(syncTileLayout);
});
observer.observe(container);
window.addEventListener("resize", syncTileLayout);
return () => {
observer.disconnect();
window.removeEventListener("resize", syncTileLayout);
};
}, [tiles.length, focusedParticipantId]);
if (focusedParticipantId == null) {
return null;
}
const focusedParticipant = room.getParticipantByIdentity(
String(focusedParticipantId),
);
return (
<div ref={containerRef} className="flex h-full w-full items-center justify-center overflow-hidden">
<div
className="flex max-h-full w-full flex-col items-center overflow-hidden"
style={{ gap: tiles.length > 0 ? STACK_GAP_PX : 0 }}
>
<div className="w-full flex justify-center overflow-hidden">
<div className="overflow-hidden" style={focusedTileSize}>
<Base
flush={isFocusedTileFlush}
type={focusedParticipant?.isScreenShareEnabled ? "stream" : "user"}
participant={room.getParticipantByIdentity(
String(focusedParticipantId),
)}
/>
</div>
</div>
{tiles.length > 0 && (
<div
className="w-full shrink-0 flex justify-center gap-2 overflow-x-auto"
style={{ height: SECONDARY_ROW_HEIGHT_PX }}
>
{tiles.map((tile) => (
<div key={tile.key} className="h-full shrink-0 aspect-video">
<Base
type={tile.kind}
participant={room.getParticipantByIdentity(
String(tile.participantId),
)}
/>
</div>
))}
</div>
)}
</div>
</div>
);
} }

View file

@ -1,6 +1,7 @@
import { RoomEvent } from "livekit-client"; import { RoomEvent } from "livekit-client";
import { useEffect, useMemo, useRef, useState } from "react"; import { useEffect, useMemo, useRef, useState } from "react";
import { useCall } from "../../store"; import { useCall } from "../../store";
import Base from "../../components/modals/base";
const TILE_ASPECT_RATIO = 16 / 9; const TILE_ASPECT_RATIO = 16 / 9;
const GRID_GAP = 12; const GRID_GAP = 12;
@ -82,12 +83,13 @@ function calculateOptimalGridLayout(
export default function View() { export default function View() {
const room = useCall((state) => state.room); const room = useCall((state) => state.room);
const activeScreenShareParticipantIds = useCall(
(state) => state.activeScreenShareParticipantIds,
);
const containerRef = useRef<HTMLDivElement>(null); const containerRef = useRef<HTMLDivElement>(null);
const [containerSize, setContainerSize] = useState({ width: 0, height: 0 }); const [containerSize, setContainerSize] = useState({ width: 0, height: 0 });
const [participantCount, setParticipantCount] = useState( const [participantVersion, setParticipantVersion] = useState(0);
() => room.remoteParticipants.size + 1,
);
useEffect(() => { useEffect(() => {
const element = containerRef.current; const element = containerRef.current;
@ -107,33 +109,53 @@ export default function View() {
}, []); }, []);
useEffect(() => { useEffect(() => {
const syncParticipantCount = () => { const syncParticipants = () => {
setParticipantCount(room.remoteParticipants.size + 1); setParticipantVersion((version) => version + 1);
}; };
syncParticipantCount(); syncParticipants();
room.on(RoomEvent.Connected, syncParticipantCount); room.on(RoomEvent.Connected, syncParticipants);
room.on(RoomEvent.Disconnected, syncParticipantCount); room.on(RoomEvent.Disconnected, syncParticipants);
room.on(RoomEvent.ParticipantConnected, syncParticipantCount); room.on(RoomEvent.ParticipantConnected, syncParticipants);
room.on(RoomEvent.ParticipantDisconnected, syncParticipantCount); room.on(RoomEvent.ParticipantDisconnected, syncParticipants);
return () => { return () => {
room.off(RoomEvent.Connected, syncParticipantCount); room.off(RoomEvent.Connected, syncParticipants);
room.off(RoomEvent.Disconnected, syncParticipantCount); room.off(RoomEvent.Disconnected, syncParticipants);
room.off(RoomEvent.ParticipantConnected, syncParticipantCount); room.off(RoomEvent.ParticipantConnected, syncParticipants);
room.off(RoomEvent.ParticipantDisconnected, syncParticipantCount); room.off(RoomEvent.ParticipantDisconnected, syncParticipants);
}; };
}, [room]); }, [room]);
const users = useMemo(() => {
const participants = [...room.remoteParticipants.values(), room.localParticipant];
return participants.filter((participant) => {
const participantId = Number(participant.identity);
return Number.isInteger(participantId) && participantId > 0;
});
// eslint-disable-next-line
}, [participantVersion, room]);
const userIds = useMemo(
() => users.map((participant) => Number(participant.identity)),
[users],
);
const layout = useMemo( const layout = useMemo(
() => () =>
calculateOptimalGridLayout( calculateOptimalGridLayout(
containerSize.width, containerSize.width,
containerSize.height, containerSize.height,
participantCount, activeScreenShareParticipantIds.length + userIds.length,
), ),
[containerSize.height, containerSize.width, participantCount], [
activeScreenShareParticipantIds.length,
containerSize.height,
containerSize.width,
userIds.length,
],
); );
const rows = useMemo(() => { const rows = useMemo(() => {
@ -148,31 +170,57 @@ export default function View() {
}); });
}, [layout.rowCounts]); }, [layout.rowCounts]);
const users = useMemo(() => { const tiles = useMemo(
const participants = [ () => [
...room.remoteParticipants.values(), ...activeScreenShareParticipantIds.map((participantId) => ({
room.localParticipant, key: `stream:${participantId}`,
]; kind: "stream" as const,
return participants; participantId,
}, [room.localParticipant, room.remoteParticipants]); })),
...userIds.map((participantId) => ({
key: `user:${participantId}`,
kind: "user" as const,
participantId,
})),
],
[activeScreenShareParticipantIds, userIds],
);
function getParticipantById(participantId: number) {
if (Number(room.localParticipant.identity) === participantId) {
return room.localParticipant;
}
return room.getParticipantByIdentity(String(participantId));
}
return ( return (
<div ref={containerRef} className="h-[75vh] w-full overflow-hidden p-3"> <div ref={containerRef} style={{
height: "calc(100% - 2rem)"
}} className="w-full overflow-hidden p-3">
<div className="flex h-full w-full flex-col items-center justify-center gap-3"> <div className="flex h-full w-full flex-col items-center justify-center gap-3">
{rows.map((row) => ( {rows.map((row) => (
<div key={row.rowIndex} className="flex justify-center gap-3"> <div key={row.rowIndex} className="flex justify-center gap-3">
{row.participants.map((userIndex) => ( {row.participants.map((userIndex) => {
<div const tile = tiles[userIndex];
key={userIndex}
className="flex items-center justify-center rounded-xl bg-red-500" if (!tile) {
style={{ return null;
width: layout.tileWidth, }
height: layout.tileHeight,
}} return (
> <div
Test {users[userIndex]?.identity} key={tile.key}
</div> className="flex items-center justify-center rounded-xl"
))} style={{
width: layout.tileWidth,
height: layout.tileHeight,
}}
>
<Base type={tile.kind} participant={getParticipantById(tile.participantId)} />
</div>
);
})}
</div> </div>
))} ))}
</div> </div>

View file

@ -4,9 +4,15 @@ import TopBar from "../../components/top";
export default function Layout({ children }: { children: React.ReactNode }) { export default function Layout({ children }: { children: React.ReactNode }) {
return ( return (
<div className="w-full h-full flex flex-col"> <div className="w-full h-full flex flex-col">
<TopBar /> <div className="shrink-0">
<div className="w-full h-full">{children}</div> <TopBar />
<Actions /> </div>
<div className="min-h-0 flex-1 w-full flex justify-center items-center overflow-hidden">
{children}
</div>
<div className="shrink-0 pb-4.5 pt-3">
<Actions />
</div>
</div> </div>
); );
} }

8
packages/call/todo.md Normal file
View file

@ -0,0 +1,8 @@
- Speaking indicator
- Overlay for stream modals
- User modals
- Bg based on avatar
- Avatar in the center
- Stream previews
- Buttons
- Preview image