diff --git a/packages/call/src/components/buttons/screenshare.tsx b/packages/call/src/components/buttons/screenshare.tsx index 21ba118..4471539 100644 --- a/packages/call/src/components/buttons/screenshare.tsx +++ b/packages/call/src/components/buttons/screenshare.tsx @@ -18,12 +18,10 @@ export default function ScreenshareButton({ className, iconSize, tooltip, - defaultPortal, }: { className?: string; iconSize?: number; tooltip?: string; - defaultPortal?: boolean; }) { const isScreensharing = useCall((state) => state.screenShareEnabled); const screenRef = useCall((state) => state.screenRef); @@ -32,9 +30,8 @@ export default function ScreenshareButton({ const [menuOpen, setMenuOpen] = useState(false); useEffect(() => { - if (defaultPortal) return; setPortalContainer(screenRef?.current ?? undefined); - }, [screenRef, defaultPortal]); + }, [screenRef]); async function startWebShare() { try { @@ -113,9 +110,7 @@ export default function ScreenshareButton({ /> } /> - Click to open call page + + Click to open call page + ); } -export function TinyPingGraph() { +export function TinyPingGraph({ + portalContainer, +}: { + portalContainer?: HTMLElement; +}) { const room = useCall((store) => store.room); const [mapData, setMapData] = useState>(() => new Map()); @@ -164,7 +176,7 @@ export function TinyPingGraph() { } /> - + {data.length > 0 ? `${data.at(-1)?.ping} ms` : "Measuring ping..."} diff --git a/packages/call/src/speakingIndicator.ts b/packages/call/src/speakingIndicator.ts deleted file mode 100644 index 0c4f5a5..0000000 --- a/packages/call/src/speakingIndicator.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { log } from "@tensamin/shared/log"; -import { useCall } from "./store"; - -const SPEAKING_THRESHOLD = 0.01; -const SPEAKING_HANGTIME_MS = 500; -const ANALYSIS_INTERVAL_MS = 30; -const FFT_SIZE = 256; - -type AnalyserEntry = { - source: MediaStreamAudioSourceNode; - analyser: AnalyserNode; - track: MediaStreamTrack; - originalTrack?: MediaStreamTrack; - lastSpeakingTime: number; - isSpeaking: boolean; -}; - -class SpeakingDetector { - private audioContext: AudioContext | null = null; - private entries = new Map(); - private intervalId: ReturnType | null = null; - private deaf = false; - private gateThresholdStart = -50; - private gateThresholdEnd = -40; - private localParticipantId: number | null = null; - private localMicGateClosed = false; - - private ensureAudioContext(): AudioContext { - if (!this.audioContext) { - this.audioContext = new AudioContext(); - } - if (this.audioContext.state === "suspended") { - void this.audioContext.resume(); - } - return this.audioContext; - } - - setLocalParticipantId(id: number) { - this.localParticipantId = id; - } - - setGateThresholds(start: number, end: number) { - this.gateThresholdStart = start; - this.gateThresholdEnd = end; - } - - addTrack( - participantId: number, - track: MediaStreamTrack, - originalTrack?: MediaStreamTrack, - ) { - if (track.kind !== "audio") return; - - this.removeParticipant(participantId); - - const ctx = this.ensureAudioContext(); - const stream = new MediaStream([track]); - const source = ctx.createMediaStreamSource(stream); - const analyser = ctx.createAnalyser(); - analyser.fftSize = FFT_SIZE; - source.connect(analyser); - - this.entries.set(participantId, { - source, - analyser, - track, - originalTrack, - lastSpeakingTime: 0, - isSpeaking: false, - }); - - if (!this.intervalId) { - this.startLoop(); - } - } - - removeParticipant(participantId: number) { - const entry = this.entries.get(participantId); - if (!entry) return; - - try { - entry.source.disconnect(); - } catch { - // ignore - } - this.entries.delete(participantId); - - useCall.setState((state) => { - if (!state.speakingParticipantIds.has(participantId)) return state; - const next = new Set(state.speakingParticipantIds); - next.delete(participantId); - return { speakingParticipantIds: next }; - }); - - if (participantId === this.localParticipantId && this.localMicGateClosed) { - this.muteLocalTrack(false); - } - } - - setDeaf(deaf: boolean) { - this.deaf = deaf; - if (deaf) { - for (const entry of this.entries.values()) { - entry.isSpeaking = false; - entry.lastSpeakingTime = 0; - } - useCall.setState({ speakingParticipantIds: new Set() }); - } - } - - private startLoop() { - if (this.intervalId) return; - this.intervalId = setInterval(() => this.analyse(), ANALYSIS_INTERVAL_MS); - } - - private stopLoop() { - if (this.intervalId) { - clearInterval(this.intervalId); - this.intervalId = null; - } - } - - private muteLocalTrack(muted: boolean) { - const entry = this.localParticipantId - ? this.entries.get(this.localParticipantId) - : undefined; - const target = entry?.originalTrack ?? entry?.track; - - if (target && target.enabled === muted) { - target.enabled = !muted; - } - - this.localMicGateClosed = muted; - useCall.setState({ micGated: muted }); - } - - private applyNoiseGate(rms: number) { - const db = 20 * Math.log10(Math.max(rms, 0.0001)); - - if (!this.localMicGateClosed && db < this.gateThresholdStart) { - log(3, "noise gate", "purple", "closed"); - this.muteLocalTrack(true); - } else if (this.localMicGateClosed && db > this.gateThresholdEnd) { - log(3, "noise gate", "purple", "opened"); - this.muteLocalTrack(false); - } - } - - private analyse() { - if (this.deaf || this.entries.size === 0) return; - - const now = Date.now(); - const changed = new Map(); - - for (const [participantId, entry] of this.entries) { - const { analyser, track } = entry; - - if (track.muted || track.readyState === "ended" || !track.enabled) { - if (entry.isSpeaking) { - entry.isSpeaking = false; - entry.lastSpeakingTime = 0; - changed.set(participantId, false); - } - continue; - } - - const bufferLength = analyser.frequencyBinCount; - const dataArray = new Uint8Array(bufferLength); - analyser.getByteTimeDomainData(dataArray); - - let sum = 0; - for (let i = 0; i < bufferLength; i++) { - const sample = (dataArray[i] - 128) / 128.0; - sum += sample * sample; - } - const rms = Math.sqrt(sum / bufferLength); - - let nextIsSpeaking = entry.isSpeaking; - if (rms > SPEAKING_THRESHOLD) { - entry.lastSpeakingTime = now; - nextIsSpeaking = true; - } else if (now - entry.lastSpeakingTime > SPEAKING_HANGTIME_MS) { - nextIsSpeaking = false; - } - - if (participantId === this.localParticipantId) { - this.applyNoiseGate(rms); - if (this.localMicGateClosed) { - nextIsSpeaking = false; - } - } - - if (nextIsSpeaking !== entry.isSpeaking) { - entry.isSpeaking = nextIsSpeaking; - changed.set(participantId, nextIsSpeaking); - } - } - - if (changed.size > 0) { - useCall.setState((state) => { - let hasDiff = false; - const next = new Set(state.speakingParticipantIds); - for (const [id, speaking] of changed) { - if (speaking) { - if (!next.has(id)) { - next.add(id); - hasDiff = true; - } - } else { - if (next.has(id)) { - next.delete(id); - hasDiff = true; - } - } - } - return hasDiff ? { speakingParticipantIds: next } : state; - }); - } - } - - dispose() { - this.stopLoop(); - for (const id of Array.from(this.entries.keys())) { - this.removeParticipant(id); - } - this.entries.clear(); - if (this.audioContext) { - void this.audioContext.close(); - this.audioContext = null; - } - } -} - -let detectorInstance: SpeakingDetector | null = null; - -export function getSpeakingDetector(): SpeakingDetector { - if (!detectorInstance) { - detectorInstance = new SpeakingDetector(); - } - return detectorInstance; -} - -export function disposeSpeakingDetector(): void { - if (detectorInstance) { - detectorInstance.dispose(); - detectorInstance = null; - } -} - -export function useIsSpeaking(participantId: number): boolean { - return useCall((state) => state.speakingParticipantIds.has(participantId)); -} diff --git a/packages/call/src/store.tsx b/packages/call/src/store.tsx index 72efd39..e605ece 100644 --- a/packages/call/src/store.tsx +++ b/packages/call/src/store.tsx @@ -11,7 +11,6 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; import { ExternalE2EEKeyProvider, LocalAudioTrack, - type LocalTrackPublication, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -29,10 +28,6 @@ import { createScreenShareController, type ScreenShareSession, } from "./screenshare"; -import { - getSpeakingDetector, - disposeSpeakingDetector, -} from "./speakingIndicator"; // logging setLogExtension( @@ -91,7 +86,6 @@ type CallStore = { screenShareEnabled: boolean; screenShareSession: ScreenShareSession | null; focusedParticipantId: number | null; - focusedParticipantType: "user" | "stream" | null; usersInFocusedViewHidden: boolean; watchedStreamParticipantIds: number[]; pendingWatchedParticipantIds: number[]; @@ -105,8 +99,6 @@ type CallStore = { keyProvider: ExternalE2EEKeyProvider; e2eeWorker: Worker; runtime: Runtime | null; - speakingParticipantIds: Set; - micGated: boolean; }; const keyProvider = new ExternalE2EEKeyProvider(); @@ -160,7 +152,7 @@ function clearRemoteAudio() { } } -export function getParticipantId(identity: string | undefined): number | null { +function getParticipantId(identity: string | undefined): number | null { if (!identity) { return null; } @@ -465,8 +457,6 @@ function syncScreenShareParticipants() { watchedStreamParticipantIds, pendingWatchedParticipantIds, focusedParticipantId, - focusedParticipantType: - focusedParticipantId == null ? null : state.focusedParticipantType, view: state.view === "focused" && focusedParticipantId == null ? "grid" @@ -637,7 +627,6 @@ export function startWatchingStream(participantId: number) { ? state.pendingWatchedParticipantIds : [...state.pendingWatchedParticipantIds, participantId], focusedParticipantId: participantId, - focusedParticipantType: "stream", })); } @@ -657,13 +646,9 @@ export function setParticipantTrackSubscribed( } // Focus a participant in the main call view even when they are not sharing a screen. -export function focusParticipant( - participantId: number, - type: "user" | "stream" = "user", -) { +export function focusParticipant(participantId: number) { useCall.setState({ focusedParticipantId: participantId, - focusedParticipantType: type, view: "focused", }); } @@ -683,10 +668,6 @@ export function stopWatchingStream(participantId: number) { state.focusedParticipantId === participantId ? null : state.focusedParticipantId, - focusedParticipantType: - state.focusedParticipantId === participantId - ? null - : state.focusedParticipantType, view: state.view === "focused" && state.focusedParticipantId === participantId ? "grid" @@ -776,7 +757,6 @@ export async function connect(callId: string) { // Tear down the active call session and return the store to a closed state. export async function disconnect() { - disposeSpeakingDetector(); await clearScreenSharePreview(); try { @@ -802,12 +782,10 @@ export async function disconnect() { view: "preview", screenShareSession: null, focusedParticipantId: null, - focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], activeScreenShareParticipantIds: [], - micGated: false, }); room.remoteParticipants.forEach((participant) => { @@ -901,7 +879,6 @@ export async function toggleDeaf() { deafened: nextDeaf ? "true" : "false", }); - getSpeakingDetector().setDeaf(nextDeaf); useCall.setState({ deaf: nextDeaf }); } @@ -963,7 +940,6 @@ export function resetCallState() { deaf: false, screenShareSession: null, focusedParticipantId: null, - focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], @@ -991,16 +967,6 @@ async function ensureNoiseFilter( await microphoneTrack.setProcessor(noiseFilter).catch((err) => { log(1, "call", "red", "Failed to enable noise filter", err); }); - - const participantId = getParticipantId(room.localParticipant.identity); - if (participantId != null) { - const processedTrack = microphoneTrack.mediaStreamTrack; - getSpeakingDetector().addTrack( - participantId, - processedTrack.clone(), - processedTrack, - ); - } } export const useCall = create(() => ({ @@ -1016,7 +982,6 @@ export const useCall = create(() => ({ screenShareEnabled: room.localParticipant.isScreenShareEnabled, screenShareSession: null, focusedParticipantId: null, - focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], @@ -1031,8 +996,6 @@ export const useCall = create(() => ({ keyProvider, e2eeWorker, runtime: null, - speakingParticipantIds: new Set(), - micGated: false, })); // Register app-level call listeners and wire React dependencies into the store. @@ -1053,7 +1016,7 @@ export function useInitializeCall() { new DeepFilterNoiseFilterProcessor({ enabled: true, enableNoiseReduction: true, - noiseReductionLevel: 60, + noiseReductionLevel: 80, sampleRate: 48000, assetConfig: { cdnUrl: "/assets", @@ -1174,42 +1137,10 @@ export function useInitializeCall() { listenersRegistered.current = true; - const onConnected = async () => { + const onConnected = () => { useCall.setState({ state: "open" }); syncParticipantState(); - const detector = getSpeakingDetector(); - const localParticipantId = getParticipantId( - room.localParticipant.identity, - ); - if (localParticipantId != null) { - detector.setLocalParticipantId(localParticipantId); - } - - const [start, end] = await Promise.all([ - load("call_mute_range_start"), - load("call_mute_range_end"), - ]); - detector.setGateThresholds(start, end); - - // Scan existing audio tracks for speaking detection - for (const participant of getAllParticipants()) { - const participantId = getParticipantId(participant.identity); - if (participantId == null) continue; - - for (const publication of participant.trackPublications.values()) { - if (publication.kind === Track.Kind.Audio && publication.track) { - const mediaTrack = publication.track.mediaStreamTrack; - if (participant === room.localParticipant) { - const clonedTrack = mediaTrack.clone(); - detector.addTrack(participantId, clonedTrack, mediaTrack); - } else { - detector.addTrack(participantId, mediaTrack); - } - } - } - } - const invitedUserId = useCall.getState().invitedUserId; if (invitedUserId != null) { @@ -1246,7 +1177,6 @@ export function useInitializeCall() { if (participantId != null) { stopWatchingStream(participantId); - getSpeakingDetector().removeParticipant(participantId); } syncParticipantState(); @@ -1267,33 +1197,6 @@ export function useInitializeCall() { void ensureNoiseFilter(noiseFilter); }; - const onLocalTrackPublished = (publication: LocalTrackPublication) => { - if (publication.kind === Track.Kind.Audio && publication.track) { - const participantId = getParticipantId(room.localParticipant.identity); - if (participantId != null) { - const originalTrack = publication.track.mediaStreamTrack; - const clonedTrack = originalTrack.clone(); - getSpeakingDetector().addTrack( - participantId, - clonedTrack, - originalTrack, - ); - } - } - syncParticipantState(); - void ensureNoiseFilter(noiseFilter); - }; - - const onLocalTrackUnpublished = (publication: LocalTrackPublication) => { - if (publication.kind === Track.Kind.Audio) { - const participantId = getParticipantId(room.localParticipant.identity); - if (participantId != null) { - getSpeakingDetector().removeParticipant(participantId); - } - } - syncParticipantState(); - }; - const onTrackPublished = ( publication: RemoteTrackPublication, participant: RemoteParticipant, @@ -1311,18 +1214,9 @@ export function useInitializeCall() { onParticipantStateChange(); }; - const onTrackSubscribed = ( - track: RemoteTrack, - _publication: RemoteTrackPublication, - participant: RemoteParticipant, - ) => { + const onTrackSubscribed = (track: RemoteTrack) => { if (track.kind === "audio" && track.sid) { attachRemoteAudio(track.sid, track.attach()); - - const participantId = getParticipantId(participant.identity); - if (participantId != null) { - getSpeakingDetector().addTrack(participantId, track.mediaStreamTrack); - } } syncParticipantState(); @@ -1333,17 +1227,13 @@ export function useInitializeCall() { _publication: unknown, participant: Participant, ) => { - const participantId = getParticipantId(participant.identity); - if (track.kind === "audio" && track.sid) { track.detach(); detachRemoteAudio(track.sid); - - if (participantId != null) { - getSpeakingDetector().removeParticipant(participantId); - } } + const participantId = getParticipantId(participant.identity); + if ( participantId != null && getTrackPublicationBySource(participant, Track.Source.ScreenShare) @@ -1366,8 +1256,8 @@ export function useInitializeCall() { room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.on(RoomEvent.TrackMuted, onParticipantStateChange); room.on(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.on(RoomEvent.LocalTrackPublished, onLocalTrackPublished); - room.on(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); + room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange); + room.on(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); room.on(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.on(RoomEvent.EncryptionError, onEncryptionError); room.on(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1387,8 +1277,8 @@ export function useInitializeCall() { room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.off(RoomEvent.TrackMuted, onParticipantStateChange); room.off(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.off(RoomEvent.LocalTrackPublished, onLocalTrackPublished); - room.off(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); + room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange); + room.off(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); room.off(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.off(RoomEvent.EncryptionError, onEncryptionError); room.off(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1397,7 +1287,7 @@ export function useInitializeCall() { room.disconnect(); e2eeWorker.terminate(); }; - }, [noiseFilter, load]); + }, [noiseFilter]); // fetch call data for preview page useEffect(() => { diff --git a/packages/call/src/views/main/focused.tsx b/packages/call/src/views/main/focused.tsx index ec654d8..d52362b 100644 --- a/packages/call/src/views/main/focused.tsx +++ b/packages/call/src/views/main/focused.tsx @@ -16,9 +16,6 @@ export default function View() { const callIsFullscreen = useCall((state) => state.callIsFullscreen); const focusedParticipantId = useCall((state) => state.focusedParticipantId); - const focusedParticipantType = useCall( - (state) => state.focusedParticipantType, - ); const activeScreenShareParticipantIds = useCall( (state) => state.activeScreenShareParticipantIds, ); @@ -160,10 +157,6 @@ export default function View() { const focusedParticipant = getParticipantById(focusedParticipantId); const focusedParticipantHasActiveScreenShare = activeScreenShareParticipantIdSet.has(focusedParticipantId); - const focusedTileType: "user" | "stream" = - focusedParticipantType === "stream" && focusedParticipantHasActiveScreenShare - ? "stream" - : "user"; const isImmersiveFocusedView = callIsFullscreen && usersInFocusedViewHidden; return ( @@ -186,7 +179,7 @@ export default function View() { diff --git a/packages/call/todo.md b/packages/call/todo.md index 61d70ed..952858f 100644 --- a/packages/call/todo.md +++ b/packages/call/todo.md @@ -1,3 +1,4 @@ +- Speaking indicator - Overlay for stream modals - User modals - Bg based on avatar @@ -10,5 +11,3 @@ - Disconnect - Desktop-App screenshares - Context menus -- Popout Window -- If micGated=true & isSpeaking=false for 5 seconds show banner with mic detection diff --git a/packages/shared/src/data.ts b/packages/shared/src/data.ts index 857e9af..e03ca3d 100644 --- a/packages/shared/src/data.ts +++ b/packages/shared/src/data.ts @@ -245,8 +245,6 @@ export interface Storage extends SettingsStorageDefaults { cached_contacts: Contacts; cached_communities: Communities; ttp_url: string; - call_mute_range_start: number; - call_mute_range_end: number; } export const storageDefaults: Storage = { @@ -280,6 +278,4 @@ export const storageDefaults: Storage = { cached_contacts: [], cached_communities: [], ttp_url: "https://tensamin.net:959", - call_mute_range_start: -55, - call_mute_range_end: -45, };