diff --git a/packages/call/src/components/modals/base.tsx b/packages/call/src/components/modals/base.tsx index b75c129..35e4ddd 100644 --- a/packages/call/src/components/modals/base.tsx +++ b/packages/call/src/components/modals/base.tsx @@ -18,6 +18,7 @@ import { import { Track, type Participant } from "livekit-client"; import { useEffect, useRef, useState } from "react"; import { useUser, type User } from "@tensamin/user/context"; +import { useIsSpeaking } from "../../speakingIndicator"; import VideoViewer from "../videoViewer"; import { HeadphoneOff, MicOff, Monitor, Plus, Shield } from "lucide-react"; @@ -103,6 +104,7 @@ export default function Base({ const focusedParticipantId = useCall((state) => state.focusedParticipantId); const view = useCall((state) => state.view); const [user, setUser] = useState(null); + const isSpeaking = useIsSpeaking(user?.user_id ?? -1); const screenSharePublication = getTrackPublicationBySource( participant, Track.Source.ScreenShare, @@ -211,9 +213,9 @@ export default function Base({
{/* Detect video / user and place here */} diff --git a/packages/call/src/speakingIndicator.ts b/packages/call/src/speakingIndicator.ts new file mode 100644 index 0000000..38fe478 --- /dev/null +++ b/packages/call/src/speakingIndicator.ts @@ -0,0 +1,249 @@ +import { useCall } from "./store"; + +const SPEAKING_THRESHOLD = 0.01; +const SPEAKING_HANGTIME_MS = 250; +const ANALYSIS_INTERVAL_MS = 30; +const FFT_SIZE = 256; + +type AnalyserEntry = { + source: MediaStreamAudioSourceNode; + analyser: AnalyserNode; + track: MediaStreamTrack; + originalTrack?: MediaStreamTrack; + lastSpeakingTime: number; + isSpeaking: boolean; +}; + +class SpeakingDetector { + private audioContext: AudioContext | null = null; + private entries = new Map(); + private intervalId: ReturnType | null = null; + private deaf = false; + private gateThresholdStart = -50; + private gateThresholdEnd = -40; + private localParticipantId: number | null = null; + private localMicGateClosed = false; + + private ensureAudioContext(): AudioContext { + if (!this.audioContext) { + this.audioContext = new AudioContext(); + } + if (this.audioContext.state === "suspended") { + void this.audioContext.resume(); + } + return this.audioContext; + } + + setLocalParticipantId(id: number) { + this.localParticipantId = id; + } + + setGateThresholds(start: number, end: number) { + this.gateThresholdStart = start; + this.gateThresholdEnd = end; + } + + addTrack( + participantId: number, + track: MediaStreamTrack, + originalTrack?: MediaStreamTrack, + ) { + if (track.kind !== "audio") return; + + this.removeParticipant(participantId); + + const ctx = this.ensureAudioContext(); + const stream = new MediaStream([track]); + const source = ctx.createMediaStreamSource(stream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = FFT_SIZE; + source.connect(analyser); + + this.entries.set(participantId, { + source, + analyser, + track, + originalTrack, + lastSpeakingTime: 0, + isSpeaking: false, + }); + + if (!this.intervalId) { + this.startLoop(); + } + } + + removeParticipant(participantId: number) { + const entry = this.entries.get(participantId); + if (!entry) return; + + try { + entry.source.disconnect(); + } catch { + // ignore + } + this.entries.delete(participantId); + + useCall.setState((state) => { + if (!state.speakingParticipantIds.has(participantId)) return state; + const next = new Set(state.speakingParticipantIds); + next.delete(participantId); + return { speakingParticipantIds: next }; + }); + + if (participantId === this.localParticipantId && this.localMicGateClosed) { + this.muteLocalTrack(false); + } + } + + setDeaf(deaf: boolean) { + this.deaf = deaf; + if (deaf) { + for (const entry of this.entries.values()) { + entry.isSpeaking = false; + entry.lastSpeakingTime = 0; + } + useCall.setState({ speakingParticipantIds: new Set() }); + } + } + + private startLoop() { + if (this.intervalId) return; + this.intervalId = setInterval(() => this.analyse(), ANALYSIS_INTERVAL_MS); + } + + private stopLoop() { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + private muteLocalTrack(muted: boolean) { + const entry = this.localParticipantId + ? this.entries.get(this.localParticipantId) + : undefined; + const target = entry?.originalTrack ?? entry?.track; + + if (target && target.enabled === muted) { + target.enabled = !muted; + } + + this.localMicGateClosed = muted; + useCall.setState({ micGated: muted }); + } + + private applyNoiseGate(rms: number) { + const db = 20 * Math.log10(Math.max(rms, 0.0001)); + + if (!this.localMicGateClosed && db < this.gateThresholdStart) { + this.muteLocalTrack(true); + } else if (this.localMicGateClosed && db > this.gateThresholdEnd) { + this.muteLocalTrack(false); + } + } + + private analyse() { + if (this.deaf || this.entries.size === 0) return; + + const now = Date.now(); + const changed = new Map(); + + for (const [participantId, entry] of this.entries) { + const { analyser, track } = entry; + + if (track.muted || track.readyState === "ended" || !track.enabled) { + if (entry.isSpeaking) { + entry.isSpeaking = false; + entry.lastSpeakingTime = 0; + changed.set(participantId, false); + } + continue; + } + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + analyser.getByteTimeDomainData(dataArray); + + let sum = 0; + for (let i = 0; i < bufferLength; i++) { + const sample = (dataArray[i] - 128) / 128.0; + sum += sample * sample; + } + const rms = Math.sqrt(sum / bufferLength); + + let nextIsSpeaking = entry.isSpeaking; + if (rms > SPEAKING_THRESHOLD) { + entry.lastSpeakingTime = now; + nextIsSpeaking = true; + } else if (now - entry.lastSpeakingTime > SPEAKING_HANGTIME_MS) { + nextIsSpeaking = false; + } + + if (participantId === this.localParticipantId) { + this.applyNoiseGate(rms); + if (this.localMicGateClosed) { + nextIsSpeaking = false; + } + } + + if (nextIsSpeaking !== entry.isSpeaking) { + entry.isSpeaking = nextIsSpeaking; + changed.set(participantId, nextIsSpeaking); + } + } + + if (changed.size > 0) { + useCall.setState((state) => { + let hasDiff = false; + const next = new Set(state.speakingParticipantIds); + for (const [id, speaking] of changed) { + if (speaking) { + if (!next.has(id)) { + next.add(id); + hasDiff = true; + } + } else { + if (next.has(id)) { + next.delete(id); + hasDiff = true; + } + } + } + return hasDiff ? { speakingParticipantIds: next } : state; + }); + } + } + + dispose() { + this.stopLoop(); + for (const id of Array.from(this.entries.keys())) { + this.removeParticipant(id); + } + this.entries.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + } + } +} + +let detectorInstance: SpeakingDetector | null = null; + +export function getSpeakingDetector(): SpeakingDetector { + if (!detectorInstance) { + detectorInstance = new SpeakingDetector(); + } + return detectorInstance; +} + +export function disposeSpeakingDetector(): void { + if (detectorInstance) { + detectorInstance.dispose(); + detectorInstance = null; + } +} + +export function useIsSpeaking(participantId: number): boolean { + return useCall((state) => state.speakingParticipantIds.has(participantId)); +} diff --git a/packages/call/src/store.tsx b/packages/call/src/store.tsx index e605ece..38498da 100644 --- a/packages/call/src/store.tsx +++ b/packages/call/src/store.tsx @@ -11,6 +11,7 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; import { ExternalE2EEKeyProvider, LocalAudioTrack, + type LocalTrackPublication, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -28,6 +29,10 @@ import { createScreenShareController, type ScreenShareSession, } from "./screenshare"; +import { + getSpeakingDetector, + disposeSpeakingDetector, +} from "./speakingIndicator"; // logging setLogExtension( @@ -99,6 +104,8 @@ type CallStore = { keyProvider: ExternalE2EEKeyProvider; e2eeWorker: Worker; runtime: Runtime | null; + speakingParticipantIds: Set; + micGated: boolean; }; const keyProvider = new ExternalE2EEKeyProvider(); @@ -152,7 +159,7 @@ function clearRemoteAudio() { } } -function getParticipantId(identity: string | undefined): number | null { +export function getParticipantId(identity: string | undefined): number | null { if (!identity) { return null; } @@ -757,6 +764,7 @@ export async function connect(callId: string) { // Tear down the active call session and return the store to a closed state. export async function disconnect() { + disposeSpeakingDetector(); await clearScreenSharePreview(); try { @@ -786,6 +794,7 @@ export async function disconnect() { watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], activeScreenShareParticipantIds: [], + micGated: false, }); room.remoteParticipants.forEach((participant) => { @@ -879,6 +888,7 @@ export async function toggleDeaf() { deafened: nextDeaf ? "true" : "false", }); + getSpeakingDetector().setDeaf(nextDeaf); useCall.setState({ deaf: nextDeaf }); } @@ -967,6 +977,16 @@ async function ensureNoiseFilter( await microphoneTrack.setProcessor(noiseFilter).catch((err) => { log(1, "call", "red", "Failed to enable noise filter", err); }); + + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + const processedTrack = microphoneTrack.mediaStreamTrack; + getSpeakingDetector().addTrack( + participantId, + processedTrack.clone(), + processedTrack, + ); + } } export const useCall = create(() => ({ @@ -996,6 +1016,8 @@ export const useCall = create(() => ({ keyProvider, e2eeWorker, runtime: null, + speakingParticipantIds: new Set(), + micGated: false, })); // Register app-level call listeners and wire React dependencies into the store. @@ -1137,10 +1159,42 @@ export function useInitializeCall() { listenersRegistered.current = true; - const onConnected = () => { + const onConnected = async () => { useCall.setState({ state: "open" }); syncParticipantState(); + const detector = getSpeakingDetector(); + const localParticipantId = getParticipantId( + room.localParticipant.identity, + ); + if (localParticipantId != null) { + detector.setLocalParticipantId(localParticipantId); + } + + const [start, end] = await Promise.all([ + load("call_mute_range_start"), + load("call_mute_range_end"), + ]); + detector.setGateThresholds(start, end); + + // Scan existing audio tracks for speaking detection + for (const participant of getAllParticipants()) { + const participantId = getParticipantId(participant.identity); + if (participantId == null) continue; + + for (const publication of participant.trackPublications.values()) { + if (publication.kind === Track.Kind.Audio && publication.track) { + const mediaTrack = publication.track.mediaStreamTrack; + if (participant === room.localParticipant) { + const clonedTrack = mediaTrack.clone(); + detector.addTrack(participantId, clonedTrack, mediaTrack); + } else { + detector.addTrack(participantId, mediaTrack); + } + } + } + } + const invitedUserId = useCall.getState().invitedUserId; if (invitedUserId != null) { @@ -1177,6 +1231,7 @@ export function useInitializeCall() { if (participantId != null) { stopWatchingStream(participantId); + getSpeakingDetector().removeParticipant(participantId); } syncParticipantState(); @@ -1197,6 +1252,33 @@ export function useInitializeCall() { void ensureNoiseFilter(noiseFilter); }; + const onLocalTrackPublished = (publication: LocalTrackPublication) => { + if (publication.kind === Track.Kind.Audio && publication.track) { + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + const originalTrack = publication.track.mediaStreamTrack; + const clonedTrack = originalTrack.clone(); + getSpeakingDetector().addTrack( + participantId, + clonedTrack, + originalTrack, + ); + } + } + syncParticipantState(); + void ensureNoiseFilter(noiseFilter); + }; + + const onLocalTrackUnpublished = (publication: LocalTrackPublication) => { + if (publication.kind === Track.Kind.Audio) { + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + getSpeakingDetector().removeParticipant(participantId); + } + } + syncParticipantState(); + }; + const onTrackPublished = ( publication: RemoteTrackPublication, participant: RemoteParticipant, @@ -1214,9 +1296,18 @@ export function useInitializeCall() { onParticipantStateChange(); }; - const onTrackSubscribed = (track: RemoteTrack) => { + const onTrackSubscribed = ( + track: RemoteTrack, + _publication: RemoteTrackPublication, + participant: RemoteParticipant, + ) => { if (track.kind === "audio" && track.sid) { attachRemoteAudio(track.sid, track.attach()); + + const participantId = getParticipantId(participant.identity); + if (participantId != null) { + getSpeakingDetector().addTrack(participantId, track.mediaStreamTrack); + } } syncParticipantState(); @@ -1227,12 +1318,16 @@ export function useInitializeCall() { _publication: unknown, participant: Participant, ) => { + const participantId = getParticipantId(participant.identity); + if (track.kind === "audio" && track.sid) { track.detach(); detachRemoteAudio(track.sid); - } - const participantId = getParticipantId(participant.identity); + if (participantId != null) { + getSpeakingDetector().removeParticipant(participantId); + } + } if ( participantId != null && @@ -1256,8 +1351,8 @@ export function useInitializeCall() { room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.on(RoomEvent.TrackMuted, onParticipantStateChange); room.on(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange); - room.on(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); + room.on(RoomEvent.LocalTrackPublished, onLocalTrackPublished); + room.on(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); room.on(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.on(RoomEvent.EncryptionError, onEncryptionError); room.on(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1277,8 +1372,8 @@ export function useInitializeCall() { room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.off(RoomEvent.TrackMuted, onParticipantStateChange); room.off(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange); - room.off(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); + room.off(RoomEvent.LocalTrackPublished, onLocalTrackPublished); + room.off(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); room.off(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.off(RoomEvent.EncryptionError, onEncryptionError); room.off(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1287,7 +1382,7 @@ export function useInitializeCall() { room.disconnect(); e2eeWorker.terminate(); }; - }, [noiseFilter]); + }, [noiseFilter, load]); // fetch call data for preview page useEffect(() => { diff --git a/packages/call/todo.md b/packages/call/todo.md index 1a9c0d1..61d70ed 100644 --- a/packages/call/todo.md +++ b/packages/call/todo.md @@ -1,4 +1,3 @@ -- Speaking indicator - Overlay for stream modals - User modals - Bg based on avatar @@ -12,3 +11,4 @@ - Desktop-App screenshares - Context menus - Popout Window +- If micGated=true & isSpeaking=false for 5 seconds show banner with mic detection diff --git a/packages/shared/src/data.ts b/packages/shared/src/data.ts index e03ca3d..82d3452 100644 --- a/packages/shared/src/data.ts +++ b/packages/shared/src/data.ts @@ -245,6 +245,8 @@ export interface Storage extends SettingsStorageDefaults { cached_contacts: Contacts; cached_communities: Communities; ttp_url: string; + call_mute_range_start: number; + call_mute_range_end: number; } export const storageDefaults: Storage = { @@ -278,4 +280,6 @@ export const storageDefaults: Storage = { cached_contacts: [], cached_communities: [], ttp_url: "https://tensamin.net:959", + call_mute_range_start: -50, + call_mute_range_end: -40, };