(feat): add speaking indicator

(feat): add microphone gate
(qol): update todo

packages/call/src/components/modals/base.tsx

@@ -18,6 +18,7 @@ import {
 import { Track, type Participant } from "livekit-client";
 import { useEffect, useRef, useState } from "react";
 import { useUser, type User } from "@tensamin/user/context";
+import { useIsSpeaking } from "../../speakingIndicator";
 import VideoViewer from "../videoViewer";
 import { HeadphoneOff, MicOff, Monitor, Plus, Shield } from "lucide-react";
 
@@ -103,6 +104,7 @@ export default function Base({
   const focusedParticipantId = useCall((state) => state.focusedParticipantId);
   const view = useCall((state) => state.view);
   const [user, setUser] = useState<User | null>(null);
+  const isSpeaking = useIsSpeaking(user?.user_id ?? -1);
   const screenSharePublication = getTrackPublicationBySource(
     participant,
     Track.Source.ScreenShare,
@@ -211,9 +213,9 @@ export default function Base({
             </div>
             <div
               ref={currentCard}
-              className={`z-10 bg-card absolute top-0 left-0 w-full h-full flex gap-2 items-center justify-center ${
+              className={`transition-all duration-150 z-10 bg-card absolute top-0 left-0 w-full h-full flex gap-2 items-center justify-center ${
                 flush ? "rounded-none" : "rounded-sm"
-              }`}
+              } ${isSpeaking ? "border-4 border-(--primary-foreground-alt)/75" : "border-0"}`}
               style={{ containerType: "size" }}
             >
               {/* Detect video / user and place here */}

packages/call/src/speakingIndicator.ts

@@ -0,0 +1,249 @@
+import { useCall } from "./store";
+
+const SPEAKING_THRESHOLD = 0.01;
+const SPEAKING_HANGTIME_MS = 250;
+const ANALYSIS_INTERVAL_MS = 30;
+const FFT_SIZE = 256;
+
+type AnalyserEntry = {
+  source: MediaStreamAudioSourceNode;
+  analyser: AnalyserNode;
+  track: MediaStreamTrack;
+  originalTrack?: MediaStreamTrack;
+  lastSpeakingTime: number;
+  isSpeaking: boolean;
+};
+
+class SpeakingDetector {
+  private audioContext: AudioContext | null = null;
+  private entries = new Map<number, AnalyserEntry>();
+  private intervalId: ReturnType<typeof setInterval> | null = null;
+  private deaf = false;
+  private gateThresholdStart = -50;
+  private gateThresholdEnd = -40;
+  private localParticipantId: number | null = null;
+  private localMicGateClosed = false;
+
+  private ensureAudioContext(): AudioContext {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext();
+    }
+    if (this.audioContext.state === "suspended") {
+      void this.audioContext.resume();
+    }
+    return this.audioContext;
+  }
+
+  setLocalParticipantId(id: number) {
+    this.localParticipantId = id;
+  }
+
+  setGateThresholds(start: number, end: number) {
+    this.gateThresholdStart = start;
+    this.gateThresholdEnd = end;
+  }
+
+  addTrack(
+    participantId: number,
+    track: MediaStreamTrack,
+    originalTrack?: MediaStreamTrack,
+  ) {
+    if (track.kind !== "audio") return;
+
+    this.removeParticipant(participantId);
+
+    const ctx = this.ensureAudioContext();
+    const stream = new MediaStream([track]);
+    const source = ctx.createMediaStreamSource(stream);
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = FFT_SIZE;
+    source.connect(analyser);
+
+    this.entries.set(participantId, {
+      source,
+      analyser,
+      track,
+      originalTrack,
+      lastSpeakingTime: 0,
+      isSpeaking: false,
+    });
+
+    if (!this.intervalId) {
+      this.startLoop();
+    }
+  }
+
+  removeParticipant(participantId: number) {
+    const entry = this.entries.get(participantId);
+    if (!entry) return;
+
+    try {
+      entry.source.disconnect();
+    } catch {
+      // ignore
+    }
+    this.entries.delete(participantId);
+
+    useCall.setState((state) => {
+      if (!state.speakingParticipantIds.has(participantId)) return state;
+      const next = new Set(state.speakingParticipantIds);
+      next.delete(participantId);
+      return { speakingParticipantIds: next };
+    });
+
+    if (participantId === this.localParticipantId && this.localMicGateClosed) {
+      this.muteLocalTrack(false);
+    }
+  }
+
+  setDeaf(deaf: boolean) {
+    this.deaf = deaf;
+    if (deaf) {
+      for (const entry of this.entries.values()) {
+        entry.isSpeaking = false;
+        entry.lastSpeakingTime = 0;
+      }
+      useCall.setState({ speakingParticipantIds: new Set() });
+    }
+  }
+
+  private startLoop() {
+    if (this.intervalId) return;
+    this.intervalId = setInterval(() => this.analyse(), ANALYSIS_INTERVAL_MS);
+  }
+
+  private stopLoop() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+  }
+
+  private muteLocalTrack(muted: boolean) {
+    const entry = this.localParticipantId
+      ? this.entries.get(this.localParticipantId)
+      : undefined;
+    const target = entry?.originalTrack ?? entry?.track;
+
+    if (target && target.enabled === muted) {
+      target.enabled = !muted;
+    }
+
+    this.localMicGateClosed = muted;
+    useCall.setState({ micGated: muted });
+  }
+
+  private applyNoiseGate(rms: number) {
+    const db = 20 * Math.log10(Math.max(rms, 0.0001));
+
+    if (!this.localMicGateClosed && db < this.gateThresholdStart) {
+      this.muteLocalTrack(true);
+    } else if (this.localMicGateClosed && db > this.gateThresholdEnd) {
+      this.muteLocalTrack(false);
+    }
+  }
+
+  private analyse() {
+    if (this.deaf || this.entries.size === 0) return;
+
+    const now = Date.now();
+    const changed = new Map<number, boolean>();
+
+    for (const [participantId, entry] of this.entries) {
+      const { analyser, track } = entry;
+
+      if (track.muted || track.readyState === "ended" || !track.enabled) {
+        if (entry.isSpeaking) {
+          entry.isSpeaking = false;
+          entry.lastSpeakingTime = 0;
+          changed.set(participantId, false);
+        }
+        continue;
+      }
+
+      const bufferLength = analyser.frequencyBinCount;
+      const dataArray = new Uint8Array(bufferLength);
+      analyser.getByteTimeDomainData(dataArray);
+
+      let sum = 0;
+      for (let i = 0; i < bufferLength; i++) {
+        const sample = (dataArray[i] - 128) / 128.0;
+        sum += sample * sample;
+      }
+      const rms = Math.sqrt(sum / bufferLength);
+
+      let nextIsSpeaking = entry.isSpeaking;
+      if (rms > SPEAKING_THRESHOLD) {
+        entry.lastSpeakingTime = now;
+        nextIsSpeaking = true;
+      } else if (now - entry.lastSpeakingTime > SPEAKING_HANGTIME_MS) {
+        nextIsSpeaking = false;
+      }
+
+      if (participantId === this.localParticipantId) {
+        this.applyNoiseGate(rms);
+        if (this.localMicGateClosed) {
+          nextIsSpeaking = false;
+        }
+      }
+
+      if (nextIsSpeaking !== entry.isSpeaking) {
+        entry.isSpeaking = nextIsSpeaking;
+        changed.set(participantId, nextIsSpeaking);
+      }
+    }
+
+    if (changed.size > 0) {
+      useCall.setState((state) => {
+        let hasDiff = false;
+        const next = new Set(state.speakingParticipantIds);
+        for (const [id, speaking] of changed) {
+          if (speaking) {
+            if (!next.has(id)) {
+              next.add(id);
+              hasDiff = true;
+            }
+          } else {
+            if (next.has(id)) {
+              next.delete(id);
+              hasDiff = true;
+            }
+          }
+        }
+        return hasDiff ? { speakingParticipantIds: next } : state;
+      });
+    }
+  }
+
+  dispose() {
+    this.stopLoop();
+    for (const id of Array.from(this.entries.keys())) {
+      this.removeParticipant(id);
+    }
+    this.entries.clear();
+    if (this.audioContext) {
+      void this.audioContext.close();
+      this.audioContext = null;
+    }
+  }
+}
+
+let detectorInstance: SpeakingDetector | null = null;
+
+export function getSpeakingDetector(): SpeakingDetector {
+  if (!detectorInstance) {
+    detectorInstance = new SpeakingDetector();
+  }
+  return detectorInstance;
+}
+
+export function disposeSpeakingDetector(): void {
+  if (detectorInstance) {
+    detectorInstance.dispose();
+    detectorInstance = null;
+  }
+}
+
+export function useIsSpeaking(participantId: number): boolean {
+  return useCall((state) => state.speakingParticipantIds.has(participantId));
+}

packages/call/src/store.tsx

@@ -11,6 +11,7 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter";
 import {
   ExternalE2EEKeyProvider,
   LocalAudioTrack,
+  type LocalTrackPublication,
   type Participant,
   type RemoteParticipant,
   type RemoteTrackPublication,
@@ -28,6 +29,10 @@ import {
   createScreenShareController,
   type ScreenShareSession,
 } from "./screenshare";
+import {
+  getSpeakingDetector,
+  disposeSpeakingDetector,
+} from "./speakingIndicator";
 
 // logging
 setLogExtension(
@@ -99,6 +104,8 @@ type CallStore = {
   keyProvider: ExternalE2EEKeyProvider;
   e2eeWorker: Worker;
   runtime: Runtime | null;
+  speakingParticipantIds: Set<number>;
+  micGated: boolean;
 };
 
 const keyProvider = new ExternalE2EEKeyProvider();
@@ -152,7 +159,7 @@ function clearRemoteAudio() {
   }
 }
 
-function getParticipantId(identity: string | undefined): number | null {
+export function getParticipantId(identity: string | undefined): number | null {
   if (!identity) {
     return null;
   }
@@ -757,6 +764,7 @@ export async function connect(callId: string) {
 
 // Tear down the active call session and return the store to a closed state.
 export async function disconnect() {
+  disposeSpeakingDetector();
   await clearScreenSharePreview();
 
   try {
@@ -786,6 +794,7 @@ export async function disconnect() {
     watchedStreamParticipantIds: [],
     pendingWatchedParticipantIds: [],
     activeScreenShareParticipantIds: [],
+    micGated: false,
   });
 
   room.remoteParticipants.forEach((participant) => {
@@ -879,6 +888,7 @@ export async function toggleDeaf() {
     deafened: nextDeaf ? "true" : "false",
   });
 
+  getSpeakingDetector().setDeaf(nextDeaf);
   useCall.setState({ deaf: nextDeaf });
 }
 
@@ -967,6 +977,16 @@ async function ensureNoiseFilter(
   await microphoneTrack.setProcessor(noiseFilter).catch((err) => {
     log(1, "call", "red", "Failed to enable noise filter", err);
   });
+
+  const participantId = getParticipantId(room.localParticipant.identity);
+  if (participantId != null) {
+    const processedTrack = microphoneTrack.mediaStreamTrack;
+    getSpeakingDetector().addTrack(
+      participantId,
+      processedTrack.clone(),
+      processedTrack,
+    );
+  }
 }
 
 export const useCall = create<CallStore>(() => ({
@@ -996,6 +1016,8 @@ export const useCall = create<CallStore>(() => ({
   keyProvider,
   e2eeWorker,
   runtime: null,
+  speakingParticipantIds: new Set(),
+  micGated: false,
 }));
 
 // Register app-level call listeners and wire React dependencies into the store.
@@ -1137,10 +1159,42 @@ export function useInitializeCall() {
 
     listenersRegistered.current = true;
 
-    const onConnected = () => {
+    const onConnected = async () => {
       useCall.setState({ state: "open" });
       syncParticipantState();
 
+      const detector = getSpeakingDetector();
+      const localParticipantId = getParticipantId(
+        room.localParticipant.identity,
+      );
+      if (localParticipantId != null) {
+        detector.setLocalParticipantId(localParticipantId);
+      }
+
+      const [start, end] = await Promise.all([
+        load("call_mute_range_start"),
+        load("call_mute_range_end"),
+      ]);
+      detector.setGateThresholds(start, end);
+
+      // Scan existing audio tracks for speaking detection
+      for (const participant of getAllParticipants()) {
+        const participantId = getParticipantId(participant.identity);
+        if (participantId == null) continue;
+
+        for (const publication of participant.trackPublications.values()) {
+          if (publication.kind === Track.Kind.Audio && publication.track) {
+            const mediaTrack = publication.track.mediaStreamTrack;
+            if (participant === room.localParticipant) {
+              const clonedTrack = mediaTrack.clone();
+              detector.addTrack(participantId, clonedTrack, mediaTrack);
+            } else {
+              detector.addTrack(participantId, mediaTrack);
+            }
+          }
+        }
+      }
+
       const invitedUserId = useCall.getState().invitedUserId;
 
       if (invitedUserId != null) {
@@ -1177,6 +1231,7 @@ export function useInitializeCall() {
 
       if (participantId != null) {
         stopWatchingStream(participantId);
+        getSpeakingDetector().removeParticipant(participantId);
       }
 
       syncParticipantState();
@@ -1197,6 +1252,33 @@ export function useInitializeCall() {
       void ensureNoiseFilter(noiseFilter);
     };
 
+    const onLocalTrackPublished = (publication: LocalTrackPublication) => {
+      if (publication.kind === Track.Kind.Audio && publication.track) {
+        const participantId = getParticipantId(room.localParticipant.identity);
+        if (participantId != null) {
+          const originalTrack = publication.track.mediaStreamTrack;
+          const clonedTrack = originalTrack.clone();
+          getSpeakingDetector().addTrack(
+            participantId,
+            clonedTrack,
+            originalTrack,
+          );
+        }
+      }
+      syncParticipantState();
+      void ensureNoiseFilter(noiseFilter);
+    };
+
+    const onLocalTrackUnpublished = (publication: LocalTrackPublication) => {
+      if (publication.kind === Track.Kind.Audio) {
+        const participantId = getParticipantId(room.localParticipant.identity);
+        if (participantId != null) {
+          getSpeakingDetector().removeParticipant(participantId);
+        }
+      }
+      syncParticipantState();
+    };
+
     const onTrackPublished = (
       publication: RemoteTrackPublication,
       participant: RemoteParticipant,
@@ -1214,9 +1296,18 @@ export function useInitializeCall() {
       onParticipantStateChange();
     };
 
-    const onTrackSubscribed = (track: RemoteTrack) => {
+    const onTrackSubscribed = (
+      track: RemoteTrack,
+      _publication: RemoteTrackPublication,
+      participant: RemoteParticipant,
+    ) => {
       if (track.kind === "audio" && track.sid) {
         attachRemoteAudio(track.sid, track.attach());
+
+        const participantId = getParticipantId(participant.identity);
+        if (participantId != null) {
+          getSpeakingDetector().addTrack(participantId, track.mediaStreamTrack);
+        }
       }
 
       syncParticipantState();
@@ -1227,12 +1318,16 @@ export function useInitializeCall() {
       _publication: unknown,
       participant: Participant,
     ) => {
+      const participantId = getParticipantId(participant.identity);
+
       if (track.kind === "audio" && track.sid) {
         track.detach();
         detachRemoteAudio(track.sid);
-      }
 
-      const participantId = getParticipantId(participant.identity);
+        if (participantId != null) {
+          getSpeakingDetector().removeParticipant(participantId);
+        }
+      }
 
       if (
         participantId != null &&
@@ -1256,8 +1351,8 @@ export function useInitializeCall() {
     room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
     room.on(RoomEvent.TrackMuted, onParticipantStateChange);
     room.on(RoomEvent.TrackUnmuted, onParticipantStateChange);
-    room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange);
-    room.on(RoomEvent.LocalTrackUnpublished, onParticipantStateChange);
+    room.on(RoomEvent.LocalTrackPublished, onLocalTrackPublished);
+    room.on(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished);
     room.on(RoomEvent.MediaDevicesError, onMediaDeviceFailure);
     room.on(RoomEvent.EncryptionError, onEncryptionError);
     room.on(RoomEvent.ConnectionStateChanged, onParticipantStateChange);
@@ -1277,8 +1372,8 @@ export function useInitializeCall() {
       room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
       room.off(RoomEvent.TrackMuted, onParticipantStateChange);
       room.off(RoomEvent.TrackUnmuted, onParticipantStateChange);
-      room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange);
-      room.off(RoomEvent.LocalTrackUnpublished, onParticipantStateChange);
+      room.off(RoomEvent.LocalTrackPublished, onLocalTrackPublished);
+      room.off(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished);
       room.off(RoomEvent.MediaDevicesError, onMediaDeviceFailure);
       room.off(RoomEvent.EncryptionError, onEncryptionError);
       room.off(RoomEvent.ConnectionStateChanged, onParticipantStateChange);
@@ -1287,7 +1382,7 @@ export function useInitializeCall() {
       room.disconnect();
       e2eeWorker.terminate();
     };
-  }, [noiseFilter]);
+  }, [noiseFilter, load]);
 
   // fetch call data for preview page
   useEffect(() => {