(fix): speaking indicator broken if screensharing, audio was autosubscribed to

(fix): weird behaviour related to users that are screensharing

packages/call/src/components/buttons/screenshare.tsx

@@ -18,10 +18,12 @@ export default function ScreenshareButton({
   className,
   iconSize,
   tooltip,
+  defaultPortal,
 }: {
   className?: string;
   iconSize?: number;
   tooltip?: string;
+  defaultPortal?: boolean;
 }) {
   const isScreensharing = useCall((state) => state.screenShareEnabled);
   const screenRef = useCall((state) => state.screenRef);
@@ -30,8 +32,9 @@ export default function ScreenshareButton({
   const [menuOpen, setMenuOpen] = useState(false);
 
   useEffect(() => {
+    if (defaultPortal) return;
     setPortalContainer(screenRef?.current ?? undefined);
-  }, [screenRef]);
+  }, [screenRef, defaultPortal]);
 
   async function startWebShare() {
     try {
@@ -110,7 +113,9 @@ export default function ScreenshareButton({
           />
           <PopoverContent
             className="flex w-40 flex-col gap-2"
-            portalProps={{ container: portalContainer }}
+            portalProps={{
+              container: defaultPortal ? undefined : portalContainer,
+            }}
           >
             <Button
               disabled={isScreensharing}
@@ -141,7 +146,11 @@ export default function ScreenshareButton({
           </PopoverContent>
         </Popover>
         {tooltip && (
-          <TooltipContent portalProps={{ container: portalContainer }}>
+          <TooltipContent
+            portalProps={{
+              container: defaultPortal ? undefined : portalContainer,
+            }}
+          >
             {tooltip}
           </TooltipContent>
         )}

packages/call/src/components/modals/base.tsx

@@ -18,6 +18,7 @@ import {
 import { Track, type Participant } from "livekit-client";
 import { useEffect, useRef, useState } from "react";
 import { useUser, type User } from "@tensamin/user/context";
+import { useIsSpeaking } from "../../speakingIndicator";
 import VideoViewer from "../videoViewer";
 import { HeadphoneOff, MicOff, Monitor, Plus, Shield } from "lucide-react";
 
@@ -103,6 +104,7 @@ export default function Base({
   const focusedParticipantId = useCall((state) => state.focusedParticipantId);
   const view = useCall((state) => state.view);
   const [user, setUser] = useState<User | null>(null);
+  const isSpeaking = useIsSpeaking(user?.user_id ?? -1);
   const screenSharePublication = getTrackPublicationBySource(
     participant,
     Track.Source.ScreenShare,
@@ -150,12 +152,12 @@ export default function Base({
 
   const onClick = () => {
     if (view === "grid") {
-      focusParticipant(user.user_id);
+      focusParticipant(user.user_id, type);
     } else {
       if (user.user_id === focusedParticipantId) {
         setCallView("grid");
       } else {
-        focusParticipant(user.user_id);
+        focusParticipant(user.user_id, type);
       }
     }
   };
@@ -211,9 +213,9 @@ export default function Base({
             </div>
             <div
               ref={currentCard}
-              className={`z-10 bg-card absolute top-0 left-0 w-full h-full flex gap-2 items-center justify-center ${
+              className={`transition-all duration-150 z-10 bg-card absolute top-0 left-0 w-full h-full flex gap-2 items-center justify-center ${
                 flush ? "rounded-none" : "rounded-sm"
-              }`}
+              } ${type === "user" && isSpeaking ? "border-4 border-(--primary-foreground-alt)/75" : "border-0"}`}
               style={{ containerType: "size" }}
             >
               {/* Detect video / user and place here */}

packages/call/src/components/sidebarBox.tsx

@@ -20,24 +20,18 @@ import LeaveButton from "./buttons/leave";
 
 export default function SidebarBox() {
   const state = useCall((store) => store.state);
-  const screenRef = useCall((store) => store.screenRef);
   const isMobile = useIsMobile();
-  const [portalContainer, setPortalContainer] = useState<HTMLElement>();
-
-  useEffect(() => {
-    setPortalContainer(screenRef?.current ?? undefined);
-  }, [screenRef]);
 
   return state === "closed" ? null : (
     <Card className="p-1.5 gap-2" hidden={isMobile}>
       <CardHeader className="p-0! pb-2! border-b-2">
-        <ConnectionBar portalContainer={portalContainer} />
+        <ConnectionBar />
       </CardHeader>
       <CardContent className="p-0! flex flex-col gap-1">
         <div className="flex justify-start gap-1">
           <MuteButton className="w-9 h-9" />
           <DeafButton className="w-9 h-9" />
-          <ScreenshareButton className="w-9 h-9" />
+          <ScreenshareButton className="w-9 h-9" defaultPortal />
           <LeaveButton className="w-9 h-9" />
         </div>
       </CardContent>
@@ -45,7 +39,7 @@ export default function SidebarBox() {
   );
 }
 
-function ConnectionBar({ portalContainer }: { portalContainer?: HTMLElement }) {
+function ConnectionBar() {
   const state = useCall((store) => store.state);
   const isEncrypted = useCall((store) => store.isEncrypted);
   const callId = useCall((store) => store.callId);
@@ -68,7 +62,7 @@ function ConnectionBar({ portalContainer }: { portalContainer?: HTMLElement }) {
             {state === "closed" && "Closed"}
             {state === "closing" && "Closing"}
 
-            <TinyPingGraph portalContainer={portalContainer} />
+            <TinyPingGraph />
 
             {isEncrypted ? (
               <Lock color="var(--primary-foreground-alt)" />
@@ -78,18 +72,12 @@ function ConnectionBar({ portalContainer }: { portalContainer?: HTMLElement }) {
           </Button>
         }
       />
-      <TooltipContent portalProps={{ container: portalContainer }}>
+      <TooltipContent>Click to open call page</TooltipContent>
-        Click to open call page
-      </TooltipContent>
     </Tooltip>
   );
 }
 
-export function TinyPingGraph({
+export function TinyPingGraph() {
-  portalContainer,
-}: {
-  portalContainer?: HTMLElement;
-}) {
   const room = useCall((store) => store.room);
 
   const [mapData, setMapData] = useState<Map<number, number>>(() => new Map());
@@ -176,7 +164,7 @@ export function TinyPingGraph({
           </div>
         }
       />
-      <TooltipContent portalProps={{ container: portalContainer }}>
+      <TooltipContent>
         {data.length > 0 ? `${data.at(-1)?.ping} ms` : "Measuring ping..."}
       </TooltipContent>
     </Tooltip>

packages/call/src/speakingIndicator.ts

@@ -0,0 +1,252 @@
+import { log } from "@tensamin/shared/log";
+import { useCall } from "./store";
+
+const SPEAKING_THRESHOLD = 0.01;
+const SPEAKING_HANGTIME_MS = 500;
+const ANALYSIS_INTERVAL_MS = 30;
+const FFT_SIZE = 256;
+
+type AnalyserEntry = {
+  source: MediaStreamAudioSourceNode;
+  analyser: AnalyserNode;
+  track: MediaStreamTrack;
+  originalTrack?: MediaStreamTrack;
+  lastSpeakingTime: number;
+  isSpeaking: boolean;
+};
+
+class SpeakingDetector {
+  private audioContext: AudioContext | null = null;
+  private entries = new Map<number, AnalyserEntry>();
+  private intervalId: ReturnType<typeof setInterval> | null = null;
+  private deaf = false;
+  private gateThresholdStart = -50;
+  private gateThresholdEnd = -40;
+  private localParticipantId: number | null = null;
+  private localMicGateClosed = false;
+
+  private ensureAudioContext(): AudioContext {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext();
+    }
+    if (this.audioContext.state === "suspended") {
+      void this.audioContext.resume();
+    }
+    return this.audioContext;
+  }
+
+  setLocalParticipantId(id: number) {
+    this.localParticipantId = id;
+  }
+
+  setGateThresholds(start: number, end: number) {
+    this.gateThresholdStart = start;
+    this.gateThresholdEnd = end;
+  }
+
+  addTrack(
+    participantId: number,
+    track: MediaStreamTrack,
+    originalTrack?: MediaStreamTrack,
+  ) {
+    if (track.kind !== "audio") return;
+
+    this.removeParticipant(participantId);
+
+    const ctx = this.ensureAudioContext();
+    const stream = new MediaStream([track]);
+    const source = ctx.createMediaStreamSource(stream);
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = FFT_SIZE;
+    source.connect(analyser);
+
+    this.entries.set(participantId, {
+      source,
+      analyser,
+      track,
+      originalTrack,
+      lastSpeakingTime: 0,
+      isSpeaking: false,
+    });
+
+    if (!this.intervalId) {
+      this.startLoop();
+    }
+  }
+
+  removeParticipant(participantId: number) {
+    const entry = this.entries.get(participantId);
+    if (!entry) return;
+
+    try {
+      entry.source.disconnect();
+    } catch {
+      // ignore
+    }
+    this.entries.delete(participantId);
+
+    useCall.setState((state) => {
+      if (!state.speakingParticipantIds.has(participantId)) return state;
+      const next = new Set(state.speakingParticipantIds);
+      next.delete(participantId);
+      return { speakingParticipantIds: next };
+    });
+
+    if (participantId === this.localParticipantId && this.localMicGateClosed) {
+      this.muteLocalTrack(false);
+    }
+  }
+
+  setDeaf(deaf: boolean) {
+    this.deaf = deaf;
+    if (deaf) {
+      for (const entry of this.entries.values()) {
+        entry.isSpeaking = false;
+        entry.lastSpeakingTime = 0;
+      }
+      useCall.setState({ speakingParticipantIds: new Set() });
+    }
+  }
+
+  private startLoop() {
+    if (this.intervalId) return;
+    this.intervalId = setInterval(() => this.analyse(), ANALYSIS_INTERVAL_MS);
+  }
+
+  private stopLoop() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+  }
+
+  private muteLocalTrack(muted: boolean) {
+    const entry = this.localParticipantId
+      ? this.entries.get(this.localParticipantId)
+      : undefined;
+    const target = entry?.originalTrack ?? entry?.track;
+
+    if (target && target.enabled === muted) {
+      target.enabled = !muted;
+    }
+
+    this.localMicGateClosed = muted;
+    useCall.setState({ micGated: muted });
+  }
+
+  private applyNoiseGate(rms: number) {
+    const db = 20 * Math.log10(Math.max(rms, 0.0001));
+
+    if (!this.localMicGateClosed && db < this.gateThresholdStart) {
+      log(3, "noise gate", "purple", "closed");
+      this.muteLocalTrack(true);
+    } else if (this.localMicGateClosed && db > this.gateThresholdEnd) {
+      log(3, "noise gate", "purple", "opened");
+      this.muteLocalTrack(false);
+    }
+  }
+
+  private analyse() {
+    if (this.deaf || this.entries.size === 0) return;
+
+    const now = Date.now();
+    const changed = new Map<number, boolean>();
+
+    for (const [participantId, entry] of this.entries) {
+      const { analyser, track } = entry;
+
+      if (track.muted || track.readyState === "ended" || !track.enabled) {
+        if (entry.isSpeaking) {
+          entry.isSpeaking = false;
+          entry.lastSpeakingTime = 0;
+          changed.set(participantId, false);
+        }
+        continue;
+      }
+
+      const bufferLength = analyser.frequencyBinCount;
+      const dataArray = new Uint8Array(bufferLength);
+      analyser.getByteTimeDomainData(dataArray);
+
+      let sum = 0;
+      for (let i = 0; i < bufferLength; i++) {
+        const sample = (dataArray[i] - 128) / 128.0;
+        sum += sample * sample;
+      }
+      const rms = Math.sqrt(sum / bufferLength);
+
+      let nextIsSpeaking = entry.isSpeaking;
+      if (rms > SPEAKING_THRESHOLD) {
+        entry.lastSpeakingTime = now;
+        nextIsSpeaking = true;
+      } else if (now - entry.lastSpeakingTime > SPEAKING_HANGTIME_MS) {
+        nextIsSpeaking = false;
+      }
+
+      if (participantId === this.localParticipantId) {
+        this.applyNoiseGate(rms);
+        if (this.localMicGateClosed) {
+          nextIsSpeaking = false;
+        }
+      }
+
+      if (nextIsSpeaking !== entry.isSpeaking) {
+        entry.isSpeaking = nextIsSpeaking;
+        changed.set(participantId, nextIsSpeaking);
+      }
+    }
+
+    if (changed.size > 0) {
+      useCall.setState((state) => {
+        let hasDiff = false;
+        const next = new Set(state.speakingParticipantIds);
+        for (const [id, speaking] of changed) {
+          if (speaking) {
+            if (!next.has(id)) {
+              next.add(id);
+              hasDiff = true;
+            }
+          } else {
+            if (next.has(id)) {
+              next.delete(id);
+              hasDiff = true;
+            }
+          }
+        }
+        return hasDiff ? { speakingParticipantIds: next } : state;
+      });
+    }
+  }
+
+  dispose() {
+    this.stopLoop();
+    for (const id of Array.from(this.entries.keys())) {
+      this.removeParticipant(id);
+    }
+    this.entries.clear();
+    if (this.audioContext) {
+      void this.audioContext.close();
+      this.audioContext = null;
+    }
+  }
+}
+
+let detectorInstance: SpeakingDetector | null = null;
+
+export function getSpeakingDetector(): SpeakingDetector {
+  if (!detectorInstance) {
+    detectorInstance = new SpeakingDetector();
+  }
+  return detectorInstance;
+}
+
+export function disposeSpeakingDetector(): void {
+  if (detectorInstance) {
+    detectorInstance.dispose();
+    detectorInstance = null;
+  }
+}
+
+export function useIsSpeaking(participantId: number): boolean {
+  return useCall((state) => state.speakingParticipantIds.has(participantId));
+}

packages/call/src/store.tsx

@@ -11,6 +11,7 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter";
 import {
   ExternalE2EEKeyProvider,
   LocalAudioTrack,
+  type LocalTrackPublication,
   type Participant,
   type RemoteParticipant,
   type RemoteTrackPublication,
@@ -28,6 +29,10 @@ import {
   createScreenShareController,
   type ScreenShareSession,
 } from "./screenshare";
+import {
+  getSpeakingDetector,
+  disposeSpeakingDetector,
+} from "./speakingIndicator";
 
 // logging
 setLogExtension(
@@ -86,6 +91,7 @@ type CallStore = {
   screenShareEnabled: boolean;
   screenShareSession: ScreenShareSession | null;
   focusedParticipantId: number | null;
+  focusedParticipantType: "user" | "stream" | null;
   usersInFocusedViewHidden: boolean;
   watchedStreamParticipantIds: number[];
   pendingWatchedParticipantIds: number[];
@@ -99,6 +105,8 @@ type CallStore = {
   keyProvider: ExternalE2EEKeyProvider;
   e2eeWorker: Worker;
   runtime: Runtime | null;
+  speakingParticipantIds: Set<number>;
+  micGated: boolean;
 };
 
 const keyProvider = new ExternalE2EEKeyProvider();
@@ -152,7 +160,7 @@ function clearRemoteAudio() {
   }
 }
 
-function getParticipantId(identity: string | undefined): number | null {
+export function getParticipantId(identity: string | undefined): number | null {
   if (!identity) {
     return null;
   }
@@ -199,7 +207,10 @@ function matchesRemoteTrackSelector(
 
 function syncRemoteParticipantTrackSubscriptions(participantId: number) {
   for (const publication of getRemoteTrackPublications(participantId)) {
-    publication.setSubscribed(publication.kind === Track.Kind.Audio);
+    publication.setSubscribed(
+      publication.kind === Track.Kind.Audio &&
+        publication.source !== Track.Source.ScreenShareAudio,
+    );
   }
 }
 
@@ -457,6 +468,8 @@ function syncScreenShareParticipants() {
     watchedStreamParticipantIds,
     pendingWatchedParticipantIds,
     focusedParticipantId,
+    focusedParticipantType:
+      focusedParticipantId == null ? null : state.focusedParticipantType,
     view:
       state.view === "focused" && focusedParticipantId == null
         ? "grid"
@@ -614,6 +627,7 @@ export function startWatchingStream(participantId: number) {
   const trackReady = getScreenShareTrackForParticipant(participantId) != null;
 
   setParticipantTrackSubscribed(participantId, Track.Source.ScreenShare);
+  setParticipantTrackSubscribed(participantId, Track.Source.ScreenShareAudio);
 
   useCall.setState((state) => ({
     watchedStreamParticipantIds: state.watchedStreamParticipantIds.includes(
@@ -627,6 +641,7 @@ export function startWatchingStream(participantId: number) {
         ? state.pendingWatchedParticipantIds
         : [...state.pendingWatchedParticipantIds, participantId],
     focusedParticipantId: participantId,
+    focusedParticipantType: "stream",
   }));
 }
 
@@ -646,9 +661,13 @@ export function setParticipantTrackSubscribed(
 }
 
 // Focus a participant in the main call view even when they are not sharing a screen.
-export function focusParticipant(participantId: number) {
+export function focusParticipant(
+  participantId: number,
+  type: "user" | "stream" = "user",
+) {
   useCall.setState({
     focusedParticipantId: participantId,
+    focusedParticipantType: type,
     view: "focused",
   });
 }
@@ -656,6 +675,11 @@ export function focusParticipant(participantId: number) {
 // Stop tracking a participant's shared screen and clean up related UI state.
 export function stopWatchingStream(participantId: number) {
   setParticipantTrackSubscribed(participantId, Track.Source.ScreenShare, false);
+  setParticipantTrackSubscribed(
+    participantId,
+    Track.Source.ScreenShareAudio,
+    false,
+  );
 
   useCall.setState((state) => ({
     watchedStreamParticipantIds: state.watchedStreamParticipantIds.filter(
@@ -668,6 +692,10 @@ export function stopWatchingStream(participantId: number) {
       state.focusedParticipantId === participantId
         ? null
         : state.focusedParticipantId,
+    focusedParticipantType:
+      state.focusedParticipantId === participantId
+        ? null
+        : state.focusedParticipantType,
     view:
       state.view === "focused" && state.focusedParticipantId === participantId
         ? "grid"
@@ -757,6 +785,7 @@ export async function connect(callId: string) {
 
 // Tear down the active call session and return the store to a closed state.
 export async function disconnect() {
+  disposeSpeakingDetector();
   await clearScreenSharePreview();
 
   try {
@@ -782,10 +811,12 @@ export async function disconnect() {
     view: "preview",
     screenShareSession: null,
     focusedParticipantId: null,
+    focusedParticipantType: null,
     usersInFocusedViewHidden: false,
     watchedStreamParticipantIds: [],
     pendingWatchedParticipantIds: [],
     activeScreenShareParticipantIds: [],
+    micGated: false,
   });
 
   room.remoteParticipants.forEach((participant) => {
@@ -879,6 +910,7 @@ export async function toggleDeaf() {
     deafened: nextDeaf ? "true" : "false",
   });
 
+  getSpeakingDetector().setDeaf(nextDeaf);
   useCall.setState({ deaf: nextDeaf });
 }
 
@@ -940,6 +972,7 @@ export function resetCallState() {
     deaf: false,
     screenShareSession: null,
     focusedParticipantId: null,
+    focusedParticipantType: null,
     usersInFocusedViewHidden: false,
     watchedStreamParticipantIds: [],
     pendingWatchedParticipantIds: [],
@@ -967,6 +1000,16 @@ async function ensureNoiseFilter(
   await microphoneTrack.setProcessor(noiseFilter).catch((err) => {
     log(1, "call", "red", "Failed to enable noise filter", err);
   });
+
+  const participantId = getParticipantId(room.localParticipant.identity);
+  if (participantId != null) {
+    const processedTrack = microphoneTrack.mediaStreamTrack;
+    getSpeakingDetector().addTrack(
+      participantId,
+      processedTrack.clone(),
+      processedTrack,
+    );
+  }
 }
 
 export const useCall = create<CallStore>(() => ({
@@ -982,6 +1025,7 @@ export const useCall = create<CallStore>(() => ({
   screenShareEnabled: room.localParticipant.isScreenShareEnabled,
   screenShareSession: null,
   focusedParticipantId: null,
+  focusedParticipantType: null,
   usersInFocusedViewHidden: false,
   watchedStreamParticipantIds: [],
   pendingWatchedParticipantIds: [],
@@ -996,6 +1040,8 @@ export const useCall = create<CallStore>(() => ({
   keyProvider,
   e2eeWorker,
   runtime: null,
+  speakingParticipantIds: new Set(),
+  micGated: false,
 }));
 
 // Register app-level call listeners and wire React dependencies into the store.
@@ -1016,7 +1062,7 @@ export function useInitializeCall() {
       new DeepFilterNoiseFilterProcessor({
         enabled: true,
         enableNoiseReduction: true,
-        noiseReductionLevel: 80,
+        noiseReductionLevel: 60,
         sampleRate: 48000,
         assetConfig: {
           cdnUrl: "/assets",
@@ -1137,10 +1183,46 @@ export function useInitializeCall() {
 
     listenersRegistered.current = true;
 
-    const onConnected = () => {
+    const onConnected = async () => {
       useCall.setState({ state: "open" });
       syncParticipantState();
 
+      const detector = getSpeakingDetector();
+      const localParticipantId = getParticipantId(
+        room.localParticipant.identity,
+      );
+      if (localParticipantId != null) {
+        detector.setLocalParticipantId(localParticipantId);
+      }
+
+      const [start, end] = await Promise.all([
+        load("call_mute_range_start"),
+        load("call_mute_range_end"),
+      ]);
+      detector.setGateThresholds(start, end);
+
+      // Scan existing audio tracks for speaking detection
+      for (const participant of getAllParticipants()) {
+        const participantId = getParticipantId(participant.identity);
+        if (participantId == null) continue;
+
+        for (const publication of participant.trackPublications.values()) {
+          if (
+            publication.kind === Track.Kind.Audio &&
+            publication.source === Track.Source.Microphone &&
+            publication.track
+          ) {
+            const mediaTrack = publication.track.mediaStreamTrack;
+            if (participant === room.localParticipant) {
+              const clonedTrack = mediaTrack.clone();
+              detector.addTrack(participantId, clonedTrack, mediaTrack);
+            } else {
+              detector.addTrack(participantId, mediaTrack);
+            }
+          }
+        }
+      }
+
       const invitedUserId = useCall.getState().invitedUserId;
 
       if (invitedUserId != null) {
@@ -1177,6 +1259,7 @@ export function useInitializeCall() {
 
       if (participantId != null) {
         stopWatchingStream(participantId);
+        getSpeakingDetector().removeParticipant(participantId);
       }
 
       syncParticipantState();
@@ -1197,6 +1280,40 @@ export function useInitializeCall() {
       void ensureNoiseFilter(noiseFilter);
     };
 
+    const onLocalTrackPublished = (publication: LocalTrackPublication) => {
+      if (
+        publication.kind === Track.Kind.Audio &&
+        publication.source === Track.Source.Microphone &&
+        publication.track
+      ) {
+        const participantId = getParticipantId(room.localParticipant.identity);
+        if (participantId != null) {
+          const originalTrack = publication.track.mediaStreamTrack;
+          const clonedTrack = originalTrack.clone();
+          getSpeakingDetector().addTrack(
+            participantId,
+            clonedTrack,
+            originalTrack,
+          );
+        }
+      }
+      syncParticipantState();
+      void ensureNoiseFilter(noiseFilter);
+    };
+
+    const onLocalTrackUnpublished = (publication: LocalTrackPublication) => {
+      if (
+        publication.kind === Track.Kind.Audio &&
+        publication.source === Track.Source.Microphone
+      ) {
+        const participantId = getParticipantId(room.localParticipant.identity);
+        if (participantId != null) {
+          getSpeakingDetector().removeParticipant(participantId);
+        }
+      }
+      syncParticipantState();
+    };
+
     const onTrackPublished = (
       publication: RemoteTrackPublication,
       participant: RemoteParticipant,
@@ -1204,7 +1321,10 @@ export function useInitializeCall() {
       const participantId = getParticipantId(participant.identity);
 
       if (participantId != null) {
-        if (publication.kind === Track.Kind.Audio) {
+        if (
+          publication.kind === Track.Kind.Audio &&
+          publication.source !== Track.Source.ScreenShareAudio
+        ) {
           publication.setSubscribed(true);
         } else {
           publication.setSubscribed(false);
@@ -1214,9 +1334,21 @@ export function useInitializeCall() {
       onParticipantStateChange();
     };
 
-    const onTrackSubscribed = (track: RemoteTrack) => {
+    const onTrackSubscribed = (
+      track: RemoteTrack,
+      publication: RemoteTrackPublication,
+      participant: RemoteParticipant,
+    ) => {
       if (track.kind === "audio" && track.sid) {
         attachRemoteAudio(track.sid, track.attach());
+
+        const participantId = getParticipantId(participant.identity);
+        if (
+          participantId != null &&
+          publication.source === Track.Source.Microphone
+        ) {
+          getSpeakingDetector().addTrack(participantId, track.mediaStreamTrack);
+        }
       }
 
       syncParticipantState();
@@ -1224,15 +1356,22 @@ export function useInitializeCall() {
 
     const onTrackUnsubscribed = (
       track: RemoteTrack,
-      _publication: unknown,
+      publication: RemoteTrackPublication,
       participant: Participant,
     ) => {
+      const participantId = getParticipantId(participant.identity);
+
       if (track.kind === "audio" && track.sid) {
         track.detach();
         detachRemoteAudio(track.sid);
-      }
 
-      const participantId = getParticipantId(participant.identity);
+        if (
+          participantId != null &&
+          publication.source === Track.Source.Microphone
+        ) {
+          getSpeakingDetector().removeParticipant(participantId);
+        }
+      }
 
       if (
         participantId != null &&
@@ -1256,8 +1395,8 @@ export function useInitializeCall() {
     room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
     room.on(RoomEvent.TrackMuted, onParticipantStateChange);
     room.on(RoomEvent.TrackUnmuted, onParticipantStateChange);
-    room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange);
+    room.on(RoomEvent.LocalTrackPublished, onLocalTrackPublished);
-    room.on(RoomEvent.LocalTrackUnpublished, onParticipantStateChange);
+    room.on(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished);
     room.on(RoomEvent.MediaDevicesError, onMediaDeviceFailure);
     room.on(RoomEvent.EncryptionError, onEncryptionError);
     room.on(RoomEvent.ConnectionStateChanged, onParticipantStateChange);
@@ -1277,8 +1416,8 @@ export function useInitializeCall() {
       room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected);
       room.off(RoomEvent.TrackMuted, onParticipantStateChange);
       room.off(RoomEvent.TrackUnmuted, onParticipantStateChange);
-      room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange);
+      room.off(RoomEvent.LocalTrackPublished, onLocalTrackPublished);
-      room.off(RoomEvent.LocalTrackUnpublished, onParticipantStateChange);
+      room.off(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished);
       room.off(RoomEvent.MediaDevicesError, onMediaDeviceFailure);
       room.off(RoomEvent.EncryptionError, onEncryptionError);
       room.off(RoomEvent.ConnectionStateChanged, onParticipantStateChange);
@@ -1287,7 +1426,7 @@ export function useInitializeCall() {
       room.disconnect();
       e2eeWorker.terminate();
     };
-  }, [noiseFilter]);
+  }, [noiseFilter, load]);
 
   // fetch call data for preview page
   useEffect(() => {

packages/call/src/views/main/focused.tsx

@@ -16,6 +16,9 @@ export default function View() {
   const callIsFullscreen = useCall((state) => state.callIsFullscreen);
 
   const focusedParticipantId = useCall((state) => state.focusedParticipantId);
+  const focusedParticipantType = useCall(
+    (state) => state.focusedParticipantType,
+  );
   const activeScreenShareParticipantIds = useCall(
     (state) => state.activeScreenShareParticipantIds,
   );
@@ -157,6 +160,10 @@ export default function View() {
   const focusedParticipant = getParticipantById(focusedParticipantId);
   const focusedParticipantHasActiveScreenShare =
     activeScreenShareParticipantIdSet.has(focusedParticipantId);
+  const focusedTileType: "user" | "stream" =
+    focusedParticipantType === "stream" && focusedParticipantHasActiveScreenShare
+      ? "stream"
+      : "user";
   const isImmersiveFocusedView = callIsFullscreen && usersInFocusedViewHidden;
 
   return (
@@ -179,7 +186,7 @@ export default function View() {
             <Base
               fill={isImmersiveFocusedView}
               flush={isImmersiveFocusedView || isFocusedTileFlush}
-              type={focusedParticipantHasActiveScreenShare ? "stream" : "user"}
+              type={focusedTileType}
               participant={focusedParticipant}
             />
           </div>

packages/call/todo.md

@@ -1,4 +1,3 @@
-- Speaking indicator
 - Overlay for stream modals
 - User modals
   - Bg based on avatar
@@ -11,3 +10,5 @@
   - Disconnect
 - Desktop-App screenshares
 - Context menus
+- Popout Window
+- If micGated=true & isSpeaking=false for 5 seconds show banner with mic detection

packages/shared/src/data.ts

@@ -245,6 +245,8 @@ export interface Storage extends SettingsStorageDefaults {
   cached_contacts: Contacts;
   cached_communities: Communities;
   ttp_url: string;
+  call_mute_range_start: number;
+  call_mute_range_end: number;
 }
 
 export const storageDefaults: Storage = {
@@ -278,4 +280,6 @@ export const storageDefaults: Storage = {
   cached_contacts: [],
   cached_communities: [],
   ttp_url: "https://tensamin.net:959",
+  call_mute_range_start: -55,
+  call_mute_range_end: -45,
 };