diff --git a/apps/web/package.json b/apps/web/package.json index 07bfd07..0748a1f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -26,6 +26,7 @@ "@tensamin/tauri": "workspace:*", "@tensamin/ttp": "workspace:*", "@tensamin/tauth": "workspace:*", + "@tensamin/markdown": "workspace:*", "@tensamin/ui": "*", "@tensamin/user": "workspace:*", "@tensamin/notifications": "workspace:*", diff --git a/apps/web/src/components/modals/basic.tsx b/apps/web/src/components/modals/basic.tsx index 6105d7a..be3e967 100644 --- a/apps/web/src/components/modals/basic.tsx +++ b/apps/web/src/components/modals/basic.tsx @@ -1,20 +1,58 @@ import type { User } from "@tensamin/user/context"; -import { Avatar, AvatarImage, AvatarFallback } from "@tensamin/ui"; -import { reduceDisplay } from "./utils"; +import { + Avatar, + AvatarImage, + AvatarFallback, + Tooltip, + TooltipTrigger, + TooltipContent, +} from "@tensamin/ui"; import { Card, CardHeader } from "@tensamin/ui"; import { Skeleton } from "@tensamin/ui"; +import { getStatusColor } from "@tensamin/shared/data"; -export function Basic(props: { user: User }) { +export function Basic({ + user, + extra, +}: { + user: User; + extra?: React.ReactNode; +}) { return ( - - - {reduceDisplay(props.user.display)} - -
-

{props.user.display}

+
+ + + + {user.display.slice(0, 2).toUpperCase()} + + + + +
+
+ } + /> + + {user.online_status + .split("_") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" ")} + +
+
+

{user.display}

+
+
{extra}
); diff --git a/apps/web/src/components/modals/utils.ts b/apps/web/src/components/modals/utils.ts deleted file mode 100644 index 248dcd5..0000000 --- a/apps/web/src/components/modals/utils.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Executes reduceDisplay. - * @param display Parameter display. - * @returns unknown. - */ -export function reduceDisplay(display: string) { - const words = display.split(" "); - if (words.length === 1) { - return display.slice(0, 2).toUpperCase(); - } else { - return words[0].charAt(0).toUpperCase() + words[1].charAt(0).toUpperCase(); - } -} diff --git a/apps/web/src/components/sidebar.tsx b/apps/web/src/components/sidebar.tsx index 6b4b6cd..f8ce056 100644 --- a/apps/web/src/components/sidebar.tsx +++ b/apps/web/src/components/sidebar.tsx @@ -7,6 +7,26 @@ import { SidebarContent, SidebarFooter, useSidebar, + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuGroup, + DropdownMenuLabel, + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, + Input, + DialogClose, + Button, + Label, + Select, + SelectTrigger, + SelectValue, + SelectContent, + SelectItem, } from "@tensamin/ui"; import { isTauri } from "@tauri-apps/api/core"; import { useIsMobile } from "@tensamin/ui"; @@ -14,18 +34,168 @@ import { MobileNavbar } from "./navbar"; import SidebarBox from "@tensamin/call/sidebarBox"; import { useShowMobileNavbar } from "@/routes/app/layout"; -/** - * Renders the conversation sidebar with account summary and conversation list. - * On mobile the sidebar is always kept in the DOM and hidden via CSS - * (opacity + translateX) instead of being unmounted. This keeps the DOM and - * React state alive while the drawer is closed, allowing the sidebar to open - * instantly on subsequent toggles. - * @returns Sidebar JSX. - */ +import { Ellipsis, Check } from "lucide-react"; +import { useState } from "react"; +import type { User } from "@tensamin/user/context"; +import type z from "zod"; +import { ttp } from "@tensamin/shared/data"; +import { useTTP } from "@tensamin/ttp"; + +type OnlineStatus = z.infer< + typeof ttp.get_user_data.response.shape.online_status +>; + +const onlineStatusLabels: Record = { + user_online: "Online", + user_offline: "Offline", + user_dnd: "Do not disturb", + user_idle: "Idle", + user_wc: "Away", + user_borked: "Borked", + iota_offline: "Iota offline", + iota_online: "Iota online", + iota_borked: "Iota borked", +}; + +function StatusDialog({ + user, + open, + onOpenChange, + send, + draftStatus, + setDraftStatus, + draftOnlineStatus, + setDraftOnlineStatus, + errorMessage, + setErrorMessage, + saveSucceeded, + setSaveSucceeded, +}: { + user: User; + open: boolean; + onOpenChange: (open: boolean) => void; + send: ReturnType["send"]; + draftStatus: string; + setDraftStatus: (value: string) => void; + draftOnlineStatus: OnlineStatus; + setDraftOnlineStatus: (value: OnlineStatus) => void; + errorMessage: string; + setErrorMessage: (value: string) => void; + saveSucceeded: boolean; + setSaveSucceeded: (value: boolean) => void; +}) { + return ( + { + if (!nextOpen) { + setDraftStatus(user.status ?? ""); + setDraftOnlineStatus(user.online_status); + setErrorMessage(""); + setSaveSucceeded(false); + } + + onOpenChange(nextOpen); + }} + > + + + Update Status + +
+ + { + setSaveSucceeded(false); + setErrorMessage(""); + setDraftStatus(e.target.value); + }} + /> + + +
+ {errorMessage && ( +

{errorMessage}

+ )} + + Cancel} /> + + +
+
+ ); +} + export default function Sidebar() { const isMobile = useIsMobile(); const showMobileNavbar = useShowMobileNavbar(); const { openMobile, setOpenMobile } = useSidebar(); + const [draftStatus, setDraftStatus] = useState(""); + const [draftOnlineStatus, setDraftOnlineStatus] = + useState("user_online"); + const [dialogOpen, setDialogOpen] = useState(false); + const [statusErrorMessage, setStatusErrorMessage] = useState(""); + const [statusSaveSucceeded, setStatusSaveSucceeded] = useState(false); + + const { send } = useTTP(); const content = ( <> @@ -39,7 +209,67 @@ export default function Sidebar() { } userId={"own"} - component={(user) => } + component={(user) => ( + <> + + + + + } + /> + + + Profile + { + setDraftStatus(user.status ?? ""); + setDraftOnlineStatus(user.online_status); + setStatusErrorMessage(""); + setStatusSaveSucceeded(false); + setDialogOpen(true); + }} + > + Set Status + + + + + } + /> + { + if (!nextOpen) { + setDraftStatus(user.status ?? ""); + setDraftOnlineStatus(user.online_status); + setStatusErrorMessage(""); + setStatusSaveSucceeded(false); + } + + setDialogOpen(nextOpen); + }} + send={send} + draftStatus={draftStatus} + setDraftStatus={setDraftStatus} + draftOnlineStatus={draftOnlineStatus} + setDraftOnlineStatus={setDraftOnlineStatus} + errorMessage={statusErrorMessage} + setErrorMessage={setStatusErrorMessage} + saveSucceeded={statusSaveSucceeded} + setSaveSucceeded={setStatusSaveSucceeded} + /> + + )} />
diff --git a/apps/web/src/routes/settings/profile.tsx b/apps/web/src/routes/settings/profile.tsx index 0b98f2d..e46cc62 100644 --- a/apps/web/src/routes/settings/profile.tsx +++ b/apps/web/src/routes/settings/profile.tsx @@ -1,3 +1,211 @@ +import { useStorage } from "@tensamin/storage/context"; +import { + Avatar, + AvatarFallback, + AvatarImage, + Button, + Input, +} from "@tensamin/ui"; +import { useUser, type User } from "@tensamin/user/context"; +import { useEffect, useRef, useState } from "react"; +import MDInput from "@tensamin/markdown/input"; +import { ttp } from "@tensamin/shared/data"; +import { useTTP } from "@tensamin/ttp"; +import { Check } from "lucide-react"; + +async function prepImage( + file: File, + size = 300, + quality = 0.8, +): Promise { + const bitmap = await createImageBitmap(file); + + const canvas = document.createElement("canvas"); + canvas.width = size; + canvas.height = size; + + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Could not get canvas context"); + + const scale = Math.max(size / bitmap.width, size / bitmap.height); + const width = bitmap.width * scale; + const height = bitmap.height * scale; + const x = (size - width) / 2; + const y = (size - height) / 2; + + ctx.drawImage(bitmap, x, y, width, height); + + return canvas.toDataURL("image/webp", quality); +} + export default function Page() { - return
; + const { get } = useUser(); + const { load } = useStorage(); + const { send } = useTTP(); + const [currentUser, setCurrentUser] = useState(null); + const [draftUser, setDraftUser] = useState>({}); + const [errorMessage, setErrorMessage] = useState(""); + const [saveSucceeded, setSaveSucceeded] = useState(false); + const avatarUploadRef = useRef(null); + const draftInitializedRef = useRef(false); + const effectiveAvatar = + draftUser.avatar === "none" ? undefined : draftUser.avatar; + + const updateDraftUser = ( + updater: (previous: Partial) => Partial, + ) => { + setSaveSucceeded(false); + setErrorMessage(""); + setDraftUser(updater); + }; + + useEffect(() => { + const fetchUser = async () => { + const user = await get(await load("user_id")); + setCurrentUser(user); + }; + + fetchUser(); + }, [load, get]); + + useEffect(() => { + if (!currentUser || draftInitializedRef.current) return; + + setDraftUser(currentUser); + draftInitializedRef.current = true; + }, [currentUser]); + + const handleAvatarUpload = async (file: File) => { + const final = await prepImage(file); + updateDraftUser((prev) => ({ ...prev, avatar: final })); + if (avatarUploadRef.current) { + avatarUploadRef.current.value = ""; + } + }; + + return currentUser ? ( + <> + + e.target.files?.[0] && handleAvatarUpload(e.target.files[0]) + } + type="file" + /> +
+
+ + + + {draftUser.display?.slice(0, 2).toUpperCase() || + currentUser.display.slice(0, 2).toUpperCase()} + + +
+

Avatar

+
+ + +
+

+ GIFs are supported in decentralised mode or with Tensamin Premium +
+ Maximum file size is 16mb. +

+
+
+ + updateDraftUser((prev) => ({ + ...prev, + display: event.target.value, + })) + } + placeholder="Display Name" + value={draftUser.display || ""} + /> + + updateDraftUser((prev) => ({ + ...prev, + username: event.target.value, + })) + } + placeholder="Username" + value={draftUser.username || ""} + /> + + updateDraftUser((prev) => ({ ...prev, about: value })) + } + value={draftUser.about || ""} + /> + + {errorMessage && ( +

{errorMessage}

+ )} +
+ + ) : ( +

Loading...

+ ); } diff --git a/bun.lock b/bun.lock index 325be20..ff51499 100644 --- a/bun.lock +++ b/bun.lock @@ -55,6 +55,7 @@ "@tensamin/call": "workspace:*", "@tensamin/chat": "workspace:*", "@tensamin/crypto": "workspace:*", + "@tensamin/markdown": "workspace:*", "@tensamin/notifications": "workspace:*", "@tensamin/shared": "workspace:*", "@tensamin/storage": "workspace:*", @@ -258,7 +259,7 @@ }, }, "overrides": { - "@tensamin/ttp-core": "https://git.methanium.net/tensamin/ttp/archive/0.0.17.tar.gz", + "@tensamin/ttp-core": "https://git.methanium.net/tensamin/ttp/archive/0.0.19.tar.gz", "@tensamin/ui": "https://git.methanium.net/tensamin/ui/archive/0.0.34.tar.gz", }, "packages": { @@ -698,7 +699,7 @@ "@tensamin/ttp": ["@tensamin/ttp@workspace:packages/ttp"], - "@tensamin/ttp-core": ["@tensamin/ttp-core@https://git.methanium.net/tensamin/ttp/archive/0.0.17.tar.gz", { "dependencies": { "@eslint/js": "^10.0.1", "@typescript-eslint/parser": "^8.59.1", "@webtransport-bun/webtransport": "^0.3.0", "globals": "^17.5.0", "typescript": "^6.0.3", "typescript-eslint": "^8.59.1", "zod": "^4.4.1" } }, "sha512-smyx+04hSnWM1oyWJfJrRWmxu7OInwyHeIlaD1h3tUrkrSxiKWHl1qCmxVO1bC+cDwyUXhJ5HUnNCbx1aWx7Dg=="], + "@tensamin/ttp-core": ["@tensamin/ttp-core@https://git.methanium.net/tensamin/ttp/archive/0.0.19.tar.gz", { "dependencies": { "@eslint/js": "^10.0.1", "@typescript-eslint/parser": "^8.59.1", "@webtransport-bun/webtransport": "^0.3.0", "globals": "^17.5.0", "typescript": "^6.0.3", "typescript-eslint": "^8.59.1", "zod": "^4.4.1" } }, "sha512-La9VqXqJFtzzsRQotXVp+3Vr6u8kj4mQ4wTlSIMRDxKFBnbCvtZyB3V/f8HiIlf7FlZ0Xg0suUUpnamvtcvs9w=="], "@tensamin/ui": ["@tensamin/ui@https://git.methanium.net/tensamin/ui/archive/0.0.34.tar.gz", { "dependencies": { "@base-ui/react": "^1.3.0", "@fontsource-variable/inter": "^5.2.6", "@tauri-apps/api": "^2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", "embla-carousel-react": "^8.6.0", "input-otp": "^1.4.2", "lucide-react": "^1.8.0", "next-themes": "^0.4.6", "react-day-picker": "^9.14.0", "react-resizable-panels": "^4.10.0", "recharts": "3.8.0", "shadcn": "^3.5.0", "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.3.0", "vaul": "^1.1.2" }, "peerDependencies": { "react": "^19.2.0", "react-dom": "^19.2.0" } }, "sha512-hp7rV0a0gfD/rNw9et+PqM1PPkgFC6/Z7eYfza6NIp/m+a8/r5Um+S/8tzBDCgZmQC9Y1sJsjesH7mXZz6Jmuw=="], diff --git a/package.json b/package.json index 7f969e8..d87ae5a 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ }, "overrides": { "@tensamin/ui": "https://git.methanium.net/tensamin/ui/archive/0.0.34.tar.gz", - "@tensamin/ttp-core": "https://git.methanium.net/tensamin/ttp/archive/0.0.17.tar.gz" + "@tensamin/ttp-core": "https://git.methanium.net/tensamin/ttp/archive/0.0.19.tar.gz" }, "dependencies": { "@tensamin/ttp-core": "*", diff --git a/packages/call/src/components/buttons/screenshare.tsx b/packages/call/src/components/buttons/screenshare.tsx index 4471539..21ba118 100644 --- a/packages/call/src/components/buttons/screenshare.tsx +++ b/packages/call/src/components/buttons/screenshare.tsx @@ -18,10 +18,12 @@ export default function ScreenshareButton({ className, iconSize, tooltip, + defaultPortal, }: { className?: string; iconSize?: number; tooltip?: string; + defaultPortal?: boolean; }) { const isScreensharing = useCall((state) => state.screenShareEnabled); const screenRef = useCall((state) => state.screenRef); @@ -30,8 +32,9 @@ export default function ScreenshareButton({ const [menuOpen, setMenuOpen] = useState(false); useEffect(() => { + if (defaultPortal) return; setPortalContainer(screenRef?.current ?? undefined); - }, [screenRef]); + }, [screenRef, defaultPortal]); async function startWebShare() { try { @@ -110,7 +113,9 @@ export default function ScreenshareButton({ />
{/* Detect video / user and place here */} diff --git a/packages/call/src/components/sidebarBox.tsx b/packages/call/src/components/sidebarBox.tsx index 294d051..6a850f0 100644 --- a/packages/call/src/components/sidebarBox.tsx +++ b/packages/call/src/components/sidebarBox.tsx @@ -20,24 +20,18 @@ import LeaveButton from "./buttons/leave"; export default function SidebarBox() { const state = useCall((store) => store.state); - const screenRef = useCall((store) => store.screenRef); const isMobile = useIsMobile(); - const [portalContainer, setPortalContainer] = useState(); - - useEffect(() => { - setPortalContainer(screenRef?.current ?? undefined); - }, [screenRef]); return state === "closed" ? null : (
} /> - + {data.length > 0 ? `${data.at(-1)?.ping} ms` : "Measuring ping..."} diff --git a/packages/call/src/speakingIndicator.ts b/packages/call/src/speakingIndicator.ts new file mode 100644 index 0000000..0c4f5a5 --- /dev/null +++ b/packages/call/src/speakingIndicator.ts @@ -0,0 +1,252 @@ +import { log } from "@tensamin/shared/log"; +import { useCall } from "./store"; + +const SPEAKING_THRESHOLD = 0.01; +const SPEAKING_HANGTIME_MS = 500; +const ANALYSIS_INTERVAL_MS = 30; +const FFT_SIZE = 256; + +type AnalyserEntry = { + source: MediaStreamAudioSourceNode; + analyser: AnalyserNode; + track: MediaStreamTrack; + originalTrack?: MediaStreamTrack; + lastSpeakingTime: number; + isSpeaking: boolean; +}; + +class SpeakingDetector { + private audioContext: AudioContext | null = null; + private entries = new Map(); + private intervalId: ReturnType | null = null; + private deaf = false; + private gateThresholdStart = -50; + private gateThresholdEnd = -40; + private localParticipantId: number | null = null; + private localMicGateClosed = false; + + private ensureAudioContext(): AudioContext { + if (!this.audioContext) { + this.audioContext = new AudioContext(); + } + if (this.audioContext.state === "suspended") { + void this.audioContext.resume(); + } + return this.audioContext; + } + + setLocalParticipantId(id: number) { + this.localParticipantId = id; + } + + setGateThresholds(start: number, end: number) { + this.gateThresholdStart = start; + this.gateThresholdEnd = end; + } + + addTrack( + participantId: number, + track: MediaStreamTrack, + originalTrack?: MediaStreamTrack, + ) { + if (track.kind !== "audio") return; + + this.removeParticipant(participantId); + + const ctx = this.ensureAudioContext(); + const stream = new MediaStream([track]); + const source = ctx.createMediaStreamSource(stream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = FFT_SIZE; + source.connect(analyser); + + this.entries.set(participantId, { + source, + analyser, + track, + originalTrack, + lastSpeakingTime: 0, + isSpeaking: false, + }); + + if (!this.intervalId) { + this.startLoop(); + } + } + + removeParticipant(participantId: number) { + const entry = this.entries.get(participantId); + if (!entry) return; + + try { + entry.source.disconnect(); + } catch { + // ignore + } + this.entries.delete(participantId); + + useCall.setState((state) => { + if (!state.speakingParticipantIds.has(participantId)) return state; + const next = new Set(state.speakingParticipantIds); + next.delete(participantId); + return { speakingParticipantIds: next }; + }); + + if (participantId === this.localParticipantId && this.localMicGateClosed) { + this.muteLocalTrack(false); + } + } + + setDeaf(deaf: boolean) { + this.deaf = deaf; + if (deaf) { + for (const entry of this.entries.values()) { + entry.isSpeaking = false; + entry.lastSpeakingTime = 0; + } + useCall.setState({ speakingParticipantIds: new Set() }); + } + } + + private startLoop() { + if (this.intervalId) return; + this.intervalId = setInterval(() => this.analyse(), ANALYSIS_INTERVAL_MS); + } + + private stopLoop() { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + private muteLocalTrack(muted: boolean) { + const entry = this.localParticipantId + ? this.entries.get(this.localParticipantId) + : undefined; + const target = entry?.originalTrack ?? entry?.track; + + if (target && target.enabled === muted) { + target.enabled = !muted; + } + + this.localMicGateClosed = muted; + useCall.setState({ micGated: muted }); + } + + private applyNoiseGate(rms: number) { + const db = 20 * Math.log10(Math.max(rms, 0.0001)); + + if (!this.localMicGateClosed && db < this.gateThresholdStart) { + log(3, "noise gate", "purple", "closed"); + this.muteLocalTrack(true); + } else if (this.localMicGateClosed && db > this.gateThresholdEnd) { + log(3, "noise gate", "purple", "opened"); + this.muteLocalTrack(false); + } + } + + private analyse() { + if (this.deaf || this.entries.size === 0) return; + + const now = Date.now(); + const changed = new Map(); + + for (const [participantId, entry] of this.entries) { + const { analyser, track } = entry; + + if (track.muted || track.readyState === "ended" || !track.enabled) { + if (entry.isSpeaking) { + entry.isSpeaking = false; + entry.lastSpeakingTime = 0; + changed.set(participantId, false); + } + continue; + } + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + analyser.getByteTimeDomainData(dataArray); + + let sum = 0; + for (let i = 0; i < bufferLength; i++) { + const sample = (dataArray[i] - 128) / 128.0; + sum += sample * sample; + } + const rms = Math.sqrt(sum / bufferLength); + + let nextIsSpeaking = entry.isSpeaking; + if (rms > SPEAKING_THRESHOLD) { + entry.lastSpeakingTime = now; + nextIsSpeaking = true; + } else if (now - entry.lastSpeakingTime > SPEAKING_HANGTIME_MS) { + nextIsSpeaking = false; + } + + if (participantId === this.localParticipantId) { + this.applyNoiseGate(rms); + if (this.localMicGateClosed) { + nextIsSpeaking = false; + } + } + + if (nextIsSpeaking !== entry.isSpeaking) { + entry.isSpeaking = nextIsSpeaking; + changed.set(participantId, nextIsSpeaking); + } + } + + if (changed.size > 0) { + useCall.setState((state) => { + let hasDiff = false; + const next = new Set(state.speakingParticipantIds); + for (const [id, speaking] of changed) { + if (speaking) { + if (!next.has(id)) { + next.add(id); + hasDiff = true; + } + } else { + if (next.has(id)) { + next.delete(id); + hasDiff = true; + } + } + } + return hasDiff ? { speakingParticipantIds: next } : state; + }); + } + } + + dispose() { + this.stopLoop(); + for (const id of Array.from(this.entries.keys())) { + this.removeParticipant(id); + } + this.entries.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + } + } +} + +let detectorInstance: SpeakingDetector | null = null; + +export function getSpeakingDetector(): SpeakingDetector { + if (!detectorInstance) { + detectorInstance = new SpeakingDetector(); + } + return detectorInstance; +} + +export function disposeSpeakingDetector(): void { + if (detectorInstance) { + detectorInstance.dispose(); + detectorInstance = null; + } +} + +export function useIsSpeaking(participantId: number): boolean { + return useCall((state) => state.speakingParticipantIds.has(participantId)); +} diff --git a/packages/call/src/store.tsx b/packages/call/src/store.tsx index e605ece..3d63963 100644 --- a/packages/call/src/store.tsx +++ b/packages/call/src/store.tsx @@ -11,6 +11,7 @@ import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; import { ExternalE2EEKeyProvider, LocalAudioTrack, + type LocalTrackPublication, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -28,6 +29,10 @@ import { createScreenShareController, type ScreenShareSession, } from "./screenshare"; +import { + getSpeakingDetector, + disposeSpeakingDetector, +} from "./speakingIndicator"; // logging setLogExtension( @@ -86,6 +91,7 @@ type CallStore = { screenShareEnabled: boolean; screenShareSession: ScreenShareSession | null; focusedParticipantId: number | null; + focusedParticipantType: "user" | "stream" | null; usersInFocusedViewHidden: boolean; watchedStreamParticipantIds: number[]; pendingWatchedParticipantIds: number[]; @@ -99,6 +105,8 @@ type CallStore = { keyProvider: ExternalE2EEKeyProvider; e2eeWorker: Worker; runtime: Runtime | null; + speakingParticipantIds: Set; + micGated: boolean; }; const keyProvider = new ExternalE2EEKeyProvider(); @@ -152,7 +160,7 @@ function clearRemoteAudio() { } } -function getParticipantId(identity: string | undefined): number | null { +export function getParticipantId(identity: string | undefined): number | null { if (!identity) { return null; } @@ -199,7 +207,10 @@ function matchesRemoteTrackSelector( function syncRemoteParticipantTrackSubscriptions(participantId: number) { for (const publication of getRemoteTrackPublications(participantId)) { - publication.setSubscribed(publication.kind === Track.Kind.Audio); + publication.setSubscribed( + publication.kind === Track.Kind.Audio && + publication.source !== Track.Source.ScreenShareAudio, + ); } } @@ -457,6 +468,8 @@ function syncScreenShareParticipants() { watchedStreamParticipantIds, pendingWatchedParticipantIds, focusedParticipantId, + focusedParticipantType: + focusedParticipantId == null ? null : state.focusedParticipantType, view: state.view === "focused" && focusedParticipantId == null ? "grid" @@ -614,6 +627,7 @@ export function startWatchingStream(participantId: number) { const trackReady = getScreenShareTrackForParticipant(participantId) != null; setParticipantTrackSubscribed(participantId, Track.Source.ScreenShare); + setParticipantTrackSubscribed(participantId, Track.Source.ScreenShareAudio); useCall.setState((state) => ({ watchedStreamParticipantIds: state.watchedStreamParticipantIds.includes( @@ -627,6 +641,7 @@ export function startWatchingStream(participantId: number) { ? state.pendingWatchedParticipantIds : [...state.pendingWatchedParticipantIds, participantId], focusedParticipantId: participantId, + focusedParticipantType: "stream", })); } @@ -646,9 +661,13 @@ export function setParticipantTrackSubscribed( } // Focus a participant in the main call view even when they are not sharing a screen. -export function focusParticipant(participantId: number) { +export function focusParticipant( + participantId: number, + type: "user" | "stream" = "user", +) { useCall.setState({ focusedParticipantId: participantId, + focusedParticipantType: type, view: "focused", }); } @@ -656,6 +675,11 @@ export function focusParticipant(participantId: number) { // Stop tracking a participant's shared screen and clean up related UI state. export function stopWatchingStream(participantId: number) { setParticipantTrackSubscribed(participantId, Track.Source.ScreenShare, false); + setParticipantTrackSubscribed( + participantId, + Track.Source.ScreenShareAudio, + false, + ); useCall.setState((state) => ({ watchedStreamParticipantIds: state.watchedStreamParticipantIds.filter( @@ -668,6 +692,10 @@ export function stopWatchingStream(participantId: number) { state.focusedParticipantId === participantId ? null : state.focusedParticipantId, + focusedParticipantType: + state.focusedParticipantId === participantId + ? null + : state.focusedParticipantType, view: state.view === "focused" && state.focusedParticipantId === participantId ? "grid" @@ -757,6 +785,7 @@ export async function connect(callId: string) { // Tear down the active call session and return the store to a closed state. export async function disconnect() { + disposeSpeakingDetector(); await clearScreenSharePreview(); try { @@ -782,10 +811,12 @@ export async function disconnect() { view: "preview", screenShareSession: null, focusedParticipantId: null, + focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], activeScreenShareParticipantIds: [], + micGated: false, }); room.remoteParticipants.forEach((participant) => { @@ -879,6 +910,7 @@ export async function toggleDeaf() { deafened: nextDeaf ? "true" : "false", }); + getSpeakingDetector().setDeaf(nextDeaf); useCall.setState({ deaf: nextDeaf }); } @@ -940,6 +972,7 @@ export function resetCallState() { deaf: false, screenShareSession: null, focusedParticipantId: null, + focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], @@ -967,6 +1000,16 @@ async function ensureNoiseFilter( await microphoneTrack.setProcessor(noiseFilter).catch((err) => { log(1, "call", "red", "Failed to enable noise filter", err); }); + + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + const processedTrack = microphoneTrack.mediaStreamTrack; + getSpeakingDetector().addTrack( + participantId, + processedTrack.clone(), + processedTrack, + ); + } } export const useCall = create(() => ({ @@ -982,6 +1025,7 @@ export const useCall = create(() => ({ screenShareEnabled: room.localParticipant.isScreenShareEnabled, screenShareSession: null, focusedParticipantId: null, + focusedParticipantType: null, usersInFocusedViewHidden: false, watchedStreamParticipantIds: [], pendingWatchedParticipantIds: [], @@ -996,6 +1040,8 @@ export const useCall = create(() => ({ keyProvider, e2eeWorker, runtime: null, + speakingParticipantIds: new Set(), + micGated: false, })); // Register app-level call listeners and wire React dependencies into the store. @@ -1016,7 +1062,7 @@ export function useInitializeCall() { new DeepFilterNoiseFilterProcessor({ enabled: true, enableNoiseReduction: true, - noiseReductionLevel: 80, + noiseReductionLevel: 60, sampleRate: 48000, assetConfig: { cdnUrl: "/assets", @@ -1137,10 +1183,46 @@ export function useInitializeCall() { listenersRegistered.current = true; - const onConnected = () => { + const onConnected = async () => { useCall.setState({ state: "open" }); syncParticipantState(); + const detector = getSpeakingDetector(); + const localParticipantId = getParticipantId( + room.localParticipant.identity, + ); + if (localParticipantId != null) { + detector.setLocalParticipantId(localParticipantId); + } + + const [start, end] = await Promise.all([ + load("call_mute_range_start"), + load("call_mute_range_end"), + ]); + detector.setGateThresholds(start, end); + + // Scan existing audio tracks for speaking detection + for (const participant of getAllParticipants()) { + const participantId = getParticipantId(participant.identity); + if (participantId == null) continue; + + for (const publication of participant.trackPublications.values()) { + if ( + publication.kind === Track.Kind.Audio && + publication.source === Track.Source.Microphone && + publication.track + ) { + const mediaTrack = publication.track.mediaStreamTrack; + if (participant === room.localParticipant) { + const clonedTrack = mediaTrack.clone(); + detector.addTrack(participantId, clonedTrack, mediaTrack); + } else { + detector.addTrack(participantId, mediaTrack); + } + } + } + } + const invitedUserId = useCall.getState().invitedUserId; if (invitedUserId != null) { @@ -1177,6 +1259,7 @@ export function useInitializeCall() { if (participantId != null) { stopWatchingStream(participantId); + getSpeakingDetector().removeParticipant(participantId); } syncParticipantState(); @@ -1197,6 +1280,40 @@ export function useInitializeCall() { void ensureNoiseFilter(noiseFilter); }; + const onLocalTrackPublished = (publication: LocalTrackPublication) => { + if ( + publication.kind === Track.Kind.Audio && + publication.source === Track.Source.Microphone && + publication.track + ) { + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + const originalTrack = publication.track.mediaStreamTrack; + const clonedTrack = originalTrack.clone(); + getSpeakingDetector().addTrack( + participantId, + clonedTrack, + originalTrack, + ); + } + } + syncParticipantState(); + void ensureNoiseFilter(noiseFilter); + }; + + const onLocalTrackUnpublished = (publication: LocalTrackPublication) => { + if ( + publication.kind === Track.Kind.Audio && + publication.source === Track.Source.Microphone + ) { + const participantId = getParticipantId(room.localParticipant.identity); + if (participantId != null) { + getSpeakingDetector().removeParticipant(participantId); + } + } + syncParticipantState(); + }; + const onTrackPublished = ( publication: RemoteTrackPublication, participant: RemoteParticipant, @@ -1204,7 +1321,10 @@ export function useInitializeCall() { const participantId = getParticipantId(participant.identity); if (participantId != null) { - if (publication.kind === Track.Kind.Audio) { + if ( + publication.kind === Track.Kind.Audio && + publication.source !== Track.Source.ScreenShareAudio + ) { publication.setSubscribed(true); } else { publication.setSubscribed(false); @@ -1214,9 +1334,21 @@ export function useInitializeCall() { onParticipantStateChange(); }; - const onTrackSubscribed = (track: RemoteTrack) => { + const onTrackSubscribed = ( + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + ) => { if (track.kind === "audio" && track.sid) { attachRemoteAudio(track.sid, track.attach()); + + const participantId = getParticipantId(participant.identity); + if ( + participantId != null && + publication.source === Track.Source.Microphone + ) { + getSpeakingDetector().addTrack(participantId, track.mediaStreamTrack); + } } syncParticipantState(); @@ -1224,15 +1356,22 @@ export function useInitializeCall() { const onTrackUnsubscribed = ( track: RemoteTrack, - _publication: unknown, + publication: RemoteTrackPublication, participant: Participant, ) => { + const participantId = getParticipantId(participant.identity); + if (track.kind === "audio" && track.sid) { track.detach(); detachRemoteAudio(track.sid); - } - const participantId = getParticipantId(participant.identity); + if ( + participantId != null && + publication.source === Track.Source.Microphone + ) { + getSpeakingDetector().removeParticipant(participantId); + } + } if ( participantId != null && @@ -1256,8 +1395,8 @@ export function useInitializeCall() { room.on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.on(RoomEvent.TrackMuted, onParticipantStateChange); room.on(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.on(RoomEvent.LocalTrackPublished, onParticipantStateChange); - room.on(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); + room.on(RoomEvent.LocalTrackPublished, onLocalTrackPublished); + room.on(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); room.on(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.on(RoomEvent.EncryptionError, onEncryptionError); room.on(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1277,8 +1416,8 @@ export function useInitializeCall() { room.off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected); room.off(RoomEvent.TrackMuted, onParticipantStateChange); room.off(RoomEvent.TrackUnmuted, onParticipantStateChange); - room.off(RoomEvent.LocalTrackPublished, onParticipantStateChange); - room.off(RoomEvent.LocalTrackUnpublished, onParticipantStateChange); + room.off(RoomEvent.LocalTrackPublished, onLocalTrackPublished); + room.off(RoomEvent.LocalTrackUnpublished, onLocalTrackUnpublished); room.off(RoomEvent.MediaDevicesError, onMediaDeviceFailure); room.off(RoomEvent.EncryptionError, onEncryptionError); room.off(RoomEvent.ConnectionStateChanged, onParticipantStateChange); @@ -1287,7 +1426,7 @@ export function useInitializeCall() { room.disconnect(); e2eeWorker.terminate(); }; - }, [noiseFilter]); + }, [noiseFilter, load]); // fetch call data for preview page useEffect(() => { diff --git a/packages/call/src/views/main/focused.tsx b/packages/call/src/views/main/focused.tsx index d52362b..78d1740 100644 --- a/packages/call/src/views/main/focused.tsx +++ b/packages/call/src/views/main/focused.tsx @@ -16,6 +16,9 @@ export default function View() { const callIsFullscreen = useCall((state) => state.callIsFullscreen); const focusedParticipantId = useCall((state) => state.focusedParticipantId); + const focusedParticipantType = useCall( + (state) => state.focusedParticipantType, + ); const activeScreenShareParticipantIds = useCall( (state) => state.activeScreenShareParticipantIds, ); @@ -157,6 +160,11 @@ export default function View() { const focusedParticipant = getParticipantById(focusedParticipantId); const focusedParticipantHasActiveScreenShare = activeScreenShareParticipantIdSet.has(focusedParticipantId); + const focusedTileType: "user" | "stream" = + focusedParticipantType === "stream" && + focusedParticipantHasActiveScreenShare + ? "stream" + : "user"; const isImmersiveFocusedView = callIsFullscreen && usersInFocusedViewHidden; return ( @@ -179,7 +187,7 @@ export default function View() { diff --git a/packages/call/todo.md b/packages/call/todo.md index 952858f..61d70ed 100644 --- a/packages/call/todo.md +++ b/packages/call/todo.md @@ -1,4 +1,3 @@ -- Speaking indicator - Overlay for stream modals - User modals - Bg based on avatar @@ -11,3 +10,5 @@ - Disconnect - Desktop-App screenshares - Context menus +- Popout Window +- If micGated=true & isSpeaking=false for 5 seconds show banner with mic detection diff --git a/packages/chat/src/components/input.tsx b/packages/chat/src/components/input.tsx index 487c813..b2aef9e 100644 --- a/packages/chat/src/components/input.tsx +++ b/packages/chat/src/components/input.tsx @@ -97,6 +97,8 @@ export default function InputComponent({ > void; onSubmit?: () => void; invertEnterBehavior?: boolean; + styled?: boolean; + fontSize?: CSSProperties["fontSize"]; + paddingX?: CSSProperties["padding"]; + paddingY?: CSSProperties["padding"]; + className?: string; }; +type InputStyle = CSSProperties & { + "--tm-md-content-padding"?: string; +}; + +function toCssLength(value: CSSProperties["padding"]): string | undefined { + if (value === undefined) { + return undefined; + } + + return typeof value === "number" ? `${value}px` : value; +} + +function toCssPadding( + vertical: CSSProperties["padding"], + horizontal: CSSProperties["padding"], + styled: boolean, +): string { + const defaultVertical = styled ? "0.25rem" : "0"; + const defaultHorizontal = styled ? "0.625rem" : "0"; + + return `${toCssLength(vertical) ?? defaultVertical} ${toCssLength(horizontal) ?? defaultHorizontal}`; +} + type TokenRange = { from: number; to: number; @@ -80,6 +109,10 @@ const markdownDecorations = ViewPlugin.fromClass( export default function Input(props: InputProps) { ensureMarkdownStyles(); + const shellClassName = props.styled + ? "min-h-8 w-full min-w-0 rounded-lg border border-input bg-transparent text-base transition-colors outline-none placeholder:text-muted-foreground disabled:pointer-events-none disabled:cursor-not-allowed disabled:bg-input/50 disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 md:text-sm dark:bg-input/30 dark:disabled:bg-input/80 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40" + : ""; + const elementRef = useRef(null); const viewRef = useRef(undefined); const ignoreSyncRef = useRef(false); @@ -143,7 +176,22 @@ export default function Input(props: InputProps) { }); }, [props.value]); - return
; + return ( +
+ ); } /** @@ -204,7 +252,7 @@ function createEditorExtensions( }), EditorView.theme({ "&": { - fontSize: "1rem", + fontSize: "inherit", }, "&.cm-editor": { width: "100%", diff --git a/packages/markdown/src/markdown.tsx b/packages/markdown/src/markdown.tsx index c3a17a6..59d0a6a 100644 --- a/packages/markdown/src/markdown.tsx +++ b/packages/markdown/src/markdown.tsx @@ -665,12 +665,12 @@ export const markdownStyles = ` .tm-md-table th { background: hsl(var(--muted)); font-weight: 600; } .tm-md-hr { margin: 0.55rem 0; } -.cm-editor.tm-md-editor { border-radius: 0.65rem; background: hsl(var(--card)); caret-color: var(--foreground); } +.cm-editor.tm-md-editor { border-radius: inherit; background: transparent; caret-color: var(--foreground); } .cm-editor.tm-md-editor.cm-focused { outline: none; box-shadow: none; } .cm-editor.tm-md-editor .cm-scroller { font-family: inherit; line-height: 1.55; max-height: 30vh; overflow-y: auto; overflow-x: hidden; } .cm-editor.tm-md-editor .cm-content { caret-color: var(--foreground); } -.cm-editor.tm-md-editor .cm-content { padding: 0.7rem 0.85rem; min-height: 2.75rem; } -.cm-editor.tm-md-editor .cm-line { padding: 0 1px; color: hsl(var(--foreground)); } +.cm-editor.tm-md-editor .cm-content { padding: var(--tm-md-content-padding, 0.25rem 0.625rem); min-height: 2rem; } +.cm-editor.tm-md-editor .cm-line { padding: 0; color: hsl(var(--foreground)); } .cm-editor.tm-md-editor .tm-md-hidden-token { color: transparent; opacity: 0; font-size: inherit; } .cm-editor.tm-md-editor .tm-md-code-line { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; background: hsl(var(--muted)); border-radius: 0.3rem; } `; diff --git a/packages/shared/src/data.ts b/packages/shared/src/data.ts index e03ca3d..54af40f 100644 --- a/packages/shared/src/data.ts +++ b/packages/shared/src/data.ts @@ -48,6 +48,31 @@ export type Communities = z.infer< export type Calls = z.infer; // TTP +const user = z.object({ + about: z.string().max(255).optional(), + avatar: z.string().optional(), + display: z.string().min(1).max(15), + iota_id: z.number(), + omikron_connections: z.array(z.number()), + omikron_id: z.number().optional(), + online_status: z.enum([ + "user_offline", + "user_online", + "user_dnd", + "user_idle", + "user_wc", + "user_borked", + "iota_offline", + "iota_online", + "iota_borked", + ]), + public_key: z.base64(), + status: z.string().max(15).optional(), + sub_end: z.number(), + sub_level: z.number(), + user_id: z.number(), + username: z.string().min(1).max(15), +}); export const ttp = { identification: { request: z.object({ @@ -92,31 +117,11 @@ export const ttp = { request: z.object({ user_id: z.number(), }), - response: z.object({ - about: z.string().max(255).optional(), - avatar: z.string().optional(), - display: z.string().max(15), - iota_id: z.number(), - omikron_connections: z.array(z.number()), - omikron_id: z.number().optional(), - online_status: z.enum([ - "user_offline", - "user_online", - "user_dnd", - "user_idle", - "user_wc", - "user_borked", - "iota_offline", - "iota_online", - "iota_borked", - ]), - public_key: z.base64(), - status: z.string().max(15).optional(), - sub_end: z.number(), - sub_level: z.number(), - user_id: z.number(), - username: z.string().max(15), - }), + response: user, + }, + change_user_data: { + request: user.partial(), + response: z.object({}), }, ping: { request: z.object({ @@ -245,6 +250,8 @@ export interface Storage extends SettingsStorageDefaults { cached_contacts: Contacts; cached_communities: Communities; ttp_url: string; + call_mute_range_start: number; + call_mute_range_end: number; } export const storageDefaults: Storage = { @@ -278,4 +285,31 @@ export const storageDefaults: Storage = { cached_contacts: [], cached_communities: [], ttp_url: "https://tensamin.net:959", + call_mute_range_start: -55, + call_mute_range_end: -45, }; + +// User Status +export function getStatusColor( + status: z.infer, +) { + switch (status) { + case "user_online": + return "#22c55e"; + case "iota_online": + return "#22c55e"; + case "user_dnd": + return "#ef4444"; + case "user_idle": + return "#f59e0b"; + case "user_wc": + return "#3b82f6"; + case "user_borked": + case "iota_borked": + return "#6b7280"; + case "user_offline": + case "iota_offline": + default: + return "#9ca3af"; + } +} diff --git a/packages/user/src/context.tsx b/packages/user/src/context.tsx index 1ab964b..962152a 100644 --- a/packages/user/src/context.tsx +++ b/packages/user/src/context.tsx @@ -51,7 +51,7 @@ export default function UserProvider(props: { children: React.ReactNode }) { const user = { ...userData.data, avatar: userData.data.avatar - ? `data:image/png;base64,${userData.data.avatar}` + ? `data:image/webp;base64,${atob(userData.data.avatar)}` : undefined, };