From 8dc8b54b26544554f98d7af32200d6bf9a01d300 Mon Sep 17 00:00:00 2001 From: Alois Date: Fri, 3 Jul 2026 14:45:55 +0200 Subject: [PATCH 01/97] (feat): add codec helpers to ts-sdk --- README.md | 9 +++++++++ docs/WASM-CLIENT.md | 10 ++++++++++ src/sdk/index.ts | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/README.md b/README.md index 265f7b3..048b882 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,15 @@ console.log("Connected MTP client", clientId, client.state); Use `MTPClient.crypto` for SDK-level crypto helpers such as `generateKeyring()`, `generateEd25519()`, `keyringFromEd25519()`, `verifyEd25519()`, `sha256()`, `sha256Double()`, `hkdfExpand()`, and `deriveEncryptionKey()`. +Use `codec` to encode and decode MTP frames from the main SDK export: + +```typescript +import { codec } from "mtp"; + +const frame = codec.encode("SomeType", { value: "hello" }); +const parsed = codec.decode(frame); +``` + ## Getting Started Add the `mtp` crate with your desired features: diff --git a/docs/WASM-CLIENT.md b/docs/WASM-CLIENT.md index b70ab3a..3cd4699 100644 --- a/docs/WASM-CLIENT.md +++ b/docs/WASM-CLIENT.md @@ -258,6 +258,16 @@ Raw message helpers that remain available include: - `format_frame(frame)` - `parse_auth_response(frame)` +The SDK export also exposes the same frame codec through `codec`: + +```typescript +import { codec } from "mtp"; + +const frame = codec.encode("SomeType", { value: "hello" }); +const parsed = codec.decode(frame); +const display = codec.format(frame); +``` + Raw crypto and key helpers include: - `ed25519_generate()` diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 9038e06..eabcc8a 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -75,6 +75,36 @@ export interface MTPRaw { export type MTPBytesInput = Uint8Array | number[]; +export interface MTPCodecOptions { + id?: number; + sender?: bigint | number; + receiver?: bigint | number; +} + +export interface MTPCodec { + encode(type: MTPCommunicationType, data: Record, options?: MTPCodecOptions): Uint8Array; + decode(frame: MTPBytesInput): ParsedFrame; + format(frame: MTPBytesInput): string; +} + +export function encode(type: MTPCommunicationType, data: Record, options?: MTPCodecOptions): Uint8Array { + return bindings.build_frame(type, data, options ?? {}); +} + +export function decode(frame: MTPBytesInput): ParsedFrame { + return bindings.parse_frame(bytesFrom(frame, "frame")); +} + +export function format(frame: MTPBytesInput): string { + return bindings.format_frame(bytesFrom(frame, "frame")); +} + +export const codec: MTPCodec = { + encode, + decode, + format, +}; + export interface MTPCredentials { clientId: bigint | string | number | null; keyring: MTPBytesInput; @@ -335,12 +365,14 @@ async function withTimeout(promise, timeoutMs, message) { export class MTPClient { static readonly crypto = crypto; + static readonly codec = codec; #credentials: InternalCredentials | null; #options: NormalizedMTPClientOptions; readonly raw: MTPRaw; readonly crypto = MTPClient.crypto; + readonly codec = MTPClient.codec; private constructor(options: NormalizedMTPClientOptions, client: RawBindings.WasmClient) { this.#options = options; From 912aa9491cb50b1a3d85970b772de84508dc0904 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:43:29 +0200 Subject: [PATCH 02/97] CryptoUtil --- Cargo.lock | 1 + crypto/Cargo.toml | 1 + crypto/src/error.rs | 2 + crypto/src/keypair.rs | 79 ++++++++++++++++++++++++++++++++++++++ example/Cargo.lock | 67 +++++++++++++++++++------------- example/Cargo.toml | 2 +- example/keygen/Cargo.toml | 7 ++++ example/keygen/src/main.rs | 13 +++++++ example/server/src/main.rs | 15 ++++---- src/lib.rs | 1 - 10 files changed, 153 insertions(+), 35 deletions(-) create mode 100644 example/keygen/Cargo.toml create mode 100644 example/keygen/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 43b5b29..2e8d29d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,6 +1029,7 @@ name = "mtp-crypto" version = "0.1.0" dependencies = [ "aes-gcm", + "base64", "chacha20poly1305", "ed25519-dalek", "getrandom 0.4.3", diff --git a/crypto/Cargo.toml b/crypto/Cargo.toml index 26031c7..cc98dc8 100644 --- a/crypto/Cargo.toml +++ b/crypto/Cargo.toml @@ -17,6 +17,7 @@ hkdf = { version = "0.13", optional = true } sha2 = { version = "0.11", optional = true } zeroize = { version = "1.9", features = ["derive"] } thiserror = "1" +base64 = "0.22" rand_core = { version = "0.6", features = ["getrandom"] } getrandom = "0.4.3" mlkem-tls = { version = "0.2", optional = true } diff --git a/crypto/src/error.rs b/crypto/src/error.rs index c1a535f..4c5cef0 100644 --- a/crypto/src/error.rs +++ b/crypto/src/error.rs @@ -28,4 +28,6 @@ pub enum CryptoError { UnknownAlgorithm, #[error("invalid hex encoding")] InvalidHex, + #[error("invalid base64 encoding")] + InvalidBase64, } diff --git a/crypto/src/keypair.rs b/crypto/src/keypair.rs index 169eaa8..3454cc9 100644 --- a/crypto/src/keypair.rs +++ b/crypto/src/keypair.rs @@ -1,4 +1,7 @@ use std::fmt; + +use base64::Engine; +use base64::engine::general_purpose; use zeroize::{Zeroize, ZeroizeOnDrop}; // --- Private key types --- @@ -197,6 +200,16 @@ fn hex_to_bytes(s: &str) -> Result, crate::error::CryptoError> { .collect() } +fn bytes_to_base64(bytes: &[u8]) -> String { + general_purpose::STANDARD.encode(bytes) +} + +fn base64_to_bytes(s: &str) -> Result, crate::error::CryptoError> { + general_purpose::STANDARD + .decode(s) + .map_err(|_| crate::error::CryptoError::InvalidBase64) +} + // --- #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -510,6 +523,22 @@ impl Keyring { sig_cl_secret_key: SignaturePrivateKey::new(read_key(&mut offset)?), }) } + + pub fn to_hex(&self) -> String { + bytes_to_hex(&self.to_bytes()) + } + + pub fn from_hex(s: &str) -> Result { + Self::from_bytes(&hex_to_bytes(s)?) + } + + pub fn to_base64(&self) -> String { + bytes_to_base64(&self.to_bytes()) + } + + pub fn from_base64(s: &str) -> Result { + Self::from_bytes(&base64_to_bytes(s)?) + } } impl TryFrom<&[u8]> for Keyring { @@ -645,6 +674,14 @@ impl PublicKeyBundle { sig_cl_public_key: cl, }) } + + pub fn to_base64(&self) -> String { + bytes_to_base64(&self.as_bytes()) + } + + pub fn from_base64(s: &str) -> Result { + Self::from_bytes(&base64_to_bytes(s)?) + } } impl TryFrom<&[u8]> for PublicKeyBundle { @@ -760,6 +797,48 @@ mod tests { assert_eq!(key.as_bytes(), recovered.as_bytes()); } + #[test] + fn keyring_hex_roundtrip() { + let keyring = Keyring::new( + KemPublicKey::new(vec![1u8; 16]), + KemPrivateKey::new(vec![2u8; 16]), + SignaturePqPublicKey::new(vec![3u8; 16]), + SignaturePqPrivateKey::new(vec![4u8; 16]), + SignaturePublicKey::new(vec![5u8; 16]), + SignaturePrivateKey::new(vec![6u8; 16]), + ); + let hex = keyring.to_hex(); + let recovered = Keyring::from_hex(&hex).unwrap(); + assert_eq!(keyring.to_bytes(), recovered.to_bytes()); + } + + #[test] + fn keyring_base64_roundtrip() { + let keyring = Keyring::new( + KemPublicKey::new(vec![1u8; 16]), + KemPrivateKey::new(vec![2u8; 16]), + SignaturePqPublicKey::new(vec![3u8; 16]), + SignaturePqPrivateKey::new(vec![4u8; 16]), + SignaturePublicKey::new(vec![5u8; 16]), + SignaturePrivateKey::new(vec![6u8; 16]), + ); + let b64 = keyring.to_base64(); + let recovered = Keyring::from_base64(&b64).unwrap(); + assert_eq!(keyring.to_bytes(), recovered.to_bytes()); + } + + #[test] + fn public_key_bundle_base64_roundtrip() { + let bundle = PublicKeyBundle::new( + KemPublicKey::new(vec![1u8; 32]), + SignaturePqPublicKey::new(vec![2u8; 64]), + SignaturePublicKey::new(vec![3u8; 32]), + ); + let b64 = bundle.to_base64(); + let recovered = PublicKeyBundle::from_base64(&b64).unwrap(); + assert_eq!(bundle.as_bytes(), recovered.as_bytes()); + } + #[test] fn hex_invalid_returns_error() { assert!(KemPublicKey::from_hex("xyz").is_err()); diff --git a/example/Cargo.lock b/example/Cargo.lock index 49da683..e07c673 100644 --- a/example/Cargo.lock +++ b/example/Cargo.lock @@ -59,9 +59,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -70,14 +70,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -595,9 +596,9 @@ checksum = "1a9fcbcc408c5526c3ab80d534e5c86e7967c1fb7aa0a8c76abd1edc27deb877" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "ctutils", "typenum", @@ -743,9 +744,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -771,6 +772,13 @@ dependencies = [ "cpufeatures 0.3.0", ] +[[package]] +name = "keygen" +version = "0.1.0" +dependencies = [ + "mtp", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -937,6 +945,7 @@ dependencies = [ name = "mtp-crypto" version = "0.1.0" dependencies = [ + "base64", "chacha20poly1305", "ed25519-dalek", "getrandom 0.4.3", @@ -995,9 +1004,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" dependencies = [ "num-integer", "num-traits", @@ -1134,6 +1143,12 @@ dependencies = [ "spki 0.8.0", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "poly1305" version = "0.8.0" @@ -1360,9 +1375,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -1412,9 +1427,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -1758,9 +1773,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -1778,9 +1793,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -1951,9 +1966,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -1964,9 +1979,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1974,9 +1989,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -1987,9 +2002,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] diff --git a/example/Cargo.toml b/example/Cargo.toml index 9ec3f02..48506d1 100644 --- a/example/Cargo.toml +++ b/example/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ "server", - "client", + "client", "keygen", ] resolver = "3" diff --git a/example/keygen/Cargo.toml b/example/keygen/Cargo.toml new file mode 100644 index 0000000..ddfadad --- /dev/null +++ b/example/keygen/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "keygen" +version = "0.1.0" +edition = "2024" + +[dependencies] +mtp = { version = "0.1.0", path = "../../", features = ["crypto"] } diff --git a/example/keygen/src/main.rs b/example/keygen/src/main.rs new file mode 100644 index 0000000..938a3e0 --- /dev/null +++ b/example/keygen/src/main.rs @@ -0,0 +1,13 @@ +use mtp::crypto::Keyring; + +fn main() { + let keyring = Keyring::generate(); + let bundle = keyring.public_key_bundle(); + + // The `Debug` impl redacts private keys by design, so use the encoding + // methods to emit the full keyring (public + secret keys) instead. + println!("Keyring (hex):\n{}\n", keyring.to_hex()); + println!("Keyring (base64):\n{}\n", keyring.to_base64()); + + println!("PublicKeyBundle (base64):\n{}", bundle.to_base64()); +} diff --git a/example/server/src/main.rs b/example/server/src/main.rs index 3de4a97..776b2b6 100644 --- a/example/server/src/main.rs +++ b/example/server/src/main.rs @@ -3,7 +3,8 @@ mod handlers; mod keys; mod tls; -use mtp::host::{HostConfig, MTPHost}; +use mtp::host::{AuthenticationPolicy, HostConfig, MTPHost}; + use mtp::type_map::TypeMap; use std::future::Future; use std::path::Path; @@ -77,7 +78,9 @@ async fn main() -> Result<(), Box> { Ok(()) => {} Err(e) => eprintln!("Failed to persist client database to {path}: {e}"), }, - Err(e) => eprintln!("Failed to serialize client database after registering {id}: {e}"), + Err(e) => { + eprintln!("Failed to serialize client database after registering {id}: {e}") + } } println!("Registered new client with ID: {}", id); id @@ -92,16 +95,14 @@ async fn main() -> Result<(), Box> { cert_pem, key_pem, ) - .with_authentication(host_keyring, get_existing_user, complete_register); + .with_authentication(host_keyring, get_existing_user, complete_register) + .with_authentication_policy(AuthenticationPolicy::ForceAuthentication); let mut host = MTPHost::new(config).await?; println!("Server listening on {}", host.local_addr()); while let Some(conn) = host.accept().await? { - let desc = conn - .description - .as_deref() - .unwrap_or("(no description)"); + let desc = conn.description.as_deref().unwrap_or("(no description)"); println!( "\n--- New connection (version {}, description: {desc}) ---", conn.version diff --git a/src/lib.rs b/src/lib.rs index e1325bc..03314f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ pub use mtp_codec as codec; pub use mtp_common as common; -pub use mtp_transport as transport; pub use mtp_type_map as type_map; #[cfg(feature = "crypto")] From efe5cca9bbe4d5695dfd6a7eba449c2ea84c3d88 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:51:27 +0200 Subject: [PATCH 03/97] getFunctions --- host/src/lib.rs | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index 9cfb5b5..89cae77 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -13,9 +13,10 @@ use tokio::time::Duration; /* ---- async callback type aliases ---- */ #[cfg(feature = "crypto")] -type GetExistingUser = Box< +pub type GetExistingUser = Box< dyn Fn( u64, + Option, ) -> Pin> + Send>> + Send @@ -23,8 +24,11 @@ type GetExistingUser = Box< >; #[cfg(feature = "crypto")] -type CompleteRegister = Box< - dyn Fn(mtp_crypto::PublicKeyBundle) -> Pin + Send>> +pub type CompleteRegister = Box< + dyn Fn( + mtp_crypto::PublicKeyBundle, + Option, + ) -> Pin + Send>> + Send + Sync, >; @@ -87,19 +91,8 @@ impl HostConfig { pub fn with_authentication( mut self, host_keyring: mtp_crypto::Keyring, - get_existing_user: impl Fn( - u64, - ) -> Pin< - Box> + Send>, - > + Send - + Sync - + 'static, - complete_register: impl Fn( - mtp_crypto::PublicKeyBundle, - ) -> Pin + Send>> - + Send - + Sync - + 'static, + get_existing_user: GetExistingUser, + complete_register: CompleteRegister, ) -> Self { self.authentication_policy = AuthenticationPolicy::ForceAuthentication; self.host_keyring = host_keyring; From b85cda505f640d343e3b0c0947a9fa0707fd5404 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:56:34 +0200 Subject: [PATCH 04/97] exports --- client/src/lib.rs | 3 +++ host/src/lib.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/client/src/lib.rs b/client/src/lib.rs index 223ea9f..a26f466 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -4,6 +4,9 @@ use mtp_transport::{Policy, Receiver, Sender}; #[cfg(feature = "crypto")] use tokio::time::Duration; +pub use MTPClient as Client; +pub use MTPConnection as Connection; + #[cfg(feature = "crypto")] fn unexpected_response_type_error( context: &str, diff --git a/host/src/lib.rs b/host/src/lib.rs index 89cae77..c285894 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -11,6 +11,9 @@ use std::{error::Error, fmt}; #[cfg(feature = "crypto")] use tokio::time::Duration; +pub use MTPConnection as Connection; +pub use MTPHost as Host; + /* ---- async callback type aliases ---- */ #[cfg(feature = "crypto")] pub type GetExistingUser = Box< From 099073dc273829c0059163b4dbda75672cbc25ef Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:00:13 +0200 Subject: [PATCH 05/97] [Fix] Swapped from User to client --- host/src/lib.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index c285894..7006305 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -16,7 +16,7 @@ pub use MTPHost as Host; /* ---- async callback type aliases ---- */ #[cfg(feature = "crypto")] -pub type GetExistingUser = Box< +pub type GetExistingClient = Box< dyn Fn( u64, Option, @@ -58,7 +58,7 @@ pub struct HostConfig { #[cfg(feature = "crypto")] pub host_keyring: mtp_crypto::Keyring, #[cfg(feature = "crypto")] - pub get_existing_user: GetExistingUser, + pub get_existing_client: GetExistingClient, #[cfg(feature = "crypto")] pub complete_register: CompleteRegister, } @@ -84,7 +84,7 @@ impl HostConfig { mtp_crypto::SignaturePrivateKey::new(Vec::new()), ), #[cfg(feature = "crypto")] - get_existing_user: Box::new(|_| Box::pin(async { None })), + get_existing_client: Box::new(|_| Box::pin(async { None })), #[cfg(feature = "crypto")] complete_register: Box::new(|_| Box::pin(async { 0 })), } @@ -94,12 +94,12 @@ impl HostConfig { pub fn with_authentication( mut self, host_keyring: mtp_crypto::Keyring, - get_existing_user: GetExistingUser, + get_existing_client: GetExistingClient, complete_register: CompleteRegister, ) -> Self { self.authentication_policy = AuthenticationPolicy::ForceAuthentication; self.host_keyring = host_keyring; - self.get_existing_user = Box::new(get_existing_user); + self.get_existing_client = Box::new(get_existing_client); self.complete_register = Box::new(complete_register); self } @@ -393,7 +393,7 @@ impl MTPHost { )); } }; - let bundle = match (self.config.get_existing_user)(cid).await { + let bundle = match (self.config.get_existing_client)(cid).await { Some(b) => b, None => { let rejection = CommunicationValue::new( @@ -724,7 +724,7 @@ impl MTPHost { }; if cid > 0 - && let Some(bundle) = (self.config.get_existing_user)(cid).await + && let Some(bundle) = (self.config.get_existing_client)(cid).await { return self .complete_auth_handshake( From 54a3ef4f1121ab7ad14e5b9bd8d653387fc8ddbf Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:33:33 +0200 Subject: [PATCH 06/97] [Fix] host crypto --- host/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index 7006305..0bc7e62 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -84,9 +84,9 @@ impl HostConfig { mtp_crypto::SignaturePrivateKey::new(Vec::new()), ), #[cfg(feature = "crypto")] - get_existing_client: Box::new(|_| Box::pin(async { None })), + get_existing_client: Box::new(|_, _| Box::pin(async { None })), #[cfg(feature = "crypto")] - complete_register: Box::new(|_| Box::pin(async { 0 })), + complete_register: Box::new(|_, _| Box::pin(async { 0 })), } } @@ -393,7 +393,8 @@ impl MTPHost { )); } }; - let bundle = match (self.config.get_existing_client)(cid).await { + let bundle = match (self.config.get_existing_client)(cid, description.clone()).await + { Some(b) => b, None => { let rejection = CommunicationValue::new( @@ -589,7 +590,7 @@ impl MTPHost { let (assigned_id, client_bundle) = match flow { Flow::Login { id, bundle } => (id, bundle), Flow::Register { bundle, .. } => { - let new_id = (self.config.complete_register)(bundle.clone()).await; + let new_id = (self.config.complete_register)(bundle.clone(), description.clone()).await; (new_id, bundle) } }; @@ -724,7 +725,7 @@ impl MTPHost { }; if cid > 0 - && let Some(bundle) = (self.config.get_existing_client)(cid).await + && let Some(bundle) = (self.config.get_existing_client)(cid, description.clone()).await { return self .complete_auth_handshake( From bed73a82f11e10dd38ec2b6e2fecbb045c6ecf16 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:37:53 +0200 Subject: [PATCH 07/97] [Add] Sender & Receiver Exports --- host/src/lib.rs | 114 +++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index 0bc7e62..c7be550 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -3,7 +3,7 @@ use mtp_codec::{ registry::{Registry, VersionedCodec}, }; use mtp_common::CommunicationError; -use mtp_transport::{Policy, Receiver, Sender}; +use mtp_transport::Policy; use std::net::IpAddr; #[cfg(feature = "crypto")] use std::pin::Pin; @@ -13,6 +13,8 @@ use tokio::time::Duration; pub use MTPConnection as Connection; pub use MTPHost as Host; +pub use mtp_transport::Receiver; +pub use mtp_transport::Sender; /* ---- async callback type aliases ---- */ #[cfg(feature = "crypto")] @@ -382,59 +384,59 @@ impl MTPHost { _ => None, }; - let (flow, response_type) = - if hello.get_type() == mtp_codec::CommunicationType::Identification.to_id(&tm) { - let cid = match hello.get_data(DataType::Id) { - DataValue::UnsignedNumber(n) => *n as u64, - _ => { - sender.close(); - return Err(AcceptError::AuthenticationFailed( - "missing client id".into(), - )); - } - }; - let bundle = match (self.config.get_existing_client)(cid, description.clone()).await - { - Some(b) => b, - None => { - let rejection = CommunicationValue::new( - mtp_codec::CommunicationType::IdentificationResponse, - ) - .add_typed_default(DataType::Connected, DataValue::BoolFalse); - let _ = sender.send(&rejection).await; - sender.close(); - return Err(AcceptError::AuthenticationFailed( - "unknown client id".into(), - )); - } - }; - ( - Flow::Login { id: cid, bundle }, - mtp_codec::CommunicationType::IdentificationResponse, - ) - } else if hello.get_type() == mtp_codec::CommunicationType::Register.to_id(&tm) { - let bundle = match hello.get_data(DataType::PublicKeys) { - DataValue::Bytes(b) => PublicKeyBundle::from_bytes(b).map_err(|_| { - AcceptError::AuthenticationFailed("invalid public key bundle".into()) - })?, - _ => { - sender.close(); - return Err(AcceptError::AuthenticationFailed( - "missing public keys".into(), - )); - } - }; - let pk_bytes = bundle.as_bytes(); - ( - Flow::Register { bundle, pk_bytes }, - mtp_codec::CommunicationType::RegisterResponse, - ) - } else { - sender.close(); - return Err(AcceptError::AuthenticationFailed( - "unexpected authentication message".into(), - )); + let (flow, response_type) = if hello.get_type() + == mtp_codec::CommunicationType::Identification.to_id(&tm) + { + let cid = match hello.get_data(DataType::Id) { + DataValue::UnsignedNumber(n) => *n as u64, + _ => { + sender.close(); + return Err(AcceptError::AuthenticationFailed( + "missing client id".into(), + )); + } }; + let bundle = match (self.config.get_existing_client)(cid, description.clone()).await { + Some(b) => b, + None => { + let rejection = CommunicationValue::new( + mtp_codec::CommunicationType::IdentificationResponse, + ) + .add_typed_default(DataType::Connected, DataValue::BoolFalse); + let _ = sender.send(&rejection).await; + sender.close(); + return Err(AcceptError::AuthenticationFailed( + "unknown client id".into(), + )); + } + }; + ( + Flow::Login { id: cid, bundle }, + mtp_codec::CommunicationType::IdentificationResponse, + ) + } else if hello.get_type() == mtp_codec::CommunicationType::Register.to_id(&tm) { + let bundle = match hello.get_data(DataType::PublicKeys) { + DataValue::Bytes(b) => PublicKeyBundle::from_bytes(b).map_err(|_| { + AcceptError::AuthenticationFailed("invalid public key bundle".into()) + })?, + _ => { + sender.close(); + return Err(AcceptError::AuthenticationFailed( + "missing public keys".into(), + )); + } + }; + let pk_bytes = bundle.as_bytes(); + ( + Flow::Register { bundle, pk_bytes }, + mtp_codec::CommunicationType::RegisterResponse, + ) + } else { + sender.close(); + return Err(AcceptError::AuthenticationFailed( + "unexpected authentication message".into(), + )); + }; self.complete_auth_handshake( sender, @@ -590,7 +592,8 @@ impl MTPHost { let (assigned_id, client_bundle) = match flow { Flow::Login { id, bundle } => (id, bundle), Flow::Register { bundle, .. } => { - let new_id = (self.config.complete_register)(bundle.clone(), description.clone()).await; + let new_id = + (self.config.complete_register)(bundle.clone(), description.clone()).await; (new_id, bundle) } }; @@ -725,7 +728,8 @@ impl MTPHost { }; if cid > 0 - && let Some(bundle) = (self.config.get_existing_client)(cid, description.clone()).await + && let Some(bundle) = + (self.config.get_existing_client)(cid, description.clone()).await { return self .complete_auth_handshake( From 7ffc1ab3fece273fed36f9729793a13a4a973d88 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:40:46 +0200 Subject: [PATCH 08/97] [Add] host with policy --- client/src/lib.rs | 4 +++- host/src/lib.rs | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index a26f466..c358d4c 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,11 +1,13 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Version}; use mtp_common::CommunicationError; -use mtp_transport::{Policy, Receiver, Sender}; +use mtp_transport::Policy; #[cfg(feature = "crypto")] use tokio::time::Duration; pub use MTPClient as Client; pub use MTPConnection as Connection; +pub use mtp_transport::Receiver; +pub use mtp_transport::Sender; #[cfg(feature = "crypto")] fn unexpected_response_type_error( diff --git a/host/src/lib.rs b/host/src/lib.rs index c7be550..3573771 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -53,6 +53,8 @@ pub struct HostConfig { pub tls_fullchain: Vec, pub tls_key: Vec, + pub policy: Policy, + #[cfg(feature = "crypto")] pub authentication_policy: AuthenticationPolicy, #[cfg(feature = "crypto")] @@ -72,6 +74,7 @@ impl HostConfig { port, tls_fullchain, tls_key, + policy: Policy::default(), #[cfg(feature = "crypto")] authentication_policy: AuthenticationPolicy::Unauthenticated, #[cfg(feature = "crypto")] @@ -92,6 +95,11 @@ impl HostConfig { } } + pub fn with_policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + #[cfg(feature = "crypto")] pub fn with_authentication( mut self, @@ -190,7 +198,7 @@ impl MTPHost { config.port, config.tls_fullchain.clone(), config.tls_key.clone(), - Policy::default(), + config.policy, ) .await?; From b78cf3f82be29108892211911fde53dc59a539d1 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:42:47 +0200 Subject: [PATCH 09/97] [Add] Policy derives --- transport/src/connection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 21f8121..9c59039 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -14,7 +14,7 @@ pub enum SendMode { SingleStreamPerMessage, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Policy { pub send_mode: SendMode, pub max_message_size: u64, From 0967120e3b315d4acc9827464179c8654ca21497 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:45:22 +0200 Subject: [PATCH 10/97] [Fix] Correct exports --- client/src/lib.rs | 3 ++- host/src/lib.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index c358d4c..a7b55d3 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,12 +1,13 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Version}; use mtp_common::CommunicationError; -use mtp_transport::Policy; #[cfg(feature = "crypto")] use tokio::time::Duration; pub use MTPClient as Client; pub use MTPConnection as Connection; +pub use mtp_transport::Policy; pub use mtp_transport::Receiver; +pub use mtp_transport::SendMode; pub use mtp_transport::Sender; #[cfg(feature = "crypto")] diff --git a/host/src/lib.rs b/host/src/lib.rs index 3573771..4e83657 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -13,7 +13,9 @@ use tokio::time::Duration; pub use MTPConnection as Connection; pub use MTPHost as Host; +pub use mtp_transport::Policy; pub use mtp_transport::Receiver; +pub use mtp_transport::SendMode; pub use mtp_transport::Sender; /* ---- async callback type aliases ---- */ From 44ff1d8781b4645dbff3dce2406155e4d9d43a4c Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:47:44 +0200 Subject: [PATCH 11/97] [Fix] Correct exports 2 --- host/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index 4e83657..3650385 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -3,7 +3,6 @@ use mtp_codec::{ registry::{Registry, VersionedCodec}, }; use mtp_common::CommunicationError; -use mtp_transport::Policy; use std::net::IpAddr; #[cfg(feature = "crypto")] use std::pin::Pin; From 75f4139dea7d807334db3be930c4150d2d93278f Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:56:54 +0200 Subject: [PATCH 12/97] mk & mpkb files --- Cargo.lock | 44 +++++--- Cargo.toml | 7 ++ example/.gitignore | 3 + example/Cargo.lock | 11 +- example/client/Cargo.toml | 4 +- example/client/src/auth.rs | 71 ++++++------ example/client/src/main.rs | 20 +--- example/client/src/messages.rs | 22 +++- example/keygen/Cargo.toml | 2 +- example/keygen/src/main.rs | 36 ++++-- example/server/Cargo.toml | 2 +- example/server/src/handlers.rs | 13 ++- example/server/src/keys.rs | 55 ++++----- example/server/src/main.rs | 13 ++- files/Cargo.toml | 10 ++ files/src/lib.rs | 197 +++++++++++++++++++++++++++++++++ src/lib.rs | 3 + 17 files changed, 382 insertions(+), 131 deletions(-) create mode 100644 files/Cargo.toml create mode 100644 files/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 2e8d29d..853e843 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,9 +95,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -106,14 +106,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -982,6 +983,7 @@ dependencies = [ "mtp-codec", "mtp-common", "mtp-crypto", + "mtp-files", "mtp-host", "mtp-transport", "mtp-type-map", @@ -1043,6 +1045,14 @@ dependencies = [ "zeroize", ] +[[package]] +name = "mtp-files" +version = "0.1.0" +dependencies = [ + "mtp-crypto", + "thiserror 1.0.69", +] + [[package]] name = "mtp-host" version = "0.1.0" @@ -1117,9 +1127,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" dependencies = [ "num-integer", "num-traits", @@ -1263,6 +1273,12 @@ dependencies = [ "spki 0.8.0", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "poly1305" version = "0.8.0" @@ -1501,9 +1517,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -1553,9 +1569,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -1896,9 +1912,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -1916,9 +1932,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", diff --git a/Cargo.toml b/Cargo.toml index bf82e29..275a012 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "host", "client", "wasm", + "files", ] # `wasm` is a wasm32-only crate: it relies on web-sys unstable APIs # (`--cfg=web_sys_unstable_apis`, set in wasm/.cargo/config.toml) and the @@ -26,6 +27,7 @@ default-members = [ "transport", "host", "client", + "files", ] resolver = "3" @@ -58,6 +60,7 @@ mtp-crypto = { version = "0.1.0", path = "crypto", optional = true, features = [ ] } mtp-host = { version = "0.1.0", path = "host", optional = true } mtp-client = { version = "0.1.0", path = "client", optional = true } +mtp-files = { version = "0.1.0", path = "files", optional = true } [features] default = [] @@ -80,6 +83,10 @@ host = ["dep:mtp-host", "mtp-codec/registry", "mtp-transport/host"] # MTP client - outgoing QUIC connections to a host. client = ["dep:mtp-client"] +# On-disk storage for keyrings (`.mk`) and public key bundles (`.mpkb`). +# Pulls in `crypto` so the `Keyring` / `PublicKeyBundle` types are in scope. +files = ["dep:mtp-files", "crypto"] + [dev-dependencies] tokio = { version = "1", features = ["full"] } rcgen = "0.14" diff --git a/example/.gitignore b/example/.gitignore index 0d3408c..94b9ca3 100644 --- a/example/.gitignore +++ b/example/.gitignore @@ -10,3 +10,6 @@ web-client/node_modules web-client/public/host_public_key_bundle.hex web-client/public/mtp_dev_cert_hash.txt web-client/dist/ + +*.mk +*.mpkb diff --git a/example/Cargo.lock b/example/Cargo.lock index e07c673..0696b49 100644 --- a/example/Cargo.lock +++ b/example/Cargo.lock @@ -207,9 +207,7 @@ dependencies = [ name = "client" version = "0.1.0" dependencies = [ - "hex", "mtp", - "serde_json", "tokio", ] @@ -902,6 +900,7 @@ dependencies = [ "mtp-codec", "mtp-common", "mtp-crypto", + "mtp-files", "mtp-host", "mtp-transport", "mtp-type-map", @@ -959,6 +958,14 @@ dependencies = [ "zeroize", ] +[[package]] +name = "mtp-files" +version = "0.1.0" +dependencies = [ + "mtp-crypto", + "thiserror 1.0.69", +] + [[package]] name = "mtp-host" version = "0.1.0" diff --git a/example/client/Cargo.toml b/example/client/Cargo.toml index 2e0556b..d68f33e 100644 --- a/example/client/Cargo.toml +++ b/example/client/Cargo.toml @@ -8,7 +8,5 @@ name = "client" path = "src/main.rs" [dependencies] -mtp = { version = "0.1.0", path = "../../", features = ["client", "crypto"] } +mtp = { version = "0.1.0", path = "../../", features = ["client", "crypto", "files"] } tokio = { version = "1", features = ["full"] } -serde_json = "1" -hex = "0.4" diff --git a/example/client/src/auth.rs b/example/client/src/auth.rs index 5a4cf35..edb5f8a 100644 --- a/example/client/src/auth.rs +++ b/example/client/src/auth.rs @@ -1,52 +1,49 @@ use std::fs; use mtp::client::{ClientConfig, MTPClient, MTPConnection}; -use mtp::crypto::{Ed25519Signer, Keyring, MlDsaSigner, PublicKeyBundle}; +use mtp::crypto::{ + Ed25519Signer, KemPrivateKey, KemPublicKey, Keyring, MlDsaSigner, PublicKeyBundle, +}; +use mtp::files::{load_keyring, save_keyring}; pub async fn connect_or_register( mut config: ClientConfig, host_public_key: PublicKeyBundle, - client_key_path: &str, + key_prefix: &str, ) -> Result<(MTPConnection, Keyring), Box> { - if let Ok(data) = fs::read_to_string(client_key_path) { - let json: serde_json::Value = serde_json::from_str(&data)?; - let client_id = json["client_id"].as_u64().expect("Invalid client_id"); - let keyring = Keyring::from_bytes(&hex::decode( - json["keyring"].as_str().expect("Missing keyring"), - )?)?; + let keyring_path = format!("{key_prefix}.mk"); + let id_path = format!("{key_prefix}.id"); - println!("Loaded client keys (ID: {})", client_id); + if let (Ok(keyring), Ok(id)) = (load_keyring(&keyring_path), fs::read_to_string(&id_path)) { + let client_id: u64 = id.trim().parse()?; + println!("Loaded client keys (ID: {client_id})"); config.client_id = client_id; let conn = MTPClient::auth_connect(config, &keyring, &host_public_key).await?; println!("Authenticated (version {})", conn.version); - Ok((conn, keyring)) - } else { - println!("No existing keys found: registering new client"); - - let (_ed_signer, sig_sk, sig_pk) = Ed25519Signer::generate(); - let (_pq_signer, sig_pq_sk, sig_pq_pk) = MlDsaSigner::generate(); - let keyring = Keyring::new( - mtp::crypto::KemPublicKey::new(vec![]), - mtp::crypto::KemPrivateKey::new(vec![]), - sig_pq_pk, - sig_pq_sk, - sig_pk, - sig_sk, - ); - - let keyring_bytes = keyring.to_bytes(); - let conn = MTPClient::auth_register(config, &keyring, &host_public_key).await?; - println!("Registered with ID: {}", conn.client_id); - - let json = serde_json::json!({ - "client_id": conn.client_id, - "keyring": hex::encode(&keyring_bytes), - }); - fs::write(client_key_path, serde_json::to_string_pretty(&json)?)?; - println!("Saved client keys -> {client_key_path}"); - - let keyring = Keyring::from_bytes(&keyring_bytes)?; - Ok((conn, keyring)) + return Ok((conn, keyring)); } + + println!("No existing keys found: registering new client"); + + /* The client authenticates with signatures only, so the KEM slot is empty. */ + let (_ed_signer, sig_sk, sig_pk) = Ed25519Signer::generate(); + let (_pq_signer, sig_pq_sk, sig_pq_pk) = MlDsaSigner::generate(); + let keyring = Keyring::new( + KemPublicKey::new(vec![]), + KemPrivateKey::new(vec![]), + sig_pq_pk, + sig_pq_sk, + sig_pk, + sig_sk, + ); + + let conn = MTPClient::auth_register(config, &keyring, &host_public_key).await?; + println!("Registered with ID: {}", conn.client_id); + + save_keyring(&keyring, &keyring_path)?; + fs::write(&id_path, conn.client_id.to_string())?; + println!("Saved client keys -> {keyring_path}"); + + Ok((conn, keyring)) } diff --git a/example/client/src/main.rs b/example/client/src/main.rs index b44e05d..1a67cac 100644 --- a/example/client/src/main.rs +++ b/example/client/src/main.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::Path; use mtp::client::ClientConfig; -use mtp::crypto::{KemPublicKey, PublicKeyBundle, SignaturePqPublicKey, SignaturePublicKey}; +use mtp::files::load_public_key_bundle; fn dev_cert_path() -> String { std::env::var("MTP_DEV_CERT").unwrap_or_else(|_| { @@ -25,19 +25,8 @@ async fn main() -> Result<(), Box> { "Missing TLS certificate at {cert_path}: enter the Nix shell first or run the server to generate it: {e}" ) }); - let host_public_key = PublicKeyBundle::new( - KemPublicKey::new( - fs::read("host_enc_kem_pk.bin") - .expect("Missing host_enc_kem_pk.bin: run server first"), - ), - SignaturePqPublicKey::new( - fs::read("host_sig_pq_pk.bin") - .expect("Missing host_sig_pq_pk.bin: run server first"), - ), - SignaturePublicKey::new( - fs::read("host_sig_pk.bin").expect("Missing host_sig_pk.bin: run server first"), - ), - ); + let host_public_key = load_public_key_bundle("host.mpkb") + .expect("Missing host.mpkb: run the server first to export it"); println!("Connecting to 127.0.0.1:8080 ..."); @@ -46,8 +35,7 @@ async fn main() -> Result<(), Box> { .with_description("MTP example client"); let server_bundle = host_public_key.clone(); - let (conn, keyring) = - auth::connect_or_register(config, host_public_key, "client_keys.json").await?; + let (conn, keyring) = auth::connect_or_register(config, host_public_key, "client").await?; messages::send_and_receive(&conn, &keyring, &server_bundle).await?; println!("\nDone"); diff --git a/example/client/src/messages.rs b/example/client/src/messages.rs index ca6ea50..45f1dc0 100644 --- a/example/client/src/messages.rs +++ b/example/client/src/messages.rs @@ -10,20 +10,26 @@ pub fn build_demo_message( ) -> CommunicationValue { // Encrypt to the server's KEM public key; the server decrypts with its keyring. let enc_type = EncryptionType::MlKemChaCha20Poly1305; - let signer = Ed25519Signer::new(&keyring.sig_cl_secret_key) - .expect("Ed25519 signer from keyring"); + let signer = + Ed25519Signer::new(&keyring.sig_cl_secret_key).expect("Ed25519 signer from keyring"); let tm = TypeMap::latest(); let inner_enc = DataValue::Container(vec![ - (DataType::Version.to_id(&tm), DataValue::Str("secret inner data".into())), + ( + DataType::Version.to_id(&tm), + DataValue::Str("secret inner data".into()), + ), (DataType::Id.to_id(&tm), DataValue::UnsignedNumber(42)), ]); let mut dv_enc = inner_enc; dv_enc.encrypt_container(enc_type, server_bundle, b"demo-aad"); let inner_sig = DataValue::Container(vec![ - (DataType::Version.to_id(&tm), DataValue::Str("signed by client".into())), + ( + DataType::Version.to_id(&tm), + DataValue::Str("signed by client".into()), + ), (DataType::Id.to_id(&tm), DataValue::UnsignedNumber(99)), ]); let mut dv_sig = inner_sig; @@ -37,7 +43,13 @@ pub fn build_demo_message( (DataType::Id.to_id(&tm), DataValue::UnsignedNumber(7)), ]); let mut dv_sec = inner_sec; - dv_sec.sign_and_encrypt_container(SigAlgorithm::ED25519, &signer, enc_type, server_bundle, b"demo-aad"); + dv_sec.sign_and_encrypt_container( + SigAlgorithm::ED25519, + &signer, + enc_type, + server_bundle, + b"demo-aad", + ); let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/example/keygen/Cargo.toml b/example/keygen/Cargo.toml index ddfadad..fd4d284 100644 --- a/example/keygen/Cargo.toml +++ b/example/keygen/Cargo.toml @@ -4,4 +4,4 @@ version = "0.1.0" edition = "2024" [dependencies] -mtp = { version = "0.1.0", path = "../../", features = ["crypto"] } +mtp = { version = "0.1.0", path = "../../", features = ["files"] } diff --git a/example/keygen/src/main.rs b/example/keygen/src/main.rs index 938a3e0..bbaa1e3 100644 --- a/example/keygen/src/main.rs +++ b/example/keygen/src/main.rs @@ -1,13 +1,35 @@ +use std::path::PathBuf; + use mtp::crypto::Keyring; +use mtp::files::{ + self, BUNDLE_EXTENSION, KEYRING_EXTENSION, load_keyring, load_public_key_bundle, save_keyring, + save_public_key_bundle, +}; + +fn main() -> Result<(), files::FileError> { + let prefix = std::env::args() + .nth(1) + .unwrap_or_else(|| "keyring".to_string()); + let keyring_path = PathBuf::from(format!("{prefix}.{KEYRING_EXTENSION}")); + let bundle_path = PathBuf::from(format!("{prefix}.{BUNDLE_EXTENSION}")); -fn main() { let keyring = Keyring::generate(); - let bundle = keyring.public_key_bundle(); + save_keyring(&keyring, &keyring_path)?; + save_public_key_bundle(&keyring.public_key_bundle(), &bundle_path)?; - // The `Debug` impl redacts private keys by design, so use the encoding - // methods to emit the full keyring (public + secret keys) instead. - println!("Keyring (hex):\n{}\n", keyring.to_hex()); - println!("Keyring (base64):\n{}\n", keyring.to_base64()); + /* Read both back to confirm the files round-trip through the on-disk format. */ + let loaded_keyring = load_keyring(&keyring_path)?; + let loaded_bundle = load_public_key_bundle(&bundle_path)?; + assert_eq!(keyring.to_bytes(), loaded_keyring.to_bytes()); + assert_eq!( + keyring.public_key_bundle().as_bytes(), + loaded_bundle.as_bytes() + ); + println!("\nPrivateKeyRing (base64):\n{}", keyring.to_base64()); - println!("PublicKeyBundle (base64):\n{}", bundle.to_base64()); + println!("\nPublicKeyBundle (base64):\n{}", loaded_bundle.to_base64()); + + println!("Wrote keyring -> {}", keyring_path.display()); + println!("Wrote bundle -> {}", bundle_path.display()); + Ok(()) } diff --git a/example/server/Cargo.toml b/example/server/Cargo.toml index cf87364..494f36f 100644 --- a/example/server/Cargo.toml +++ b/example/server/Cargo.toml @@ -8,7 +8,7 @@ name = "server" path = "src/main.rs" [dependencies] -mtp = { version = "0.1.0", path = "../../", features = ["crypto", "host"] } +mtp = { version = "0.1.0", path = "../../", features = ["crypto", "host", "files"] } rcgen = "0.14" tokio = { version = "1", features = ["full"] } serde_json = { version = "1" } diff --git a/example/server/src/handlers.rs b/example/server/src/handlers.rs index 5a797d3..789cd4a 100644 --- a/example/server/src/handlers.rs +++ b/example/server/src/handlers.rs @@ -1,7 +1,5 @@ use mtp::codec::{CommunicationType, CommunicationValue, DataType, DataTypeId, DataValue, TypeMap}; -use mtp::crypto::{ - CryptoError, Keyring, SignaturePublicKey, SignatureScheme, verify_ed25519, -}; +use mtp::crypto::{CryptoError, Keyring, SignaturePublicKey, SignatureScheme, verify_ed25519}; struct Ed25519Verifier(SignaturePublicKey); @@ -57,7 +55,10 @@ pub fn process_and_respond( let enc = msg.get_data(DataType::EncryptedPayload); if matches!(enc, DataValue::EncryptedContainer(_)) { let mut dv = enc.clone(); - if dv.decrypt_into_container(host_keyring, b"demo-aad").is_some() { + if dv + .decrypt_into_container(host_keyring, b"demo-aad") + .is_some() + { if let Some(entries) = dv.as_container() { println!(" Decrypted EncryptedPayload: {:?}", entries); enc_status = format!("EncryptedPayload decrypted OK ({} entries)", entries.len()); @@ -93,7 +94,9 @@ pub fn process_and_respond( if let Some(pk_bundle) = client_pk { let verifier = Ed25519Verifier(pk_bundle.sig_cl_public_key.clone()); let mut dv = secure.clone(); - if dv.decrypt_signed_encrypted_container(host_keyring, b"demo-aad").is_some() + if dv + .decrypt_signed_encrypted_container(host_keyring, b"demo-aad") + .is_some() && dv.verify_into_container(&verifier).is_some() { if let Some(entries) = dv.as_container() { diff --git a/example/server/src/keys.rs b/example/server/src/keys.rs index 133eeba..1cf391f 100644 --- a/example/server/src/keys.rs +++ b/example/server/src/keys.rs @@ -1,50 +1,33 @@ use std::fs; -use mtp::crypto::kem::HybridKem; -use mtp::crypto::{Ed25519Signer, Keyring, MlDsaSigner}; +use mtp::crypto::Keyring; +use mtp::files::{load_keyring, save_keyring, save_public_key_bundle}; + +/* Host id is fixed for the example; only the keyring itself is persisted. */ +const HOST_ID: u64 = 1; pub fn load_or_generate_host_keys( - path: &str, + keyring_path: &str, ) -> Result<(u64, Keyring), Box> { - if let Ok(data) = fs::read_to_string(path) { - let json: serde_json::Value = serde_json::from_str(&data)?; - let hid = json["host_id"].as_u64().unwrap_or(1); - let keyring = Keyring::from_bytes(&hex::decode(json["keyring"].as_str().unwrap())?)?; - println!("Loaded host keys (ID: {})", hid); - return Ok((hid, keyring)); + if let Ok(keyring) = load_keyring(keyring_path) { + println!("Loaded host keyring from {keyring_path}"); + return Ok((HOST_ID, keyring)); } - let (_ed_signer, sig_sk, sig_pk) = Ed25519Signer::generate(); - let (_pq_signer, sig_pq_sk, sig_pq_pk) = MlDsaSigner::generate(); - let (kem_sk, kem_pk) = HybridKem::generate_keypair(); - let keyring = Keyring::new(kem_pk, kem_sk, sig_pq_pk, sig_pq_sk, sig_pk, sig_sk); - - let json = serde_json::json!({ - "host_id": 1, - "keyring": hex::encode(keyring.to_bytes()), - }); - fs::write(path, serde_json::to_string_pretty(&json)?)?; - println!("Generated host keys -> {path}"); - Ok((1u64, keyring)) + let keyring = Keyring::generate(); + save_keyring(&keyring, keyring_path)?; + println!("Generated host keyring -> {keyring_path}"); + Ok((HOST_ID, keyring)) } pub fn export_host_public_keys(host_keyring: &Keyring) -> Result<(), Box> { - let public_key_bundle_hex = hex::encode(host_keyring.public_key_bundle().as_bytes()); + let bundle = host_keyring.public_key_bundle(); + save_public_key_bundle(&bundle, "host.mpkb")?; - fs::write("host_public_key_bundle.hex", &public_key_bundle_hex)?; + /* The web client fetches the bundle as hex over HTTP. */ + let bundle_hex = hex::encode(bundle.as_bytes()); + fs::write("host_public_key_bundle.hex", &bundle_hex)?; fs::create_dir_all("web-client/public")?; - fs::write( - "web-client/public/host_public_key_bundle.hex", - &public_key_bundle_hex, - )?; - fs::write( - "host_enc_kem_pk.bin", - host_keyring.kem_public_key.as_bytes(), - )?; - fs::write("host_sig_pk.bin", host_keyring.sig_cl_public_key.as_bytes())?; - fs::write( - "host_sig_pq_pk.bin", - host_keyring.sig_pq_public_key.as_bytes(), - )?; + fs::write("web-client/public/host_public_key_bundle.hex", &bundle_hex)?; Ok(()) } diff --git a/example/server/src/main.rs b/example/server/src/main.rs index 776b2b6..86aafdc 100644 --- a/example/server/src/main.rs +++ b/example/server/src/main.rs @@ -36,7 +36,7 @@ async fn main() -> Result<(), Box> { tls::export_webtransport_cert_hash(&cert_hash)?; println!("WebTransport certificate sha256: {cert_hash}"); - let (_host_id, host_keyring) = keys::load_or_generate_host_keys("host_keys.json")?; + let (_host_id, host_keyring) = keys::load_or_generate_host_keys("host.mk")?; keys::export_host_public_keys(&host_keyring)?; // The keyring is moved into the host config; keep a copy for decrypting the @@ -47,7 +47,7 @@ async fn main() -> Result<(), Box> { let (clients, next_id) = clients::load_client_db("clients.json")?; let clients_for_get = clients.clone(); - let get_existing_user = move |id: u64| { + let get_existing_user = move |id: u64, _description: Option| { let clients = clients_for_get.clone(); Box::pin(async move { let result = clients.lock().unwrap().get(&id).cloned(); @@ -63,7 +63,8 @@ async fn main() -> Result<(), Box> { let clients_for_register = clients.clone(); let next_id_for_register = next_id.clone(); let clients_path = "clients.json".to_string(); - let complete_register = move |bundle: mtp::crypto::PublicKeyBundle| { + let complete_register = move |bundle: mtp::crypto::PublicKeyBundle, + _description: Option| { let db_arc = clients_for_register.clone(); let nid_arc = next_id_for_register.clone(); let path = clients_path.clone(); @@ -95,7 +96,11 @@ async fn main() -> Result<(), Box> { cert_pem, key_pem, ) - .with_authentication(host_keyring, get_existing_user, complete_register) + .with_authentication( + host_keyring, + Box::new(get_existing_user), + Box::new(complete_register), + ) .with_authentication_policy(AuthenticationPolicy::ForceAuthentication); let mut host = MTPHost::new(config).await?; diff --git a/files/Cargo.toml b/files/Cargo.toml new file mode 100644 index 0000000..2ea9f40 --- /dev/null +++ b/files/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mtp-files" +version = "0.1.0" +edition = "2024" + +[dependencies] +# Only the plain key types (`Keyring`, `PublicKeyBundle`, `CryptoError`) are +# needed here; those are always compiled, so no crypto features are required. +mtp-crypto = { version = "0.1.0", path = "../crypto", default-features = false } +thiserror = "1" diff --git a/files/src/lib.rs b/files/src/lib.rs new file mode 100644 index 0000000..1e40f26 --- /dev/null +++ b/files/src/lib.rs @@ -0,0 +1,197 @@ +/* + * On-disk storage for methanium key material. + * + * `.mk` files hold a full Keyring (public and secret keys) and are written with + * owner-only permissions (0600) on Unix. `.mpkb` files hold a PublicKeyBundle + * (public keys only) and are safe to share. Each file opens with a 4-byte magic + * that doubles as a type tag, so a bundle never loads as a keyring, followed by + * a version byte for future format changes. + */ + +use std::fs; +use std::io; +use std::path::Path; + +use thiserror::Error; + +pub use mtp_crypto::{CryptoError, Keyring, PublicKeyBundle}; + +/// File extension for a stored [`Keyring`]. +pub const KEYRING_EXTENSION: &str = "mk"; +/// File extension for a stored [`PublicKeyBundle`]. +pub const BUNDLE_EXTENSION: &str = "mpkb"; + +/* Container layout: magic (4 bytes) || version (1 byte) || payload. */ +const KEYRING_MAGIC: [u8; 4] = *b"MTMK"; /* Methanium Keyring */ +const BUNDLE_MAGIC: [u8; 4] = *b"MPKB"; /* Methanium Public Key Bundle */ +const FORMAT_VERSION: u8 = 1; +const HEADER_LEN: usize = 4 + 1; + +#[derive(Error, Debug)] +pub enum FileError { + #[error("io error: {0}")] + Io(#[from] io::Error), + #[error("crypto error: {0}")] + Crypto(#[from] CryptoError), + #[error("not a valid methanium {expected} file (bad magic)")] + BadMagic { expected: &'static str }, + #[error("unsupported {kind} format version {found} (expected {FORMAT_VERSION})")] + UnsupportedVersion { kind: &'static str, found: u8 }, + #[error("file is truncated: {0} bytes, need at least {HEADER_LEN}")] + Truncated(usize), +} + +fn encode(magic: [u8; 4], payload: &[u8]) -> Vec { + let mut out = Vec::with_capacity(HEADER_LEN + payload.len()); + out.extend_from_slice(&magic); + out.push(FORMAT_VERSION); + out.extend_from_slice(payload); + out +} + +fn decode<'a>(bytes: &'a [u8], magic: [u8; 4], kind: &'static str) -> Result<&'a [u8], FileError> { + if bytes.len() < HEADER_LEN { + return Err(FileError::Truncated(bytes.len())); + } + if bytes[..4] != magic { + return Err(FileError::BadMagic { expected: kind }); + } + let found = bytes[4]; + if found != FORMAT_VERSION { + return Err(FileError::UnsupportedVersion { kind, found }); + } + Ok(&bytes[HEADER_LEN..]) +} + +/* + * `OpenOptions::mode` only applies when the file is created, so the mode is + * re-set afterwards to also tighten a pre-existing, more-permissive file. + */ +#[cfg(unix)] +fn write_secret(path: &Path, bytes: &[u8]) -> io::Result<()> { + use std::io::Write; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path)?; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + file.write_all(bytes)?; + file.sync_all() +} + +#[cfg(not(unix))] +fn write_secret(path: &Path, bytes: &[u8]) -> io::Result<()> { + fs::write(path, bytes) +} + +// Writes secret keys, so the file is created owner-only (0600) on Unix. +pub fn save_keyring(keyring: &Keyring, path: impl AsRef) -> Result<(), FileError> { + let bytes = encode(KEYRING_MAGIC, &keyring.to_bytes()); + write_secret(path.as_ref(), &bytes)?; + Ok(()) +} + +pub fn load_keyring(path: impl AsRef) -> Result { + let bytes = fs::read(path)?; + let payload = decode(&bytes, KEYRING_MAGIC, "keyring")?; + Ok(Keyring::from_bytes(payload)?) +} + +pub fn save_public_key_bundle( + bundle: &PublicKeyBundle, + path: impl AsRef, +) -> Result<(), FileError> { + let bytes = encode(BUNDLE_MAGIC, &bundle.as_bytes()); + fs::write(path, bytes)?; + Ok(()) +} + +pub fn load_public_key_bundle(path: impl AsRef) -> Result { + let bytes = fs::read(path)?; + let payload = decode(&bytes, BUNDLE_MAGIC, "public key bundle")?; + Ok(PublicKeyBundle::from_bytes(payload)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use mtp_crypto::keypair::{ + KemPrivateKey, KemPublicKey, SignaturePqPrivateKey, SignaturePqPublicKey, + SignaturePrivateKey, SignaturePublicKey, + }; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU32, Ordering}; + + fn temp_path(ext: &str) -> PathBuf { + static COUNTER: AtomicU32 = AtomicU32::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let mut path = std::env::temp_dir(); + path.push(format!("mtp-files-test-{}-{n}.{ext}", std::process::id())); + path + } + + fn sample_keyring() -> Keyring { + Keyring::new( + KemPublicKey::new(vec![1u8; 32]), + KemPrivateKey::new(vec![2u8; 32]), + SignaturePqPublicKey::new(vec![3u8; 64]), + SignaturePqPrivateKey::new(vec![4u8; 64]), + SignaturePublicKey::new(vec![5u8; 32]), + SignaturePrivateKey::new(vec![6u8; 32]), + ) + } + + #[test] + fn keyring_save_load_roundtrip() { + let path = temp_path(KEYRING_EXTENSION); + let keyring = sample_keyring(); + save_keyring(&keyring, &path).unwrap(); + let loaded = load_keyring(&path).unwrap(); + assert_eq!(keyring.to_bytes(), loaded.to_bytes()); + let _ = fs::remove_file(&path); + } + + #[test] + fn bundle_save_load_roundtrip() { + let path = temp_path(BUNDLE_EXTENSION); + let bundle = sample_keyring().public_key_bundle(); + save_public_key_bundle(&bundle, &path).unwrap(); + let loaded = load_public_key_bundle(&path).unwrap(); + assert_eq!(bundle.as_bytes(), loaded.as_bytes()); + let _ = fs::remove_file(&path); + } + + #[test] + fn loading_bundle_as_keyring_fails_on_magic() { + let path = temp_path(BUNDLE_EXTENSION); + save_public_key_bundle(&sample_keyring().public_key_bundle(), &path).unwrap(); + assert!(matches!( + load_keyring(&path), + Err(FileError::BadMagic { .. }) + )); + let _ = fs::remove_file(&path); + } + + #[test] + fn truncated_file_is_rejected() { + let path = temp_path(KEYRING_EXTENSION); + fs::write(&path, b"MT").unwrap(); + assert!(matches!(load_keyring(&path), Err(FileError::Truncated(2)))); + let _ = fs::remove_file(&path); + } + + #[cfg(unix)] + #[test] + fn keyring_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let path = temp_path(KEYRING_EXTENSION); + save_keyring(&sample_keyring(), &path).unwrap(); + let mode = fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + let _ = fs::remove_file(&path); + } +} diff --git a/src/lib.rs b/src/lib.rs index 03314f0..c8ec1b3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,3 +10,6 @@ pub use mtp_host as host; #[cfg(feature = "client")] pub use mtp_client as client; + +#[cfg(feature = "files")] +pub use mtp_files as files; From 0679a241c210940927a51b04e3c098471f3b5edd Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:00:23 +0200 Subject: [PATCH 13/97] Client Policy --- client/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index a7b55d3..008f77d 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -29,6 +29,7 @@ pub struct ClientConfig { pub tls: ClientTlsConfig, pub client_id: u64, pub description: Option, + pub policy: Policy, #[cfg(feature = "crypto")] pub auth_timeout: Duration, } @@ -46,6 +47,7 @@ impl ClientConfig { tls: ClientTlsConfig::SystemRoots, client_id: 0, description: None, + policy: Policy::default(), #[cfg(feature = "crypto")] auth_timeout: Duration::from_secs(30), } @@ -70,6 +72,11 @@ impl ClientConfig { self } + pub fn with_policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + #[cfg(feature = "crypto")] pub fn with_auth_timeout(mut self, timeout: Duration) -> Self { self.auth_timeout = timeout; @@ -391,7 +398,7 @@ impl MTPClient { use mtp_crypto::auth; let (sender, receiver) = - mtp_transport::connect(&config.url, config.server_cert(), Policy::default()).await?; + mtp_transport::connect(&config.url, config.server_cert(), config.policy).await?; let tm = mtp_codec::TypeMap::latest(); let version_str = format!("{}", PROTOCOL_VERSION); @@ -533,7 +540,7 @@ impl MTPClient { use mtp_crypto::auth; let (sender, receiver) = - mtp_transport::connect(&config.url, config.server_cert(), Policy::default()).await?; + mtp_transport::connect(&config.url, config.server_cert(), config.policy).await?; let tm = mtp_codec::TypeMap::latest(); let version_str = format!("{}", PROTOCOL_VERSION); From b96c072a0f87de7828e45cf3dcd44aef4a9e459d Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:37:06 +0200 Subject: [PATCH 14/97] keynames --- example/keygen/src/main.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/example/keygen/src/main.rs b/example/keygen/src/main.rs index bbaa1e3..3b28c78 100644 --- a/example/keygen/src/main.rs +++ b/example/keygen/src/main.rs @@ -7,11 +7,8 @@ use mtp::files::{ }; fn main() -> Result<(), files::FileError> { - let prefix = std::env::args() - .nth(1) - .unwrap_or_else(|| "keyring".to_string()); - let keyring_path = PathBuf::from(format!("{prefix}.{KEYRING_EXTENSION}")); - let bundle_path = PathBuf::from(format!("{prefix}.{BUNDLE_EXTENSION}")); + let keyring_path = PathBuf::from(format!("keyring.{KEYRING_EXTENSION}")); + let bundle_path = PathBuf::from(format!("bundle.{BUNDLE_EXTENSION}")); let keyring = Keyring::generate(); save_keyring(&keyring, &keyring_path)?; From 391f92c9c5e21ad8b5414da593512d6b0d7bd3c3 Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 02:52:30 +0200 Subject: [PATCH 15/97] (fix): STOP_SENDING && git push --- transport/src/connection.rs | 200 +++++++++++++++++++++++++----------- wasm/src/transport.rs | 66 +++++++++++- 2 files changed, 200 insertions(+), 66 deletions(-) diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 9c59039..653a822 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -129,19 +129,30 @@ impl Sender { return Err(CommunicationError::MessageTooLarge); } - use tokio::io::AsyncWriteExt; + let len_bytes = (bytes.len() as u32).to_be_bytes(); + let write_result = async { + stream.write_all(&len_bytes).await?; + stream.write_all(&bytes).await?; + Ok::<(), wtransport::error::StreamWriteError>(()) + }; - timeout(policy.write_timeout, stream.write_u32(bytes.len() as u32)) - .await - .map_err(|_| CommunicationError::StreamError)? - .map_err(|_| CommunicationError::StreamError)?; - - timeout(policy.write_timeout, stream.write_all(&bytes)) - .await - .map_err(|_| CommunicationError::StreamError)? - .map_err(CommunicationError::from)?; - - Ok(()) + match timeout(policy.write_timeout, write_result).await { + Ok(Ok(())) => Ok(()), + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { + log::warn!( + "[Sender] write failed: peer sent STOP_SENDING (error code {code})" + ); + Err(CommunicationError::StreamClosed) + } + Ok(Err(other)) => { + log::warn!("[Sender] write failed: {other}"); + Err(CommunicationError::StreamError) + } + Err(_) => { + log::warn!("[Sender] write timed out (len={})", bytes.len()); + Err(CommunicationError::StreamError) + } + } } fn normalize_send_error(error: CommunicationError) -> CommunicationError { @@ -227,12 +238,23 @@ impl Sender { let mut stream = Self::open_uni_stream(conn, policy).await?; Self::write_frame(&mut stream, data, policy).await?; - timeout(policy.write_timeout, stream.finish()) - .await - .map_err(|_| CommunicationError::StreamError)? - .map_err(|_| CommunicationError::StreamError)?; - - Ok(()) + match timeout(policy.write_timeout, stream.finish()).await { + Ok(Ok(())) => Ok(()), + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { + log::warn!( + "[Sender] finish failed: peer sent STOP_SENDING (error code {code})" + ); + Err(CommunicationError::StreamClosed) + } + Ok(Err(other)) => { + log::warn!("[Sender] finish failed: {other}"); + Err(CommunicationError::StreamError) + } + Err(_) => { + log::warn!("[Sender] finish timed out"); + Err(CommunicationError::StreamError) + } + } } async fn send_close_frame( conn: &Connection, @@ -240,20 +262,35 @@ impl Sender { ) -> Result<(), CommunicationError> { let mut stream = Self::open_uni_stream(conn, policy).await?; - use tokio::io::AsyncWriteExt; - timeout( - policy.write_timeout, - stream.write_u32(policy.close_frame_len), - ) - .await - .map_err(|_| CommunicationError::StreamError)? - .map_err(|_| CommunicationError::StreamError)?; + let len_bytes = policy.close_frame_len.to_be_bytes(); + match timeout(policy.write_timeout, stream.write_all(&len_bytes)).await { + Ok(Ok(())) => {} + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { + log::warn!( + "[Sender] close frame write failed: peer sent STOP_SENDING (error code {code})" + ); + } + Ok(Err(other)) => { + log::warn!("[Sender] close frame write failed: {other}"); + } + Err(_) => { + log::warn!("[Sender] close frame write timed out"); + } + } - if let Err(e) = timeout(policy.write_timeout, stream.finish()) - .await - .map_err(|_| CommunicationError::StreamError)? - { - log::warn!("[Sender] close frame finish failed: {e}"); + match timeout(policy.write_timeout, stream.finish()).await { + Ok(Ok(())) => {} + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { + log::warn!( + "[Sender] close frame finish failed: peer sent STOP_SENDING (error code {code})" + ); + } + Ok(Err(other)) => { + log::warn!("[Sender] close frame finish failed: {other}"); + } + Err(_) => { + log::warn!("[Sender] close frame finish timed out"); + } } Ok(()) @@ -318,10 +355,23 @@ impl Sender { let _send_lock = self.send_guard.lock().await; let mut stream_opt = self.stream_guard.lock().await; if let Some(mut stream) = stream_opt.take() { - timeout(self.policy.write_timeout, stream.finish()) - .await - .map_err(|_| CommunicationError::StreamError)? - .map_err(|_| CommunicationError::StreamError)?; + match timeout(self.policy.write_timeout, stream.finish()).await { + Ok(Ok(())) => {} + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { + log::warn!( + "[Sender] finish_stream: peer sent STOP_SENDING (error code {code})" + ); + return Err(CommunicationError::StreamClosed); + } + Ok(Err(other)) => { + log::warn!("[Sender] finish_stream failed: {other}"); + return Err(CommunicationError::StreamError); + } + Err(_) => { + log::warn!("[Sender] finish_stream timed out"); + return Err(CommunicationError::StreamError); + } + } } Ok(()) } @@ -345,7 +395,12 @@ impl Sender { if let Some(mut stream) = stream_guard.lock().await.take() { match timeout(policy.write_timeout, stream.finish()).await { Ok(Ok(())) => {} - Ok(Err(e)) => log::warn!("[Sender] persistent stream finish failed: {e}"), + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => log::warn!( + "[Sender] persistent stream finish failed: peer sent STOP_SENDING (error code {code})" + ), + Ok(Err(e)) => { + log::warn!("[Sender] persistent stream finish failed: {e}") + } Err(_) => log::warn!("[Sender] persistent stream finish timed out"), } } @@ -500,46 +555,67 @@ impl Receiver { stream: &mut wtransport::RecvStream, policy: &Policy, ) -> Result { - use std::io::ErrorKind; - use tokio::io::AsyncReadExt; + use wtransport::error::{StreamReadError, StreamReadExactError}; - let mut attempts = 0; - let len = loop { - match stream.read_u32().await { - Ok(len) => break len, - Err(e) => { - if e.kind() == ErrorKind::Interrupted && attempts < 3 { - attempts += 1; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - continue; - } - if e.kind() == ErrorKind::UnexpectedEof { - return Ok(ReceivedFrame::Idle); - } - log::warn!("[Receiver] read_u32 failed: {e}"); - return Err(CommunicationError::StreamError); - } + let mut len_buf = [0u8; 4]; + match timeout(policy.read_timeout, stream.read_exact(&mut len_buf)).await { + Ok(Ok(())) => {} + Ok(Err(StreamReadExactError::FinishedEarly(0))) => { + return Ok(ReceivedFrame::Idle); } - }; + Ok(Err(StreamReadExactError::FinishedEarly(n))) => { + log::warn!( + "[Receiver] length-prefix read ended early ({n}/4 bytes): stream closed by peer" + ); + return Err(CommunicationError::StreamError); + } + Ok(Err(StreamReadExactError::Read(StreamReadError::Reset(code)))) => { + log::warn!( + "[Receiver] length-prefix read failed: peer sent RESET_STREAM (error code {code})" + ); + return Err(CommunicationError::StreamError); + } + Ok(Err(other)) => { + log::warn!("[Receiver] length-prefix read failed: {other}"); + return Err(CommunicationError::StreamError); + } + Err(_) => { + log::warn!("[Receiver] length-prefix read timed out"); + return Err(CommunicationError::StreamError); + } + } + let len = u32::from_be_bytes(len_buf); if len == policy.close_frame_len { return Ok(ReceivedFrame::ClosedByPeer); } - let len = len as usize; + let len_usize = len as usize; if len as u64 > policy.max_message_size { return Err(CommunicationError::MessageTooLarge); } - let mut buf = vec![0u8; len]; + let mut buf = vec![0u8; len_usize]; match timeout(policy.read_timeout, stream.read_exact(&mut buf)).await { Ok(Ok(())) => {} - Ok(Err(e)) => match timeout(policy.read_timeout, stream.read_exact(&mut buf)).await { - Ok(Ok(())) => {} - _ => return Err(e.into()), - }, + Ok(Err(StreamReadExactError::FinishedEarly(n))) => { + log::warn!( + "[Receiver] body read ended early ({n}/{len_usize} bytes): stream closed by peer" + ); + return Err(CommunicationError::StreamError); + } + Ok(Err(StreamReadExactError::Read(StreamReadError::Reset(code)))) => { + log::warn!( + "[Receiver] body read failed: peer sent RESET_STREAM (error code {code})" + ); + return Err(CommunicationError::StreamError); + } + Ok(Err(other)) => { + log::warn!("[Receiver] body read failed: {other}"); + return Err(CommunicationError::StreamError); + } Err(_) => { - log::warn!("[Receiver] read_exact timed out (len={})", len); + log::warn!("[Receiver] body read timed out (len={len_usize})"); return Err(CommunicationError::StreamError); } } diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index 1bdf39c..4708b47 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -10,6 +10,52 @@ use crate::frame::parse_frame_value; const CLOSE_FRAME_LEN: u32 = u32::MAX; +/// Inspect a JS error value for a WebTransport stream error and log the +/// `streamErrorCode` carried by STOP_SENDING / RESET_STREAM. +/// +/// The browser's WebTransport API rejects write/close/read promises with a +/// `WebTransportError` whose `source` is `"stream"` and whose +/// `streamErrorCode` is the application error code from the peer's +/// STOP_SENDING (for send streams) or RESET_STREAM (for receive streams). +/// Per draft-ietf-webtrans-http3-15 §4.4, a WebTransport application MUST +/// provide an error code for those operations, so it is always present on +/// stream-level errors. +fn log_webtransport_error(error: &JsValue, context: &str) { + let source = js_sys::Reflect::get(error, &JsValue::from_str("source")) + .ok() + .and_then(|v| v.as_string()); + let stream_error_code = js_sys::Reflect::get(error, &JsValue::from_str("streamErrorCode")) + .ok() + .and_then(|v| v.as_f64()); + let message = error + .as_string() + .or_else(|| { + js_sys::Reflect::get(error, &JsValue::from_str("message")) + .ok() + .and_then(|v| v.as_string()) + }) + .unwrap_or_else(|| format!("{:?}", error)); + + let formatted = match (&source, stream_error_code) { + (Some(src), Some(code)) => format!( + "[WasmTransport] {context}: WebTransportError source={src} \ + streamErrorCode={code} ({message})" + ), + (Some(src), None) => { + format!("[WasmTransport] {context}: WebTransportError source={src} ({message})") + } + (None, _) => format!("[WasmTransport] {context}: {message}"), + }; + + if let Ok(console) = js_sys::Reflect::get(&js_sys::global(), &JsValue::from_str("console")) { + if let Ok(warn) = js_sys::Reflect::get(&console, &JsValue::from_str("warn")) + .and_then(|f| f.dyn_into::()) + { + let _ = warn.call1(&console, &JsValue::from_str(&formatted)); + } + } +} + /// Given a `SendStream` (old API with `.writable` or new API where stream IS a WritableStream), /// return the object to call `.getWriter()` on. fn resolve_stream_writable(send_stream: &JsValue) -> Result { @@ -169,7 +215,10 @@ impl WasmTransport { let write_promise = write_fn .call1(&writer_val, &chunk) .map_err(|e| js_error(&format!("write failed: {:?}", e)))?; - JsFuture::from(write_promise.unchecked_into::()).await?; + if let Err(e) = JsFuture::from(write_promise.unchecked_into::()).await { + log_webtransport_error(&e, "send_frame write"); + return Err(e); + } let close_fn = js_sys::Reflect::get(&writer_val, &JsValue::from_str("close")) .map_err(|_| js_error("missing close"))? @@ -178,7 +227,10 @@ impl WasmTransport { let close_promise = close_fn .call0(&writer_val) .map_err(|e| js_error(&format!("close failed: {:?}", e)))?; - JsFuture::from(close_promise.unchecked_into::()).await?; + if let Err(e) = JsFuture::from(close_promise.unchecked_into::()).await { + log_webtransport_error(&e, "send_frame close"); + return Err(e); + } Ok(()) } @@ -218,7 +270,10 @@ impl WasmTransport { .unchecked_into::(), ) .await - .map_err(|e| js_error(&format!("accept stream failed: {:?}", e)))?; + .map_err(|e| { + log_webtransport_error(&e, "open_next_stream accept"); + js_error(&format!("accept stream failed: {:?}", e)) + })?; let done = js_sys::Reflect::get(&result, &JsValue::from_str("done")) .ok() @@ -260,7 +315,10 @@ impl WasmTransport { .unchecked_into::(), ) .await - .map_err(|e| js_error(&format!("read failed: {:?}", e)))?; + .map_err(|e| { + log_webtransport_error(&e, "read_chunk"); + js_error(&format!("read failed: {:?}", e)) + })?; let done = js_sys::Reflect::get(&result, &JsValue::from_str("done")) .ok() From 870b8002d28a985babfd760f9d68ec1aff2f84cc Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 03:04:16 +0200 Subject: [PATCH 16/97] (fix): formatting --- transport/src/connection.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 653a822..b379008 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -139,9 +139,7 @@ impl Sender { match timeout(policy.write_timeout, write_result).await { Ok(Ok(())) => Ok(()), Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { - log::warn!( - "[Sender] write failed: peer sent STOP_SENDING (error code {code})" - ); + log::warn!("[Sender] write failed: peer sent STOP_SENDING (error code {code})"); Err(CommunicationError::StreamClosed) } Ok(Err(other)) => { @@ -241,9 +239,7 @@ impl Sender { match timeout(policy.write_timeout, stream.finish()).await { Ok(Ok(())) => Ok(()), Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => { - log::warn!( - "[Sender] finish failed: peer sent STOP_SENDING (error code {code})" - ); + log::warn!("[Sender] finish failed: peer sent STOP_SENDING (error code {code})"); Err(CommunicationError::StreamClosed) } Ok(Err(other)) => { @@ -256,6 +252,7 @@ impl Sender { } } } + async fn send_close_frame( conn: &Connection, policy: &Policy, From 4d75f23a171d4c2ea0fd97544f9f1370b4a6ce8b Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 12:42:18 +0200 Subject: [PATCH 17/97] (fix): builds --- transport/src/connection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/src/connection.rs b/transport/src/connection.rs index b379008..8a88fc3 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -695,7 +695,7 @@ mod tests { #[test] fn test_policy_clone() { let p = Policy::default(); - let cloned = p.clone(); + let cloned = p; assert_eq!(p.send_mode, cloned.send_mode); } From a4447ebce8f6696a2b59931f72fd065345533e6d Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 12:47:46 +0200 Subject: [PATCH 18/97] (fix): builds --- Cargo.lock | 1 - Cargo.toml | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 853e843..1a87082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -985,7 +985,6 @@ dependencies = [ "mtp-crypto", "mtp-files", "mtp-host", - "mtp-transport", "mtp-type-map", "rand 0.8.6", "rcgen", diff --git a/Cargo.toml b/Cargo.toml index 275a012..daac4c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,8 +51,6 @@ edition = "2024" mtp-common = { version = "0.1.0", path = "common" } mtp-type-map = { version = "0.1.0", path = "type-map" } mtp-codec = { version = "0.1.0", path = "codec" } -mtp-transport = { version = "0.1.0", path = "transport" } - # --- optional, behind features --- mtp-crypto = { version = "0.1.0", path = "crypto", optional = true, features = [ "serde", @@ -78,7 +76,7 @@ crypto = [ ] # MTP server host - version negotiation, Registry, incoming QUIC connections. -host = ["dep:mtp-host", "mtp-codec/registry", "mtp-transport/host"] +host = ["dep:mtp-host", "mtp-codec/registry"] # MTP client - outgoing QUIC connections to a host. client = ["dep:mtp-client"] From 0bb3f607c7377c754a0da5d2ac1f3dc56ac8279d Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 13:33:13 +0200 Subject: [PATCH 19/97] (fix): STOP_SENDING --- example/Cargo.lock | 1 - wasm/src/transport.rs | 106 +++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/example/Cargo.lock b/example/Cargo.lock index 0696b49..a5b5094 100644 --- a/example/Cargo.lock +++ b/example/Cargo.lock @@ -902,7 +902,6 @@ dependencies = [ "mtp-crypto", "mtp-files", "mtp-host", - "mtp-transport", "mtp-type-map", ] diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index 4708b47..cb46238 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -10,20 +10,25 @@ use crate::frame::parse_frame_value; const CLOSE_FRAME_LEN: u32 = u32::MAX; -/// Inspect a JS error value for a WebTransport stream error and log the -/// `streamErrorCode` carried by STOP_SENDING / RESET_STREAM. +/// Inspect a JS error value for a WebTransport **stream-level** error and, if +/// present, log the `streamErrorCode` carried by STOP_SENDING / RESET_STREAM. /// -/// The browser's WebTransport API rejects write/close/read promises with a -/// `WebTransportError` whose `source` is `"stream"` and whose -/// `streamErrorCode` is the application error code from the peer's -/// STOP_SENDING (for send streams) or RESET_STREAM (for receive streams). /// Per draft-ietf-webtrans-http3-15 §4.4, a WebTransport application MUST -/// provide an error code for those operations, so it is always present on -/// stream-level errors. -fn log_webtransport_error(error: &JsValue, context: &str) { +/// provide an error code for those operations. The browser surfaces these as +/// `WebTransportError` with `source = "stream"` and a numeric `streamErrorCode`. +/// +/// Session-level errors (`source = "session"`) are normal connection +/// closures and are **not** logged here — they propagate to `on_error` +/// in the receive loop like any other transport error. +fn log_stream_error_code(error: &JsValue, context: &str) { let source = js_sys::Reflect::get(error, &JsValue::from_str("source")) .ok() .and_then(|v| v.as_string()); + + if source.as_deref() != Some("stream") { + return; + } + let stream_error_code = js_sys::Reflect::get(error, &JsValue::from_str("streamErrorCode")) .ok() .and_then(|v| v.as_f64()); @@ -36,15 +41,12 @@ fn log_webtransport_error(error: &JsValue, context: &str) { }) .unwrap_or_else(|| format!("{:?}", error)); - let formatted = match (&source, stream_error_code) { - (Some(src), Some(code)) => format!( - "[WasmTransport] {context}: WebTransportError source={src} \ - streamErrorCode={code} ({message})" + let formatted = match stream_error_code { + Some(code) => format!( + "[WasmTransport] {context}: STOP_SENDING/RESET_STREAM streamErrorCode={code} \ + ({message})" ), - (Some(src), None) => { - format!("[WasmTransport] {context}: WebTransportError source={src} ({message})") - } - (None, _) => format!("[WasmTransport] {context}: {message}"), + None => format!("[WasmTransport] {context}: stream error ({message})"), }; if let Ok(console) = js_sys::Reflect::get(&js_sys::global(), &JsValue::from_str("console")) { @@ -76,6 +78,18 @@ fn resolve_stream_readable(recv_stream: &JsValue) -> Result { } } +/// Release a `WritableStreamDefaultWriter`'s lock on its stream. Called after +/// `writer.close()` (or on write failure) so the runtime does not interpret an +/// abandoned locked writer as an abort, which would surface as STOP_SENDING to +/// the peer. Errors are ignored — `releaseLock` is best-effort cleanup. +fn release_writer_lock(writer: &JsValue) { + if let Ok(release) = js_sys::Reflect::get(writer, &JsValue::from_str("releaseLock")) + .and_then(|f| f.dyn_into::().map_err(Into::into)) + { + let _ = release.call0(writer); + } +} + /// Outcome of reading the next framed message from the incoming stream(s). enum FrameOutcome { /// A complete application frame. @@ -216,7 +230,8 @@ impl WasmTransport { .call1(&writer_val, &chunk) .map_err(|e| js_error(&format!("write failed: {:?}", e)))?; if let Err(e) = JsFuture::from(write_promise.unchecked_into::()).await { - log_webtransport_error(&e, "send_frame write"); + log_stream_error_code(&e, "send_frame write"); + release_writer_lock(&writer_val); return Err(e); } @@ -228,10 +243,17 @@ impl WasmTransport { .call0(&writer_val) .map_err(|e| js_error(&format!("close failed: {:?}", e)))?; if let Err(e) = JsFuture::from(close_promise.unchecked_into::()).await { - log_webtransport_error(&e, "send_frame close"); - return Err(e); + // The write already succeeded; a STOP_SENDING on close just means + // the peer stopped reading before we could send FIN. The data is in + // flight, so this is not a send failure — log and return success. + log_stream_error_code(&e, "send_frame close"); } + // Always release the writer's lock on the WritableStream. Abandoning a + // locked writer (e.g. via drop) can be interpreted by the runtime as an + // abort, which may surface as STOP_SENDING to the peer. + release_writer_lock(&writer_val); + Ok(()) } @@ -263,17 +285,17 @@ impl WasmTransport { .map_err(|_| js_error("missing read"))? .dyn_into::() .map_err(|_| js_error("read not a function"))?; - let result = JsFuture::from( - read_fn - .call0(&streams_reader) - .map_err(|_| js_error("read call failed"))? - .unchecked_into::(), - ) - .await - .map_err(|e| { - log_webtransport_error(&e, "open_next_stream accept"); - js_error(&format!("accept stream failed: {:?}", e)) - })?; + let promise = read_fn + .call0(&streams_reader) + .map_err(|_| js_error("read call failed"))? + .unchecked_into::(); + let result = match JsFuture::from(promise).await { + Ok(r) => r, + Err(e) => { + log_stream_error_code(&e, "open_next_stream accept"); + return Err(js_error(&format!("accept stream failed: {:?}", e))); + } + }; let done = js_sys::Reflect::get(&result, &JsValue::from_str("done")) .ok() @@ -308,17 +330,17 @@ impl WasmTransport { .map_err(|_| js_error("missing read"))? .dyn_into::() .map_err(|_| js_error("read not a function"))?; - let result = JsFuture::from( - read_fn - .call0(&reader) - .map_err(|_| js_error("read call failed"))? - .unchecked_into::(), - ) - .await - .map_err(|e| { - log_webtransport_error(&e, "read_chunk"); - js_error(&format!("read failed: {:?}", e)) - })?; + let promise = read_fn + .call0(&reader) + .map_err(|_| js_error("read call failed"))? + .unchecked_into::(); + let result = match JsFuture::from(promise).await { + Ok(r) => r, + Err(e) => { + log_stream_error_code(&e, "read_chunk"); + return Err(js_error(&format!("read failed: {:?}", e))); + } + }; let done = js_sys::Reflect::get(&result, &JsValue::from_str("done")) .ok() From 92f1190b113d913c0c8ce9d0785460c2da8658b6 Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 13:35:45 +0200 Subject: [PATCH 20/97] (fix): STOP_SENDING in wasm --- wasm/src/transport.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index cb46238..9a8c69d 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -90,6 +90,19 @@ fn release_writer_lock(writer: &JsValue) { } } +/// Release a `ReadableStreamDefaultReader`'s lock on its stream. Mirrors +/// `release_writer_lock`: abandoning a locked reader can be interpreted by the +/// runtime as a `reader.cancel()` (sending STOP_SENDING to the peer) even on an +/// already-closed or errored stream. Calling `releaseLock` explicitly avoids +/// that. Errors are ignored — best-effort cleanup. +fn release_reader_lock(reader: &JsValue) { + if let Ok(release) = js_sys::Reflect::get(reader, &JsValue::from_str("releaseLock")) + .and_then(|f| f.dyn_into::().map_err(Into::into)) + { + let _ = release.call0(reader); + } +} + /// Outcome of reading the next framed message from the incoming stream(s). enum FrameOutcome { /// A complete application frame. @@ -408,7 +421,12 @@ impl WasmTransport { // Current stream finished; the next frame (if any) is on a // subsequent stream. Any trailing partial bytes are dropped // since the host never splits a frame across streams. - *self.stream_reader.borrow_mut() = None; + // Release the reader's lock explicitly so the runtime does + // not treat the abandoned lock as a cancel (which would send + // STOP_SENDING to the peer on an already-closed stream). + if let Some(reader) = self.stream_reader.borrow_mut().take() { + release_reader_lock(&reader); + } self.buffer.borrow_mut().clear(); } } @@ -452,6 +470,17 @@ impl WasmTransport { } pub fn close(&self) { + // Release any held reader locks before tearing down the session, so the + // runtime does not interpret an abandoned locked reader as a cancel + // (which would send STOP_SENDING to the peer). Once the locks are + // released the underlying streams can be torn down cleanly. + if let Some(reader) = self.stream_reader.borrow_mut().take() { + release_reader_lock(&reader); + } + if let Some(reader) = self.streams_reader.borrow_mut().take() { + release_reader_lock(&reader); + } + if let Ok(close) = js_sys::Reflect::get(&self.inner, &JsValue::from_str("close")) .and_then(|value| value.dyn_into::().map_err(Into::into)) { From c1761aae2bff7f7b6495292e5646485e380d371a Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 14:06:03 +0200 Subject: [PATCH 21/97] (feat): improve wasm/ts-sdk logger (fix): wasm rust runtime error (qol): update comments --- src/sdk/index.ts | 24 ++++++++++++++++++++---- wasm/src/client.rs | 36 +++++++++++++++++++++++++++++++++--- wasm/src/transport.rs | 42 +++++++----------------------------------- 3 files changed, 60 insertions(+), 42 deletions(-) diff --git a/src/sdk/index.ts b/src/sdk/index.ts index eabcc8a..9cd87e9 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -19,8 +19,8 @@ export interface MTPCredentialStorage { export type MTPStorage = MTPCredentialStorage; export type MTPLogEvent = - | { hint: "info" | "warning"; type: string; data: unknown } - | { hint: "error"; type: string | "error"; error: string; data?: unknown }; + | { hint: "info" | "warning"; type: string; data: unknown; direction?: "send" | "recv" } + | { hint: "error"; type: string | "error"; error: string; data?: unknown; direction?: "send" | "recv" }; export type ParsedFrame = RawBindings.ParsedFrame; @@ -605,13 +605,14 @@ export class MTPClient { try { const frame = this.raw.bindings.parse_frame(message); emit(this.#options.logger, isErrorType(frame.type) - ? { hint: "error", type: frame.type, error: errorMessage(frame), data: frame.data } - : { hint: "info", type: frame.type, data: frame.data }); + ? { hint: "error", type: frame.type, error: errorMessage(frame), data: frame.data, direction: "send" } + : { hint: "info", type: frame.type, data: frame.data, direction: "send" }); } catch (error) { emit(this.#options.logger, { hint: "error", type: "error", error: String(error), + direction: "send", }); } @@ -622,6 +623,19 @@ export class MTPClient { async request(type: MTPCommunicationType, data: Record, options?: MTPRequestOptions): Promise; async request(typeOrFrame: Uint8Array | MTPCommunicationType, data?: Record, options: MTPRequestOptions = {}): Promise { const frame = this.#buildFrame(typeOrFrame, data, options); + try { + const parsed = this.raw.bindings.parse_frame(frame); + emit(this.#options.logger, isErrorType(parsed.type) + ? { hint: "error", type: parsed.type, error: errorMessage(parsed), data: parsed.data, direction: "send" } + : { hint: "info", type: parsed.type, data: parsed.data, direction: "send" }); + } catch (error) { + emit(this.#options.logger, { + hint: "error", + type: "error", + error: String(error), + direction: "send", + }); + } return await this.raw.client.request(frame, options.responseType ?? null); } @@ -643,12 +657,14 @@ export class MTPClient { type: frame.type, error: errorMessage(frame), data: frame.data, + direction: "recv", }); } else { emit(this.#options.logger, { hint: "info", type: frame.type, data: frame.data, + direction: "recv", }); } } diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 16c33eb..2a24408 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -687,9 +687,39 @@ impl WasmClient { fn set_state(&self, new_state: ConnectionState) { self.state.set(new_state); - let _ = self - .on_state_change - .call1(&JsValue::NULL, &JsValue::from(new_state as u8)); + + // Defer the callback to a microtask so re-entrant &mut self calls don't alias. + let cb = self.on_state_change.clone(); + let val = JsValue::from(new_state as u8); + let closure = Closure::wrap(Box::new(move || { + let _ = cb.call1(&JsValue::NULL, &val); + }) as Box); + + let global = js_sys::global(); + let mut closure_opt = Some(closure); + + let qmt = js_sys::Reflect::get(&global, &JsValue::from_str("queueMicrotask")) + .and_then(|f| f.dyn_into::().map_err(Into::into)); + let scheduled = match qmt { + Ok(qmt) => { + if let Some(c) = closure_opt.take() { + let _ = qmt.call1(&global, c.as_ref()); + c.forget(); + } + true + } + Err(_) => false, + }; + if !scheduled { + if let Ok(set_timeout) = js_sys::Reflect::get(&global, &JsValue::from_str("setTimeout")) + .and_then(|f| f.dyn_into::().map_err(Into::into)) + { + if let Some(c) = closure_opt.take() { + let _ = set_timeout.call2(&global, c.as_ref(), &JsValue::from_f64(0.0)); + c.forget(); + } + } + } } fn start_receive_loop(&mut self, transport: WasmTransport) { diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index 9a8c69d..6860552 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -10,16 +10,7 @@ use crate::frame::parse_frame_value; const CLOSE_FRAME_LEN: u32 = u32::MAX; -/// Inspect a JS error value for a WebTransport **stream-level** error and, if -/// present, log the `streamErrorCode` carried by STOP_SENDING / RESET_STREAM. -/// -/// Per draft-ietf-webtrans-http3-15 §4.4, a WebTransport application MUST -/// provide an error code for those operations. The browser surfaces these as -/// `WebTransportError` with `source = "stream"` and a numeric `streamErrorCode`. -/// -/// Session-level errors (`source = "session"`) are normal connection -/// closures and are **not** logged here — they propagate to `on_error` -/// in the receive loop like any other transport error. +/// Logs the `streamErrorCode` from a stream-level WebTransportError (STOP_SENDING / RESET_STREAM). Session errors are skipped. fn log_stream_error_code(error: &JsValue, context: &str) { let source = js_sys::Reflect::get(error, &JsValue::from_str("source")) .ok() @@ -78,10 +69,7 @@ fn resolve_stream_readable(recv_stream: &JsValue) -> Result { } } -/// Release a `WritableStreamDefaultWriter`'s lock on its stream. Called after -/// `writer.close()` (or on write failure) so the runtime does not interpret an -/// abandoned locked writer as an abort, which would surface as STOP_SENDING to -/// the peer. Errors are ignored — `releaseLock` is best-effort cleanup. +/// Releases a writer's lock so an abandoned writer isn't treated as an abort (which sends STOP_SENDING). fn release_writer_lock(writer: &JsValue) { if let Ok(release) = js_sys::Reflect::get(writer, &JsValue::from_str("releaseLock")) .and_then(|f| f.dyn_into::().map_err(Into::into)) @@ -90,11 +78,7 @@ fn release_writer_lock(writer: &JsValue) { } } -/// Release a `ReadableStreamDefaultReader`'s lock on its stream. Mirrors -/// `release_writer_lock`: abandoning a locked reader can be interpreted by the -/// runtime as a `reader.cancel()` (sending STOP_SENDING to the peer) even on an -/// already-closed or errored stream. Calling `releaseLock` explicitly avoids -/// that. Errors are ignored — best-effort cleanup. +/// Releases a reader's lock so an abandoned reader isn't treated as a cancel (which sends STOP_SENDING). fn release_reader_lock(reader: &JsValue) { if let Ok(release) = js_sys::Reflect::get(reader, &JsValue::from_str("releaseLock")) .and_then(|f| f.dyn_into::().map_err(Into::into)) @@ -256,15 +240,11 @@ impl WasmTransport { .call0(&writer_val) .map_err(|e| js_error(&format!("close failed: {:?}", e)))?; if let Err(e) = JsFuture::from(close_promise.unchecked_into::()).await { - // The write already succeeded; a STOP_SENDING on close just means - // the peer stopped reading before we could send FIN. The data is in - // flight, so this is not a send failure — log and return success. + // Write succeeded; STOP_SENDING on close just means peer stopped reading before FIN. log_stream_error_code(&e, "send_frame close"); } - // Always release the writer's lock on the WritableStream. Abandoning a - // locked writer (e.g. via drop) can be interpreted by the runtime as an - // abort, which may surface as STOP_SENDING to the peer. + // Release the lock so the writer isn't treated as an abort. release_writer_lock(&writer_val); Ok(()) @@ -418,12 +398,7 @@ impl WasmTransport { } } None => { - // Current stream finished; the next frame (if any) is on a - // subsequent stream. Any trailing partial bytes are dropped - // since the host never splits a frame across streams. - // Release the reader's lock explicitly so the runtime does - // not treat the abandoned lock as a cancel (which would send - // STOP_SENDING to the peer on an already-closed stream). + // Stream finished; release the reader's lock to avoid a spurious cancel. if let Some(reader) = self.stream_reader.borrow_mut().take() { release_reader_lock(&reader); } @@ -470,10 +445,7 @@ impl WasmTransport { } pub fn close(&self) { - // Release any held reader locks before tearing down the session, so the - // runtime does not interpret an abandoned locked reader as a cancel - // (which would send STOP_SENDING to the peer). Once the locks are - // released the underlying streams can be torn down cleanly. + // Release reader locks before closing so they aren't treated as cancels. if let Some(reader) = self.stream_reader.borrow_mut().take() { release_reader_lock(&reader); } From c7a57a851ebd929789183c800026fd0ca3eac857 Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 14:22:05 +0200 Subject: [PATCH 22/97] (fix): rust errors --- wasm/src/client.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 2a24408..054ee0b 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -76,10 +76,14 @@ fn route_incoming_frame( let Some(message_type) = message_type else { return; }; - for (_, (subscription_type, callback)) in subscriptions.borrow().iter() { - if subscription_type == &message_type { - let _ = callback.call1(&JsValue::NULL, frame); - } + let callbacks: Vec = subscriptions + .borrow() + .iter() + .filter(|(_, (t, _))| t == &message_type) + .map(|(_, (_, cb))| cb.clone()) + .collect(); + for callback in callbacks { + let _ = callback.call1(&JsValue::NULL, frame); } } From 2e7c0b489360d5d05fe53326811616e730c1a556 Mon Sep 17 00:00:00 2001 From: Alois Date: Sat, 4 Jul 2026 15:36:50 +0200 Subject: [PATCH 23/97] (fix): obscure rust wasm error --- wasm/src/client.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 054ee0b..280bbf1 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -252,7 +252,7 @@ pub enum ConnectionState { #[wasm_bindgen] pub struct WasmClient { - transport: Option, + transport: Rc>>, state: Rc>, on_state_change: js_sys::Function, pub(crate) on_message: js_sys::Function, @@ -272,7 +272,7 @@ impl WasmClient { on_error: &js_sys::Function, ) -> Self { Self { - transport: None, + transport: Rc::new(RefCell::new(None)), state: Rc::new(Cell::new(ConnectionState::Disconnected)), on_state_change: on_state_change.clone(), on_message: on_message.clone(), @@ -296,7 +296,7 @@ impl WasmClient { /// Unauthenticated connect (sends basic Identification, enables receive loop). #[wasm_bindgen] - pub async fn connect(&mut self, config: &ConnectionConfig) -> Result<(), JsValue> { + pub async fn connect(&self, config: &ConnectionConfig) -> Result<(), JsValue> { self.set_state(ConnectionState::Connecting); let transport = WasmTransport::connect( &config.url, @@ -334,7 +334,7 @@ impl WasmClient { /// Returns the confirmed (same) client ID on success. #[wasm_bindgen] pub async fn auth_connect( - &mut self, + &self, config: &ConnectionConfig, host_public_key_bytes: &[u8], keyring_bytes: &[u8], @@ -450,7 +450,7 @@ impl WasmClient { /// Returns the newly assigned client ID. #[wasm_bindgen] pub async fn auth_register( - &mut self, + &self, config: &ConnectionConfig, host_public_key_bytes: &[u8], keyring_bytes: &[u8], @@ -552,7 +552,7 @@ impl WasmClient { #[wasm_bindgen] pub async fn send(&self, frame: Vec) -> Result<(), JsValue> { - match &self.transport { + match self.transport.borrow().clone() { Some(t) => t.send_frame(&frame).await, None => Err(js_error("not connected")), } @@ -571,7 +571,7 @@ impl WasmClient { return Err(js_error("request frame must have a non-zero id")); } - let Some(transport) = self.transport.clone() else { + let Some(transport) = self.transport.borrow().clone() else { return Err(js_error("not connected")); }; @@ -613,7 +613,7 @@ impl WasmClient { #[wasm_bindgen] pub fn start_protocol_pings(&self, interval_ms: u32, client_id: u64) -> Result<(), JsValue> { self.stop_protocol_pings(); - let Some(transport) = self.transport.clone() else { + let Some(transport) = self.transport.borrow().clone() else { return Err(js_error("not connected")); }; let interval_ms = interval_ms.max(1_000) as i32; @@ -678,12 +678,11 @@ impl WasmClient { } #[wasm_bindgen] - pub fn disconnect(&mut self) { + pub fn disconnect(&self) { self.stop_protocol_pings(); - if let Some(t) = &self.transport { + if let Some(t) = self.transport.borrow_mut().take() { t.close(); } - self.transport = None; self.subscriptions.borrow_mut().clear(); self.reject_pending_requests("disconnected"); self.set_state(ConnectionState::Disconnected); @@ -726,9 +725,9 @@ impl WasmClient { } } - fn start_receive_loop(&mut self, transport: WasmTransport) { + fn start_receive_loop(&self, transport: WasmTransport) { let loop_transport = transport.clone(); - self.transport = Some(transport); + *self.transport.borrow_mut() = Some(transport); self.set_state(ConnectionState::Connected); let state = self.state.clone(); From 20cbb4574323e42430fe2c590e2f6ba84760533f Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 00:23:27 +0200 Subject: [PATCH 24/97] (fix): some ts-sdk stuff --- src/sdk/index.ts | 6 +++--- src/vite/index.ts | 52 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 9cd87e9..63977ae 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -405,7 +405,7 @@ export class MTPClient { }, (error) => emit(normalizedOptions.logger, { hint: "error", - type: "error", + type: "Error", error: String(error), }), ); @@ -610,7 +610,7 @@ export class MTPClient { } catch (error) { emit(this.#options.logger, { hint: "error", - type: "error", + type: "Error", error: String(error), direction: "send", }); @@ -631,7 +631,7 @@ export class MTPClient { } catch (error) { emit(this.#options.logger, { hint: "error", - type: "error", + type: "Error", error: String(error), direction: "send", }); diff --git a/src/vite/index.ts b/src/vite/index.ts index a906000..838f698 100644 --- a/src/vite/index.ts +++ b/src/vite/index.ts @@ -369,7 +369,7 @@ export function mtp(options: MTPVitePluginOptions): VitePlugin { buildStart() { this.addWatchFile(state.typeMapsPath); }, - configureServer(server) { + async configureServer(server) { const wasmPath = path.join(state.outDir, wasmEntryName); const wasmUrl = devServerPath(server.config.root, wasmPath); if (wasmUrl) { @@ -381,6 +381,7 @@ export function mtp(options: MTPVitePluginOptions): VitePlugin { try { res.setHeader("Content-Type", "application/wasm"); + res.setHeader("Cache-Control", "no-cache, no-store, must-revalidate"); res.end(await fs.readFile(wasmPath)); } catch (error) { next(error); @@ -388,19 +389,50 @@ export function mtp(options: MTPVitePluginOptions): VitePlugin { }); } + const sourceWatchDirs = [ + "wasm/src", + "common/src", + "codec/src", + "crypto/src", + "type-map/src", + ].map((rel) => path.join(packageRoot, rel)); + server.watcher.add(state.typeMapsPath); - server.watcher.on("change", async (changedPath) => { - if (path.resolve(changedPath) !== state.typeMapsPath) { + for (const dir of sourceWatchDirs) { + if (await pathExists(dir)) { + server.watcher.add(dir); + } + } + + let rebuildTimer: ReturnType | null = null; + const scheduleRebuild = (changedPath: string) => { + const resolved = path.resolve(changedPath); + const isTypeMap = resolved === state.typeMapsPath; + const isSource = sourceWatchDirs.some((dir) => resolved.startsWith(`${dir}${path.sep}`)); + if (!isTypeMap && !isSource) { return; } - try { - await buildIfNeeded(state, true); - server.moduleGraph.invalidateAll(); - server.ws.send({ type: "full-reload" }); - } catch (error) { - server.config.logger.error(error instanceof Error ? error.message : String(error)); + + if (rebuildTimer) { + clearTimeout(rebuildTimer); } - }); + + rebuildTimer = setTimeout(() => { + rebuildTimer = null; + void (async () => { + try { + await buildIfNeeded(state, true); + server.moduleGraph.invalidateAll(); + server.ws.send({ type: "full-reload" }); + } catch (error) { + server.config.logger.error(error instanceof Error ? error.message : String(error)); + } + })(); + }, 200); + }; + + server.watcher.on("change", scheduleRebuild); + server.watcher.on("add", scheduleRebuild); }, }; } From 3e12257cf3f8581e8e50ebae9c85544800df0007 Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 01:11:21 +0200 Subject: [PATCH 25/97] (feat): add crypto stuff to ts-sdk --- src/sdk/index.ts | 194 +++++++++++++++++++++++++++++++++++++++ wasm/src/crypto.rs | 91 +++++++++++++++++- wasm/types/mtp_wasm.d.ts | 15 +++ 3 files changed, 297 insertions(+), 3 deletions(-) diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 63977ae..3c8488d 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -26,6 +26,8 @@ export type ParsedFrame = RawBindings.ParsedFrame; export type Ed25519GenerateResult = ReturnType; +export type WasmEncapsulated = RawBindings.WasmEncapsulated; + export interface MTPCrypto { generateKeyring(): Uint8Array; generateEd25519(): Ed25519GenerateResult; @@ -35,6 +37,13 @@ export interface MTPCrypto { hkdfExpand(ikm: Uint8Array, salt: Uint8Array, info: Uint8Array, len: number): Uint8Array; sha256(data: Uint8Array): Uint8Array; sha256Double(data: Uint8Array): Uint8Array; + encrypt(secret: string, input: Uint8Array): Promise; + decrypt(secret: string, input: Uint8Array): Promise; + encryptText(secret: string, plaintext: string): Promise; + decryptText(secret: string, ciphertext: string): Promise; + encapsulate(otherPublicKey: Uint8Array): WasmEncapsulated; + decapsulate(ownPrivateKey: Uint8Array, ciphertext: Uint8Array): Uint8Array; + getSharedSecret(ownPrivateKey: string, ownPublicKey: string, otherPublicKey: string): Promise; } export const crypto: MTPCrypto = { @@ -46,6 +55,67 @@ export const crypto: MTPCrypto = { hkdfExpand: (ikm, salt, info, len) => bindings.wasm_hkdf_expand(ikm, salt, info, len), sha256: (data) => bindings.wasm_sha256(data), sha256Double: (data) => bindings.wasm_sha256_double(data), + + encrypt: async (secret, input) => { + const key = secretKeyFromString(secret); + const cipher = new bindings.WasmChaCha20Poly1305(key); + try { + return cipher.encrypt(input, new Uint8Array(0)); + } finally { + cipher.free(); + } + }, + + decrypt: async (secret, input) => { + const key = secretKeyFromString(secret); + const cipher = new bindings.WasmChaCha20Poly1305(key); + try { + return cipher.decrypt(input, new Uint8Array(0)); + } finally { + cipher.free(); + } + }, + + encryptText: async (secret, plaintext) => { + const key = secretKeyFromString(secret); + const cipher = new bindings.WasmChaCha20Poly1305(key); + try { + const ciphertext = cipher.encrypt(utf8Encode(plaintext), new Uint8Array(0)); + return bytesToBase64(ciphertext); + } finally { + cipher.free(); + } + }, + + decryptText: async (secret, ciphertext) => { + const key = secretKeyFromString(secret); + const cipher = new bindings.WasmChaCha20Poly1305(key); + try { + const decoded = bytesFromString(ciphertext, "ciphertext"); + const plaintext = cipher.decrypt(decoded, new Uint8Array(0)); + return utf8Decode(plaintext); + } finally { + cipher.free(); + } + }, + + encapsulate: (otherPublicKey) => bindings.wasm_kem_encapsulate(otherPublicKey), + + decapsulate: (ownPrivateKey, ciphertext) => + bindings.wasm_kem_decapsulate(ownPrivateKey, ciphertext), + + getSharedSecret: async (ownPrivateKey, ownPublicKey, otherPublicKey) => { + const ownPub = bytesFromString(ownPublicKey, "ownPublicKey"); + const otherPub = bytesFromString(otherPublicKey, "otherPublicKey"); + const enc = bindings.wasm_kem_encapsulate(otherPub); + try { + const sharedSecret = enc.shared_secret; + const derived = bindings.wasm_hkdf_expand(sharedSecret, ownPub, otherPub, 32); + return bytesToHex(derived); + } finally { + enc.free(); + } + }, }; export type MTPRawBindings = typeof bindings; @@ -250,6 +320,130 @@ function bytesFromString(value, name) { throw new TypeError(`${name} must be bytes, hex, or base64`); } +const HEX_DIGITS = "0123456789abcdef"; + +function bytesToHex(bytes) { + let out = ""; + for (let i = 0; i < bytes.length; i += 1) { + out += HEX_DIGITS[(bytes[i] >> 4) & 0xf] + HEX_DIGITS[bytes[i] & 0xf]; + } + return out; +} + +function bytesToBase64(bytes) { + if (typeof btoa === "function") { + let binary = ""; + for (let i = 0; i < bytes.length; i += 1) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); + } + if (typeof Buffer !== "undefined") { + return Buffer.from(bytes).toString("base64"); + } + throw new TypeError("base64 encoding is not available in this environment"); +} + +function utf8Encode(text) { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(text); + } + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(text, "utf-8")); + } + const bytes = new Uint8Array(text.length * 4); + let len = 0; + for (let i = 0; i < text.length; i += 1) { + const code = text.codePointAt(i); + if (code < 0x80) { + bytes[len++] = code; + } else if (code < 0x800) { + bytes[len++] = 0xc0 | (code >> 6); + bytes[len++] = 0x80 | (code & 0x3f); + } else if (code < 0x10000) { + bytes[len++] = 0xe0 | (code >> 12); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + } else { + bytes[len++] = 0xf0 | (code >> 18); + bytes[len++] = 0x80 | ((code >> 12) & 0x3f); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + i += 1; + } + } + return bytes.subarray(0, len); +} + +function utf8Decode(bytes) { + if (typeof TextDecoder !== "undefined") { + return new TextDecoder().decode(bytes); + } + if (typeof Buffer !== "undefined") { + return Buffer.from(bytes).toString("utf-8"); + } + let out = ""; + let i = 0; + while (i < bytes.length) { + const b = bytes[i]; + if (b < 0x80) { + out += String.fromCharCode(b); + i += 1; + } else if (b < 0xc0) { + i += 1; + } else if (b < 0xe0) { + out += String.fromCharCode(((b & 0x1f) << 6) | (bytes[i + 1] & 0x3f)); + i += 2; + } else if (b < 0xf0) { + out += String.fromCharCode( + ((b & 0x0f) << 12) | ((bytes[i + 1] & 0x3f) << 6) | (bytes[i + 2] & 0x3f), + ); + i += 3; + } else { + const cp = + ((b & 0x07) << 18) | + ((bytes[i + 1] & 0x3f) << 12) | + ((bytes[i + 2] & 0x3f) << 6) | + (bytes[i + 3] & 0x3f); + out += String.fromCodePoint(cp); + i += 4; + } + } + return out; +} + +const SYMMETRIC_KEY_SALT = utf8Encode("mtp-symmetric-key"); + +function secretKeyFromString(secret) { + if (typeof secret !== "string" || !secret.trim()) { + throw new TypeError("secret must be a non-empty string"); + } + + const trimmed = secret.trim(); + const hex = trimmed.replace(/^(0x)/i, "").replace(/[\s:_-]/g, ""); + if (/^[0-9a-fA-F]+$/.test(hex) && hex.length === 64) { + const bytes = new Uint8Array(32); + for (let i = 0; i < 32; i += 1) { + bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + if (typeof atob === "function" || typeof Buffer !== "undefined") { + try { + const decoded = bytesFromString(trimmed, "secret"); + if (decoded.length === 32) { + return decoded; + } + } catch { + // fall through to HKDF derivation + } + } + + const ikm = utf8Encode(trimmed); + return bindings.wasm_derive_encryption_key(ikm, SYMMETRIC_KEY_SALT, SYMMETRIC_KEY_SALT); +} + function normalizeBytes(value, name) { if (typeof value === "string") { return bytesFromString(value, name); diff --git a/wasm/src/crypto.rs b/wasm/src/crypto.rs index 48aa4dc..cfa1ebf 100644 --- a/wasm/src/crypto.rs +++ b/wasm/src/crypto.rs @@ -1,9 +1,9 @@ use wasm_bindgen::prelude::*; use mtp_crypto::{ - AeadDecrypt, AeadEncrypt, ChaCha20Poly1305, Ed25519Signer, KemPrivateKey, KemPublicKey, - Keyring, PublicKeyBundle, SignaturePqPrivateKey, SignaturePqPublicKey, SignaturePrivateKey, - SignaturePublicKey, SignatureScheme, sha256, sha256_double, + AeadDecrypt, AeadEncrypt, ChaCha20Poly1305, Ed25519Signer, HybridKem, KemPrivateKey, + KemPublicKey, Keyring, PublicKeyBundle, SignaturePqPrivateKey, SignaturePqPublicKey, + SignaturePrivateKey, SignaturePublicKey, SignatureScheme, sha256, sha256_double, }; use crate::error::js_error; @@ -110,6 +110,66 @@ impl WasmPublicKeyBundle { } } +// =========================================================================== +// Hybrid KEM (X25519 + ML-KEM-768) +// =========================================================================== + +/// KEM encapsulation result returned to JavaScript. +/// +/// `shared_secret` is the symmetric key both parties will derive; `ciphertext` +/// is the KEM ciphertext that must be sent to the recipient so they can +/// decapsulate and recover the same shared secret. +#[wasm_bindgen] +pub struct WasmEncapsulated { + inner_shared_secret: Vec, + inner_ciphertext: Vec, +} + +#[wasm_bindgen] +impl WasmEncapsulated { + /// Symmetric secret derived during encapsulation. + #[wasm_bindgen(getter)] + pub fn shared_secret(&self) -> Vec { + self.inner_shared_secret.clone() + } + + /// KEM ciphertext to transmit to the recipient. + #[wasm_bindgen(getter)] + pub fn ciphertext(&self) -> Vec { + self.inner_ciphertext.clone() + } +} + +/// Encapsulate a fresh shared secret for `recipient_public_key`. +/// +/// Returns a [`WasmEncapsulated`] containing the shared secret and the KEM +/// ciphertext that the recipient needs to recover it via +/// [`wasm_kem_decapsulate`]. +#[wasm_bindgen] +pub fn wasm_kem_encapsulate(recipient_public_key: &[u8]) -> Result { + let pk = KemPublicKey::new(recipient_public_key.to_vec()); + let enc = HybridKem::encapsulate(&pk) + .map_err(|e| js_error(&format!("kem_encapsulate failed: {}", e)))?; + Ok(WasmEncapsulated { + inner_shared_secret: enc.shared_secret, + inner_ciphertext: enc.ciphertext, + }) +} + +/// Decapsulate a KEM `ciphertext` with the recipient's `private_key`. +/// +/// Returns the same shared secret the initiator obtained from +/// [`wasm_kem_encapsulate`]. +#[wasm_bindgen] +pub fn wasm_kem_decapsulate( + recipient_private_key: &[u8], + ciphertext: &[u8], +) -> Result, JsValue> { + let sk = KemPrivateKey::new(recipient_private_key.to_vec()); + HybridKem::decapsulate(&sk, ciphertext) + .map_err(|e| js_error(&format!("kem_decapsulate failed: {}", e))) +} + // =========================================================================== // ChaCha20-Poly1305 AEAD // =========================================================================== @@ -326,6 +386,31 @@ mod tests { assert_eq!(restored.sig_cl_public_key(), pk); } + // ------------------------------------------------------------------ + // KEM encapsulate / decapsulate + // ------------------------------------------------------------------ + + #[wasm_bindgen_test] + fn kem_encapsulate_decapsulate_roundtrip() { + let (sk, pk) = HybridKem::generate_keypair(); + let enc = wasm_kem_encapsulate(pk.as_bytes()).expect("encapsulate failed"); + let ss = wasm_kem_decapsulate(sk.as_bytes(), &enc.ciphertext()).expect("decapsulate failed"); + assert_eq!(enc.shared_secret(), ss); + } + + #[wasm_bindgen_test] + fn kem_encapsulate_invalid_public_key_fails() { + let bad = vec![0u8; 16]; + assert!(wasm_kem_encapsulate(&bad).is_err()); + } + + #[wasm_bindgen_test] + fn kem_decapsulate_invalid_ciphertext_fails() { + let (sk, _pk) = HybridKem::generate_keypair(); + let bad = vec![0u8; 32]; + assert!(wasm_kem_decapsulate(sk.as_bytes(), &bad).is_err()); + } + // ------------------------------------------------------------------ // ChaCha20-Poly1305 // ------------------------------------------------------------------ diff --git a/wasm/types/mtp_wasm.d.ts b/wasm/types/mtp_wasm.d.ts index b7ac775..47b16f7 100644 --- a/wasm/types/mtp_wasm.d.ts +++ b/wasm/types/mtp_wasm.d.ts @@ -75,6 +75,19 @@ export class WasmChaCha20Poly1305 implements DisposableWasmObject { encrypt(plaintext: Uint8Array, aad: Uint8Array): Uint8Array; } +export interface KemEncapsulateResult { + shared_secret: Uint8Array; + ciphertext: Uint8Array; +} + +export class WasmEncapsulated implements DisposableWasmObject { + private constructor(); + free(): void; + [Symbol.dispose](): void; + readonly shared_secret: Uint8Array; + readonly ciphertext: Uint8Array; +} + export class WasmClient implements DisposableWasmObject { constructor( on_state_change: StateChangeCallback, @@ -166,6 +179,8 @@ export function parse_auth_response(response: Uint8Array): AuthResponse; export function parse_frame(frame: Uint8Array): ParsedFrame; export function wasm_derive_encryption_key(ikm: Uint8Array, salt: Uint8Array, context: Uint8Array): Uint8Array; export function wasm_hkdf_expand(ikm: Uint8Array, salt: Uint8Array, info: Uint8Array, len: number): Uint8Array; +export function wasm_kem_decapsulate(recipient_private_key: Uint8Array, ciphertext: Uint8Array): Uint8Array; +export function wasm_kem_encapsulate(recipient_public_key: Uint8Array): WasmEncapsulated; export function wasm_sha256(data: Uint8Array): Uint8Array; export function wasm_sha256_double(data: Uint8Array): Uint8Array; From fdc694bdcd64801b4bb3957fb00f272d5ae1fe64 Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 01:16:45 +0200 Subject: [PATCH 26/97] (fix): formatting issue --- wasm/src/crypto.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wasm/src/crypto.rs b/wasm/src/crypto.rs index cfa1ebf..eb275fa 100644 --- a/wasm/src/crypto.rs +++ b/wasm/src/crypto.rs @@ -394,7 +394,8 @@ mod tests { fn kem_encapsulate_decapsulate_roundtrip() { let (sk, pk) = HybridKem::generate_keypair(); let enc = wasm_kem_encapsulate(pk.as_bytes()).expect("encapsulate failed"); - let ss = wasm_kem_decapsulate(sk.as_bytes(), &enc.ciphertext()).expect("decapsulate failed"); + let ss = + wasm_kem_decapsulate(sk.as_bytes(), &enc.ciphertext()).expect("decapsulate failed"); assert_eq!(enc.shared_secret(), ss); } From 13432c1ac202de1844f869b95fca9f89d255e1d5 Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 11:59:35 +0200 Subject: [PATCH 27/97] (feat): add function to easily split keyring --- src/sdk/index.ts | 92 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 3c8488d..4c0e19c 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -37,13 +37,13 @@ export interface MTPCrypto { hkdfExpand(ikm: Uint8Array, salt: Uint8Array, info: Uint8Array, len: number): Uint8Array; sha256(data: Uint8Array): Uint8Array; sha256Double(data: Uint8Array): Uint8Array; - encrypt(secret: string, input: Uint8Array): Promise; - decrypt(secret: string, input: Uint8Array): Promise; - encryptText(secret: string, plaintext: string): Promise; - decryptText(secret: string, ciphertext: string): Promise; + keyringToKeys(keyring: string | MTPBytesInput): MTPKeyringKeys; + encrypt(key: Uint8Array, input: Uint8Array): Promise; + decrypt(key: Uint8Array, input: Uint8Array): Promise; + encryptText(key: Uint8Array, plaintext: string): Promise; + decryptText(key: Uint8Array, ciphertext: string): Promise; encapsulate(otherPublicKey: Uint8Array): WasmEncapsulated; decapsulate(ownPrivateKey: Uint8Array, ciphertext: Uint8Array): Uint8Array; - getSharedSecret(ownPrivateKey: string, ownPublicKey: string, otherPublicKey: string): Promise; } export const crypto: MTPCrypto = { @@ -55,9 +55,9 @@ export const crypto: MTPCrypto = { hkdfExpand: (ikm, salt, info, len) => bindings.wasm_hkdf_expand(ikm, salt, info, len), sha256: (data) => bindings.wasm_sha256(data), sha256Double: (data) => bindings.wasm_sha256_double(data), + keyringToKeys: (keyring) => keyringToKeys(keyring), - encrypt: async (secret, input) => { - const key = secretKeyFromString(secret); + encrypt: async (key, input) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { return cipher.encrypt(input, new Uint8Array(0)); @@ -66,8 +66,7 @@ export const crypto: MTPCrypto = { } }, - decrypt: async (secret, input) => { - const key = secretKeyFromString(secret); + decrypt: async (key, input) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { return cipher.decrypt(input, new Uint8Array(0)); @@ -76,8 +75,7 @@ export const crypto: MTPCrypto = { } }, - encryptText: async (secret, plaintext) => { - const key = secretKeyFromString(secret); + encryptText: async (key, plaintext) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { const ciphertext = cipher.encrypt(utf8Encode(plaintext), new Uint8Array(0)); @@ -87,8 +85,7 @@ export const crypto: MTPCrypto = { } }, - decryptText: async (secret, ciphertext) => { - const key = secretKeyFromString(secret); + decryptText: async (key, ciphertext) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { const decoded = bytesFromString(ciphertext, "ciphertext"); @@ -103,19 +100,6 @@ export const crypto: MTPCrypto = { decapsulate: (ownPrivateKey, ciphertext) => bindings.wasm_kem_decapsulate(ownPrivateKey, ciphertext), - - getSharedSecret: async (ownPrivateKey, ownPublicKey, otherPublicKey) => { - const ownPub = bytesFromString(ownPublicKey, "ownPublicKey"); - const otherPub = bytesFromString(otherPublicKey, "otherPublicKey"); - const enc = bindings.wasm_kem_encapsulate(otherPub); - try { - const sharedSecret = enc.shared_secret; - const derived = bindings.wasm_hkdf_expand(sharedSecret, ownPub, otherPub, 32); - return bytesToHex(derived); - } finally { - enc.free(); - } - }, }; export type MTPRawBindings = typeof bindings; @@ -191,6 +175,15 @@ export interface MTPClientCredentials { hostPublicKey?: Uint8Array; } +export interface MTPKeyringKeys { + kemPublicKey: Uint8Array; + kemSecretKey: Uint8Array; + sigPqPublicKey: Uint8Array; + sigPqSecretKey: Uint8Array; + sigClPublicKey: Uint8Array; + sigClSecretKey: Uint8Array; +} + export interface MTPClientOptions { url: string; descriptor?: string; @@ -330,7 +323,7 @@ function bytesToHex(bytes) { return out; } -function bytesToBase64(bytes) { +export function bytesToBase64(bytes) { if (typeof btoa === "function") { let binary = ""; for (let i = 0; i < bytes.length; i += 1) { @@ -344,6 +337,21 @@ function bytesToBase64(bytes) { throw new TypeError("base64 encoding is not available in this environment"); } +export function base64ToBytes(input) { + if (typeof atob === "function") { + const binary = atob(input); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i += 1) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; + } + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(input, "base64")); + } + throw new TypeError("base64 decoding is not available in this environment"); +} + function utf8Encode(text) { if (typeof TextEncoder !== "undefined") { return new TextEncoder().encode(text); @@ -414,7 +422,7 @@ function utf8Decode(bytes) { const SYMMETRIC_KEY_SALT = utf8Encode("mtp-symmetric-key"); -function secretKeyFromString(secret) { +export function secretKeyFromString(secret) { if (typeof secret !== "string" || !secret.trim()) { throw new TypeError("secret must be a non-empty string"); } @@ -474,6 +482,34 @@ function generateKeyringBytes() { return keyring_generate(); } +export function keyringToKeys(keyring) { + const bytes = typeof keyring === "string" + ? bytesFromString(keyring, "keyring") + : bytesFrom(keyring, "keyring"); + + if (bytes.length < 12) { + throw new TypeError("keyring data is too short to contain 6 keys"); + } + + let offset = 0; + const readKey = () => { + const len = (bytes[offset] << 8) | bytes[offset + 1]; + offset += 2; + const key = bytes.slice(offset, offset + len); + offset += len; + return key; + }; + + return { + kemPublicKey: readKey(), + kemSecretKey: readKey(), + sigPqPublicKey: readKey(), + sigPqSecretKey: readKey(), + sigClPublicKey: readKey(), + sigClSecretKey: readKey(), + }; +} + function serializeCredentials(credentials) { return JSON.stringify({ clientId: credentials.clientId?.toString() ?? null, From e1fcb90e19255f844154d5c23c37f85193da0e14 Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 21:45:43 +0200 Subject: [PATCH 28/97] (feat): crypto migrations --- example/type-maps.yaml | 6 + src/sdk/encrypted-device-secret.ts | 74 +++ src/sdk/encrypted-message.ts | 351 ++++++++++++ src/sdk/index.ts | 831 ++++++++++++++++++++++++++--- src/sdk/ratchet.ts | 70 +++ src/sdk/session.ts | 249 +++++++++ test/e2ee.mjs | 503 +++++++++++++++++ tsconfig.json | 8 +- type-map/build.rs | 23 +- 9 files changed, 2032 insertions(+), 83 deletions(-) create mode 100644 src/sdk/encrypted-device-secret.ts create mode 100644 src/sdk/encrypted-message.ts create mode 100644 src/sdk/ratchet.ts create mode 100644 src/sdk/session.ts create mode 100644 test/e2ee.mjs diff --git a/example/type-maps.yaml b/example/type-maps.yaml index 0289011..6e411e4 100644 --- a/example/type-maps.yaml +++ b/example/type-maps.yaml @@ -6,6 +6,7 @@ type_maps: DataTypes: "1.0": CommunicationTypes: + CommunicationType: 32 DataTypes: Data: 32 Flags: 33 @@ -15,8 +16,11 @@ type_maps: EncryptedPayload: 37 SignedPayload: 38 SecurePayload: 39 + CommunicationType: 40 + DataType: 41 "2.0": CommunicationTypes: + CommunicationType: 32 DataTypes: Data: 34 Flags: 33 @@ -26,3 +30,5 @@ type_maps: EncryptedPayload: 38 SignedPayload: 39 SecurePayload: 40 + CommunicationType: 41 + DataType: 42 diff --git a/src/sdk/encrypted-device-secret.ts b/src/sdk/encrypted-device-secret.ts new file mode 100644 index 0000000..bc4587d --- /dev/null +++ b/src/sdk/encrypted-device-secret.ts @@ -0,0 +1,74 @@ +export interface EncryptedDeviceSecretRecord { + userId: string; + deviceId: string; + secretId: string; + version: number; + encryptedSecret: Uint8Array; + wrappingPublicKeyId?: string; + wrappingScheme: string; + createdAt: number; + updatedAt: number; +} + +export interface MTPEncryptedDeviceSecretProvider { + setEncryptedDeviceSecret(record: EncryptedDeviceSecretRecord): Promise; + getEncryptedDeviceSecret(query: { + userId: string; + deviceId?: string; + secretId?: string; + }): Promise; +} + +function keyFor(record: Pick): string { + return `${record.userId}\0${record.deviceId}\0${record.secretId}`; +} + +function cloneRecord(record: EncryptedDeviceSecretRecord): EncryptedDeviceSecretRecord { + return { + ...record, + encryptedSecret: new Uint8Array(record.encryptedSecret), + }; +} + +function validateEncryptedRecord(record: EncryptedDeviceSecretRecord): void { + if (!record.userId || !record.deviceId || !record.secretId) { + throw new Error("encrypted device secret requires userId, deviceId, and secretId"); + } + if (!(record.encryptedSecret instanceof Uint8Array) || record.encryptedSecret.length === 0) { + throw new Error("encrypted device secret requires non-empty encryptedSecret bytes"); + } + if (!record.wrappingScheme) { + throw new Error("encrypted device secret requires wrappingScheme"); + } +} + +export class InMemoryEncryptedDeviceSecretProvider implements MTPEncryptedDeviceSecretProvider { + private store = new Map(); + + async setEncryptedDeviceSecret(record: EncryptedDeviceSecretRecord): Promise { + validateEncryptedRecord(record); + const now = Date.now(); + this.store.set(keyFor(record), cloneRecord({ ...record, updatedAt: record.updatedAt || now })); + } + + async getEncryptedDeviceSecret(query: { + userId: string; + deviceId?: string; + secretId?: string; + }): Promise { + if (!query.userId) { + throw new Error("userId is required"); + } + if (query.deviceId && query.secretId) { + const found = this.store.get(`${query.userId}\0${query.deviceId}\0${query.secretId}`); + return found ? cloneRecord(found) : null; + } + for (const record of this.store.values()) { + if (record.userId !== query.userId) continue; + if (query.deviceId && record.deviceId !== query.deviceId) continue; + if (query.secretId && record.secretId !== query.secretId) continue; + return cloneRecord(record); + } + return null; + } +} diff --git a/src/sdk/encrypted-message.ts b/src/sdk/encrypted-message.ts new file mode 100644 index 0000000..3f816ea --- /dev/null +++ b/src/sdk/encrypted-message.ts @@ -0,0 +1,351 @@ +import * as bindings from "mtp/raw"; +import { MTPRatchet } from "./ratchet.js"; +import type { MTPSessionState } from "./session"; + +export const MTP_E2EE_VERSION = 1; +export const FLAG_INIT = 0x01; +export const FLAG_DEVICE_SECRET = 0x02; +export const FLAG_KEY_ROTATION = 0x04; +export const MAX_RATCHET_SKIP = 100; +const SUPPORTED_FLAGS = FLAG_INIT | FLAG_DEVICE_SECRET | FLAG_KEY_ROTATION; +const HEADER_FIXED_LEN = 1 + 1 + 8 + 8 + 4 + 2 + 4; + +export interface ParsedEncryptedMessage { + version: 1; + flags: number; + senderClientId: bigint; + recipientClientId: bigint; + messageNumber: number; + kemCiphertext?: Uint8Array; + ciphertext: Uint8Array; + /** Compatibility alias for older SDK tests/callers. */ + header?: EncryptedMessageHeader; + /** Compatibility alias for older SDK tests/callers. */ + aeadPayload?: Uint8Array; +} + +export interface EncryptedMessageHeader { + version: 1; + flags: number; + senderClientId: bigint; + recipientClientId: bigint; + messageNumber: number; + kemCiphertext?: Uint8Array; +} +export interface SerializedEncryptedMessage { + header: EncryptedMessageHeader; + aeadPayload: Uint8Array; +} + +function writeU64BE(value: bigint): Uint8Array { + if (value < 0n || value > 0xffff_ffff_ffff_ffffn) { + throw new Error("u64 value out of range"); + } + const buf = new Uint8Array(8); + for (let i = 7; i >= 0; i--) { + buf[i] = Number(value & 0xffn); + value >>= 8n; + } + return buf; +} + +function readU64BE(bytes: Uint8Array, offset: number): bigint { + let value = 0n; + for (let i = 0; i < 8; i++) { + value = (value << 8n) | BigInt(bytes[offset + i]); + } + return value; +} + +function writeU32BE(value: number): Uint8Array { + if (!Number.isSafeInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new Error("u32 value out of range"); + } + return new Uint8Array([ + (value >>> 24) & 0xff, + (value >>> 16) & 0xff, + (value >>> 8) & 0xff, + value & 0xff, + ]); +} + +function concatBytes(parts: Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +function assertSupported(message: ParsedEncryptedMessage): void { + if (message.version !== MTP_E2EE_VERSION) { + throw new Error( + `Unsupported encrypted message version: ${message.version}`, + ); + } + if ((message.flags & ~SUPPORTED_FLAGS) !== 0) { + throw new Error(`Unsupported encrypted message flags: ${message.flags}`); + } + const isInit = (message.flags & FLAG_INIT) !== 0; + if (isInit && !message.kemCiphertext?.length) { + throw new Error("Init message must include KEM ciphertext"); + } + if (!isInit && message.kemCiphertext?.length) { + throw new Error("Non-init message must not include KEM ciphertext"); + } + if (!message.ciphertext.length) { + throw new Error("Encrypted message ciphertext must be non-empty"); + } +} + +export function serializeEncryptedMessage( + message: ParsedEncryptedMessage, +): Uint8Array; +export function serializeEncryptedMessage( + message: SerializedEncryptedMessage, +): Uint8Array; +export function serializeEncryptedMessage( + message: ParsedEncryptedMessage | SerializedEncryptedMessage, +): Uint8Array { + const normalized: ParsedEncryptedMessage = + "header" in message + ? { + ...message.header, + ciphertext: message.aeadPayload, + } + : message; + + assertSupported(normalized); + const kemCiphertext = normalized.kemCiphertext ?? new Uint8Array(0); + if (kemCiphertext.length > 0xffff) { + throw new Error("KEM ciphertext too long"); + } + + return concatBytes([ + new Uint8Array([normalized.version]), + new Uint8Array([normalized.flags]), + writeU64BE(normalized.senderClientId), + writeU64BE(normalized.recipientClientId), + writeU32BE(normalized.messageNumber), + new Uint8Array([ + (kemCiphertext.length >>> 8) & 0xff, + kemCiphertext.length & 0xff, + ]), + kemCiphertext, + writeU32BE(normalized.ciphertext.length), + normalized.ciphertext, + ]); +} + +export function parseEncryptedMessage( + bytes: Uint8Array, +): ParsedEncryptedMessage { + let offset = 0; + if (!(bytes instanceof Uint8Array)) { + throw new Error("Encrypted message must be bytes"); + } + if (bytes.length < HEADER_FIXED_LEN) { + throw new Error("Encrypted message too short"); + } + + const version = bytes[offset++]; + const flags = bytes[offset++]; + const senderClientId = readU64BE(bytes, offset); + offset += 8; + const recipientClientId = readU64BE(bytes, offset); + offset += 8; + const messageNumber = + ((bytes[offset] << 24) | + (bytes[offset + 1] << 16) | + (bytes[offset + 2] << 8) | + bytes[offset + 3]) >>> + 0; + offset += 4; + const kemLen = (bytes[offset] << 8) | bytes[offset + 1]; + offset += 2; + + let kemCiphertext: Uint8Array | undefined; + if (kemLen > 0) { + if (bytes.length < offset + kemLen + 4) { + throw new Error("Encrypted message KEM ciphertext truncated"); + } + kemCiphertext = bytes.slice(offset, offset + kemLen); + offset += kemLen; + } + + if (bytes.length < offset + 4) { + throw new Error("Encrypted message missing ciphertext length"); + } + const ciphertextLen = + ((bytes[offset] << 24) | + (bytes[offset + 1] << 16) | + (bytes[offset + 2] << 8) | + bytes[offset + 3]) >>> + 0; + offset += 4; + if (bytes.length < offset + ciphertextLen) { + throw new Error("Encrypted message ciphertext truncated"); + } + const ciphertext = bytes.slice(offset, offset + ciphertextLen); + offset += ciphertextLen; + if (offset !== bytes.length) { + throw new Error("Encrypted message has trailing data"); + } + + const parsed: ParsedEncryptedMessage = { + version: version as 1, + flags, + senderClientId, + recipientClientId, + messageNumber, + kemCiphertext, + ciphertext, + }; + parsed.header = { + version: parsed.version, + flags: parsed.flags, + senderClientId: parsed.senderClientId, + recipientClientId: parsed.recipientClientId, + messageNumber: parsed.messageNumber, + kemCiphertext: parsed.kemCiphertext, + }; + parsed.aeadPayload = parsed.ciphertext; + assertSupported(parsed); + return parsed; +} + +function buildAAD(header: EncryptedMessageHeader): Uint8Array { + return concatBytes([ + new Uint8Array([header.version]), + new Uint8Array([header.flags]), + writeU64BE(header.senderClientId), + writeU64BE(header.recipientClientId), + writeU32BE(header.messageNumber), + ]); +} + +export function encryptedMessageAAD( + header: EncryptedMessageHeader, + extra?: Uint8Array, +): Uint8Array { + return extra?.length + ? concatBytes([buildAAD(header), extra]) + : buildAAD(header); +} + +export async function encryptPayload(args: { + plaintext: Uint8Array; + session: MTPSessionState; + kemCiphertext?: Uint8Array; + aad?: Uint8Array; +}): Promise<{ + payload: Uint8Array; + session: MTPSessionState; +}> { + const step = await MTPRatchet.stepSend(args.session.sendChainKey); + const header: EncryptedMessageHeader = { + version: 1, + flags: args.kemCiphertext ? FLAG_INIT : 0, + senderClientId: args.session.ownClientId, + recipientClientId: args.session.peerClientId, + messageNumber: args.session.sendCount, + kemCiphertext: args.kemCiphertext, + }; + const aad = args.aad ?? encryptedMessageAAD(header); + const cipher = new bindings.WasmChaCha20Poly1305(step.key); + let ciphertext: Uint8Array; + try { + ciphertext = cipher.encrypt(args.plaintext, aad); + } finally { + cipher.free(); + step.key.fill(0); + } + + const payload = serializeEncryptedMessage({ ...header, ciphertext }); + return { + payload, + session: { + ...args.session, + sendChainKey: step.chainKey, + sendCount: args.session.sendCount + 1, + updatedAt: Date.now(), + }, + }; +} + +export async function decryptPayload(args: { + payload: Uint8Array; + session: MTPSessionState; + expectedRecipientClientId?: bigint; + aad?: Uint8Array; +}): Promise<{ + plaintext: Uint8Array; + session: MTPSessionState; +}> { + const parsed = parseEncryptedMessage(args.payload); + const expectedRecipientClientId = + args.expectedRecipientClientId ?? args.session.ownClientId; + if (parsed.recipientClientId !== expectedRecipientClientId) { + throw new Error("Encrypted message recipient mismatch"); + } + if (parsed.senderClientId !== args.session.peerClientId) { + throw new Error("Encrypted message sender mismatch"); + } + if (parsed.messageNumber < args.session.recvCount) { + throw new Error("Encrypted message replay or out-of-order message number"); + } + + let chainKey = args.session.recvChainKey; + let messageKey: Uint8Array | undefined; + const gap = parsed.messageNumber - args.session.recvCount; + if (gap > MAX_RATCHET_SKIP) { + throw new Error( + `Encrypted message receive gap exceeds max skip (${MAX_RATCHET_SKIP})`, + ); + } + + const steps = gap + 1; + for (let i = 0; i < steps; i += 1) { + const step = await MTPRatchet.stepRecv(chainKey); + if (i === steps - 1) { + messageKey = step.key; + } else { + step.key.fill(0); + } + if (chainKey !== args.session.recvChainKey) chainKey.fill(0); + chainKey = step.chainKey; + } + if (!messageKey) { + throw new Error("Failed to derive receive message key"); + } + + const header: EncryptedMessageHeader = { + version: parsed.version, + flags: parsed.flags, + senderClientId: parsed.senderClientId, + recipientClientId: parsed.recipientClientId, + messageNumber: parsed.messageNumber, + kemCiphertext: parsed.kemCiphertext, + }; + const aad = args.aad ?? encryptedMessageAAD(header); + const cipher = new bindings.WasmChaCha20Poly1305(messageKey); + let plaintext: Uint8Array; + try { + plaintext = cipher.decrypt(parsed.ciphertext, aad); + } finally { + cipher.free(); + messageKey.fill(0); + } + + return { + plaintext, + session: { + ...args.session, + recvChainKey: chainKey, + recvCount: parsed.messageNumber + 1, + updatedAt: Date.now(), + }, + }; +} diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 4c0e19c..b7a7331 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -7,6 +7,24 @@ import initWasm, { import * as bindings from "mtp/raw"; import type * as RawBindings from "../raw/index"; import type { MTPCommunicationType } from "../type-map/index"; +import type { MTPSessionStorage, MTPSessionState } from "./session"; +import { + MTPSessionManager, + getConversationId, + deriveSessionKeys, +} from "./session.js"; +import type { + EncryptedDeviceSecretRecord, + MTPEncryptedDeviceSecretProvider, +} from "./encrypted-device-secret"; +import { InMemoryEncryptedDeviceSecretProvider } from "./encrypted-device-secret.js"; +import { InMemorySessionStorage } from "./session.js"; +import { + parseEncryptedMessage, + encryptPayload, + decryptPayload, + FLAG_INIT, +} from "./encrypted-message.js"; export type StorageValue = string | null; @@ -19,12 +37,25 @@ export interface MTPCredentialStorage { export type MTPStorage = MTPCredentialStorage; export type MTPLogEvent = - | { hint: "info" | "warning"; type: string; data: unknown; direction?: "send" | "recv" } - | { hint: "error"; type: string | "error"; error: string; data?: unknown; direction?: "send" | "recv" }; + | { + hint: "info" | "warning"; + type: string; + data: unknown; + direction?: "send" | "recv"; + } + | { + hint: "error"; + type: string | "error"; + error: string; + data?: unknown; + direction?: "send" | "recv"; + }; export type ParsedFrame = RawBindings.ParsedFrame; -export type Ed25519GenerateResult = ReturnType; +export type Ed25519GenerateResult = ReturnType< + typeof bindings.ed25519_generate +>; export type WasmEncapsulated = RawBindings.WasmEncapsulated; @@ -32,12 +63,26 @@ export interface MTPCrypto { generateKeyring(): Uint8Array; generateEd25519(): Ed25519GenerateResult; keyringFromEd25519(secretKey: Uint8Array, publicKey: Uint8Array): Uint8Array; - verifyEd25519(publicKey: Uint8Array, message: Uint8Array, signature: Uint8Array): void; - deriveEncryptionKey(ikm: Uint8Array, salt: Uint8Array, context: Uint8Array): Uint8Array; - hkdfExpand(ikm: Uint8Array, salt: Uint8Array, info: Uint8Array, len: number): Uint8Array; + verifyEd25519( + publicKey: Uint8Array, + message: Uint8Array, + signature: Uint8Array, + ): void; + deriveEncryptionKey( + ikm: Uint8Array, + salt: Uint8Array, + context: Uint8Array, + ): Uint8Array; + hkdfExpand( + ikm: Uint8Array, + salt: Uint8Array, + info: Uint8Array, + len: number, + ): Uint8Array; sha256(data: Uint8Array): Uint8Array; sha256Double(data: Uint8Array): Uint8Array; keyringToKeys(keyring: string | MTPBytesInput): MTPKeyringKeys; + publicKeyBundleToKeys(publicKeyBundle: string | MTPBytesInput): MTPPublicKeyBundleKeys; encrypt(key: Uint8Array, input: Uint8Array): Promise; decrypt(key: Uint8Array, input: Uint8Array): Promise; encryptText(key: Uint8Array, plaintext: string): Promise; @@ -49,13 +94,19 @@ export interface MTPCrypto { export const crypto: MTPCrypto = { generateKeyring: () => bindings.keyring_generate(), generateEd25519: () => bindings.ed25519_generate(), - keyringFromEd25519: (secretKey, publicKey) => bindings.keyring_from_ed25519(secretKey, publicKey), - verifyEd25519: (publicKey, message, signature) => bindings.ed25519_verify(publicKey, message, signature), - deriveEncryptionKey: (ikm, salt, context) => bindings.wasm_derive_encryption_key(ikm, salt, context), - hkdfExpand: (ikm, salt, info, len) => bindings.wasm_hkdf_expand(ikm, salt, info, len), + keyringFromEd25519: (secretKey, publicKey) => + bindings.keyring_from_ed25519(secretKey, publicKey), + verifyEd25519: (publicKey, message, signature) => + bindings.ed25519_verify(publicKey, message, signature), + deriveEncryptionKey: (ikm, salt, context) => + bindings.wasm_derive_encryption_key(ikm, salt, context), + hkdfExpand: (ikm, salt, info, len) => + bindings.wasm_hkdf_expand(ikm, salt, info, len), sha256: (data) => bindings.wasm_sha256(data), sha256Double: (data) => bindings.wasm_sha256_double(data), keyringToKeys: (keyring) => keyringToKeys(keyring), + publicKeyBundleToKeys: (publicKeyBundle) => + publicKeyBundleToKeys(publicKeyBundle), encrypt: async (key, input) => { const cipher = new bindings.WasmChaCha20Poly1305(key); @@ -78,7 +129,10 @@ export const crypto: MTPCrypto = { encryptText: async (key, plaintext) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { - const ciphertext = cipher.encrypt(utf8Encode(plaintext), new Uint8Array(0)); + const ciphertext = cipher.encrypt( + utf8Encode(plaintext), + new Uint8Array(0), + ); return bytesToBase64(ciphertext); } finally { cipher.free(); @@ -96,7 +150,8 @@ export const crypto: MTPCrypto = { } }, - encapsulate: (otherPublicKey) => bindings.wasm_kem_encapsulate(otherPublicKey), + encapsulate: (otherPublicKey) => + bindings.wasm_kem_encapsulate(otherPublicKey), decapsulate: (ownPrivateKey, ciphertext) => bindings.wasm_kem_decapsulate(ownPrivateKey, ciphertext), @@ -136,12 +191,20 @@ export interface MTPCodecOptions { } export interface MTPCodec { - encode(type: MTPCommunicationType, data: Record, options?: MTPCodecOptions): Uint8Array; + encode( + type: MTPCommunicationType, + data: Record, + options?: MTPCodecOptions, + ): Uint8Array; decode(frame: MTPBytesInput): ParsedFrame; format(frame: MTPBytesInput): string; } -export function encode(type: MTPCommunicationType, data: Record, options?: MTPCodecOptions): Uint8Array { +export function encode( + type: MTPCommunicationType, + data: Record, + options?: MTPCodecOptions, +): Uint8Array { return bindings.build_frame(type, data, options ?? {}); } @@ -184,6 +247,12 @@ export interface MTPKeyringKeys { sigClSecretKey: Uint8Array; } +export interface MTPPublicKeyBundleKeys { + kemPublicKey: Uint8Array; + sigPqPublicKey: Uint8Array; + sigClPublicKey: Uint8Array; +} + export interface MTPClientOptions { url: string; descriptor?: string; @@ -195,8 +264,15 @@ export interface MTPClientOptions { maxMessageSize?: number; authTimeoutMs?: number; pings?: boolean | { intervalMs?: number }; - wasm?: RawBindings.InitInput | Promise | { module_or_path: RawBindings.InitInput | Promise }; + wasm?: + | RawBindings.InitInput + | Promise + | { + module_or_path: RawBindings.InitInput | Promise; + }; logger?: (event: MTPLogEvent) => void; + sessionStorage?: MTPSessionStorage; + encryptedDeviceSecretProvider?: MTPEncryptedDeviceSecretProvider; } export type Unsubscribe = () => void; @@ -211,7 +287,10 @@ export interface MTPRequestOptions extends MTPSendOptions { responseType?: MTPCommunicationType; } -type InternalCredentials = Omit & { +type InternalCredentials = Omit< + MTPCredentials, + "clientId" | "keyring" | "hostPublicKey" +> & { clientId: bigint | null; keyringBytes: Uint8Array; hostPublicKey?: Uint8Array; @@ -231,22 +310,31 @@ function emit(logger, event) { } function isErrorType(type) { - return type === "Error" || type.startsWith("Error") || [ - "BadRequest", - "Unauthorized", - "Forbidden", - "NotFound", - "TooManyRequests", - "InternalServerError", - "BadGateway", - "ServiceUnavailable", - "GatewayTimeout", - ].includes(type); + return ( + type === "Error" || + type.startsWith("Error") || + [ + "BadRequest", + "Unauthorized", + "Forbidden", + "NotFound", + "TooManyRequests", + "InternalServerError", + "BadGateway", + "ServiceUnavailable", + "GatewayTimeout", + ].includes(type) + ); } function errorMessage(frame) { const data = frame?.data ?? {}; - return String(data.ErrorMessage ?? data.Error ?? data.Description ?? `Received ${frame?.type ?? "error"} frame`); + return String( + data.ErrorMessage ?? + data.Error ?? + data.Description ?? + `Received ${frame?.type ?? "error"} frame`, + ); } async function storageGet(storage, key) { @@ -404,7 +492,9 @@ function utf8Decode(bytes) { i += 2; } else if (b < 0xf0) { out += String.fromCharCode( - ((b & 0x0f) << 12) | ((bytes[i + 1] & 0x3f) << 6) | (bytes[i + 2] & 0x3f), + ((b & 0x0f) << 12) | + ((bytes[i + 1] & 0x3f) << 6) | + (bytes[i + 2] & 0x3f), ); i += 3; } else { @@ -449,7 +539,11 @@ export function secretKeyFromString(secret) { } const ikm = utf8Encode(trimmed); - return bindings.wasm_derive_encryption_key(ikm, SYMMETRIC_KEY_SALT, SYMMETRIC_KEY_SALT); + return bindings.wasm_derive_encryption_key( + ikm, + SYMMETRIC_KEY_SALT, + SYMMETRIC_KEY_SALT, + ); } function normalizeBytes(value, name) { @@ -483,9 +577,10 @@ function generateKeyringBytes() { } export function keyringToKeys(keyring) { - const bytes = typeof keyring === "string" - ? bytesFromString(keyring, "keyring") - : bytesFrom(keyring, "keyring"); + const bytes = + typeof keyring === "string" + ? bytesFromString(keyring, "keyring") + : bytesFrom(keyring, "keyring"); if (bytes.length < 12) { throw new TypeError("keyring data is too short to contain 6 keys"); @@ -510,11 +605,51 @@ export function keyringToKeys(keyring) { }; } +export function publicKeyBundleToKeys(publicKeyBundle) { + const bytes = + typeof publicKeyBundle === "string" + ? bytesFromString(publicKeyBundle, "publicKeyBundle") + : bytesFrom(publicKeyBundle, "publicKeyBundle"); + + if (bytes.length < 6) { + throw new TypeError("public key bundle data is too short to contain 3 keys"); + } + + let offset = 0; + const readKey = () => { + if (offset + 2 > bytes.length) { + throw new TypeError("public key bundle is truncated"); + } + const len = (bytes[offset] << 8) | bytes[offset + 1]; + offset += 2; + if (offset + len > bytes.length) { + throw new TypeError("public key bundle is truncated"); + } + const key = bytes.slice(offset, offset + len); + offset += len; + return key; + }; + + const result = { + kemPublicKey: readKey(), + sigPqPublicKey: readKey(), + sigClPublicKey: readKey(), + }; + + if (offset !== bytes.length) { + throw new TypeError("public key bundle has trailing data"); + } + + return result; +} + function serializeCredentials(credentials) { return JSON.stringify({ clientId: credentials.clientId?.toString() ?? null, keyring: Array.from(credentials.keyringBytes ?? []), - hostPublicKey: credentials.hostPublicKey ? Array.from(credentials.hostPublicKey) : undefined, + hostPublicKey: credentials.hostPublicKey + ? Array.from(credentials.hostPublicKey) + : undefined, }); } @@ -532,9 +667,10 @@ function deserializeCredentials(credentials) { return { clientId: toBigInt(normalized.clientId), keyringBytes: bytesFrom(keyring, "credentials.keyring"), - hostPublicKey: normalized.hostPublicKey == null - ? undefined - : normalizeBytes(normalized.hostPublicKey, "credentials.hostPublicKey"), + hostPublicKey: + normalized.hostPublicKey == null + ? undefined + : normalizeBytes(normalized.hostPublicKey, "credentials.hostPublicKey"), }; } @@ -567,10 +703,17 @@ function validateOptions(options) { } } } - if (options.maxMessageSize != null && (!Number.isSafeInteger(options.maxMessageSize) || options.maxMessageSize <= 0)) { + if ( + options.maxMessageSize != null && + (!Number.isSafeInteger(options.maxMessageSize) || + options.maxMessageSize <= 0) + ) { throw new TypeError("maxMessageSize must be a positive safe integer"); } - if (options.authTimeoutMs != null && (!Number.isSafeInteger(options.authTimeoutMs) || options.authTimeoutMs <= 0)) { + if ( + options.authTimeoutMs != null && + (!Number.isSafeInteger(options.authTimeoutMs) || options.authTimeoutMs <= 0) + ) { throw new TypeError("authTimeoutMs must be a positive safe integer"); } } @@ -604,10 +747,22 @@ export class MTPClient { readonly crypto = MTPClient.crypto; readonly codec = MTPClient.codec; - private constructor(options: NormalizedMTPClientOptions, client: RawBindings.WasmClient) { + readonly sessionManager: MTPSessionManager; + readonly encryptedDeviceSecretProvider: MTPEncryptedDeviceSecretProvider; + + private constructor( + options: NormalizedMTPClientOptions, + client: RawBindings.WasmClient, + ) { this.#options = options; this.#credentials = deserializeCredentials(options.credentials); this.raw = { client, bindings }; + this.encryptedDeviceSecretProvider = + options.encryptedDeviceSecretProvider ?? + new InMemoryEncryptedDeviceSecretProvider(); + this.sessionManager = new MTPSessionManager( + options.sessionStorage ?? new InMemorySessionStorage(), + ); } static async create(options: MTPClientOptions): Promise { @@ -616,28 +771,31 @@ export class MTPClient { const normalizedOptions = { ...options, - hostPublicKey: options.hostPublicKey == null - ? undefined - : normalizeBytes(options.hostPublicKey, "hostPublicKey"), + hostPublicKey: + options.hostPublicKey == null + ? undefined + : normalizeBytes(options.hostPublicKey, "hostPublicKey"), }; let sdk: MTPClient | undefined; const client = new WasmClient( - (state) => emit(normalizedOptions.logger, { - hint: "info", - type: "state", - data: ConnectionState[state] ?? state, - }), + (state) => + emit(normalizedOptions.logger, { + hint: "info", + type: "state", + data: ConnectionState[state] ?? state, + }), (frame) => { if (sdk) { sdk.#handleFrame(frame); } }, - (error) => emit(normalizedOptions.logger, { - hint: "error", - type: "Error", - error: String(error), - }), + (error) => + emit(normalizedOptions.logger, { + hint: "error", + type: "Error", + error: String(error), + }), ); sdk = new MTPClient(normalizedOptions, client); @@ -648,10 +806,22 @@ export class MTPClient { keyringBytes: generateKeyringBytes(), hostPublicKey: normalizedOptions.hostPublicKey, }; - } else if (!sdk.#credentials.hostPublicKey && normalizedOptions.hostPublicKey) { - sdk.#credentials = { ...sdk.#credentials, hostPublicKey: normalizedOptions.hostPublicKey }; - } else if (!normalizedOptions.hostPublicKey && sdk.#credentials.hostPublicKey) { - sdk.#options = { ...sdk.#options, hostPublicKey: sdk.#credentials.hostPublicKey }; + } else if ( + !sdk.#credentials.hostPublicKey && + normalizedOptions.hostPublicKey + ) { + sdk.#credentials = { + ...sdk.#credentials, + hostPublicKey: normalizedOptions.hostPublicKey, + }; + } else if ( + !normalizedOptions.hostPublicKey && + sdk.#credentials.hostPublicKey + ) { + sdk.#options = { + ...sdk.#options, + hostPublicKey: sdk.#credentials.hostPublicKey, + }; } return sdk; } @@ -660,7 +830,9 @@ export class MTPClient { return WasmClient.is_supported(); } - static async init(wasm?: MTPClientOptions["wasm"]): Promise>> { + static async init( + wasm?: MTPClientOptions["wasm"], + ): Promise>> { wasmInitPromise ??= initWasm(wasm); return await wasmInitPromise; } @@ -729,10 +901,17 @@ export class MTPClient { async #connectAuthenticated() { if (!this.#options.hostPublicKey) { - throw new Error("MTPClient.connect requires hostPublicKey for authenticated connections"); + throw new Error( + "MTPClient.connect requires hostPublicKey for authenticated connections", + ); } - if (!this.#credentials?.keyringBytes?.length || this.#credentials.clientId == null) { - throw new Error("MTPClient.connect requires credentials with clientId and keyring"); + if ( + !this.#credentials?.keyringBytes?.length || + this.#credentials.clientId == null + ) { + throw new Error( + "MTPClient.connect requires credentials with clientId and keyring", + ); } const config = this.#connectionConfig(); @@ -810,7 +989,8 @@ export class MTPClient { this.raw.client.stop_protocol_pings(); return; } - const intervalMs = typeof pings === "object" ? pings.intervalMs ?? 30_000 : 30_000; + const intervalMs = + typeof pings === "object" ? (pings.intervalMs ?? 30_000) : 30_000; this.raw.client.start_protocol_pings(intervalMs, clientId); } @@ -819,7 +999,9 @@ export class MTPClient { return typeOrFrame; } if (typeof typeOrFrame !== "string" || !typeOrFrame) { - throw new TypeError("message type must be a non-empty string or Uint8Array frame"); + throw new TypeError( + "message type must be a non-empty string or Uint8Array frame", + ); } if (data == null || typeof data !== "object" || Array.isArray(data)) { throw new TypeError("message data must be an object"); @@ -828,15 +1010,37 @@ export class MTPClient { } async send(message: Uint8Array): Promise; - async send(type: MTPCommunicationType, data: Record, options?: MTPSendOptions): Promise; - async send(typeOrFrame: Uint8Array | MTPCommunicationType, data?: Record, options?: MTPSendOptions): Promise { + async send( + type: MTPCommunicationType, + data: Record, + options?: MTPSendOptions, + ): Promise; + async send( + typeOrFrame: Uint8Array | MTPCommunicationType, + data?: Record, + options?: MTPSendOptions, + ): Promise { const message = this.#buildFrame(typeOrFrame, data, options); try { const frame = this.raw.bindings.parse_frame(message); - emit(this.#options.logger, isErrorType(frame.type) - ? { hint: "error", type: frame.type, error: errorMessage(frame), data: frame.data, direction: "send" } - : { hint: "info", type: frame.type, data: frame.data, direction: "send" }); + emit( + this.#options.logger, + isErrorType(frame.type) + ? { + hint: "error", + type: frame.type, + error: errorMessage(frame), + data: frame.data, + direction: "send", + } + : { + hint: "info", + type: frame.type, + data: frame.data, + direction: "send", + }, + ); } catch (error) { emit(this.#options.logger, { hint: "error", @@ -849,15 +1053,41 @@ export class MTPClient { await this.raw.client.send(message); } - async request(message: Uint8Array, data?: never, options?: MTPRequestOptions): Promise; - async request(type: MTPCommunicationType, data: Record, options?: MTPRequestOptions): Promise; - async request(typeOrFrame: Uint8Array | MTPCommunicationType, data?: Record, options: MTPRequestOptions = {}): Promise { + async request( + message: Uint8Array, + data?: never, + options?: MTPRequestOptions, + ): Promise; + async request( + type: MTPCommunicationType, + data: Record, + options?: MTPRequestOptions, + ): Promise; + async request( + typeOrFrame: Uint8Array | MTPCommunicationType, + data?: Record, + options: MTPRequestOptions = {}, + ): Promise { const frame = this.#buildFrame(typeOrFrame, data, options); try { const parsed = this.raw.bindings.parse_frame(frame); - emit(this.#options.logger, isErrorType(parsed.type) - ? { hint: "error", type: parsed.type, error: errorMessage(parsed), data: parsed.data, direction: "send" } - : { hint: "info", type: parsed.type, data: parsed.data, direction: "send" }); + emit( + this.#options.logger, + isErrorType(parsed.type) + ? { + hint: "error", + type: parsed.type, + error: errorMessage(parsed), + data: parsed.data, + direction: "send", + } + : { + hint: "info", + type: parsed.type, + data: parsed.data, + direction: "send", + }, + ); } catch (error) { emit(this.#options.logger, { hint: "error", @@ -869,7 +1099,10 @@ export class MTPClient { return await this.raw.client.request(frame, options.responseType ?? null); } - subscribe(type: MTPCommunicationType, handler: (message: ParsedFrame) => void): Unsubscribe { + subscribe( + type: MTPCommunicationType, + handler: (message: ParsedFrame) => void, + ): Unsubscribe { if (typeof type !== "string" || !type) { throw new TypeError("subscription type must be a non-empty string"); } @@ -899,6 +1132,420 @@ export class MTPClient { } } + #getKemPublicKey(): Uint8Array { + if (!this.#credentials?.keyringBytes?.length) { + throw new Error("No keyring available"); + } + const keys = keyringToKeys(this.#credentials.keyringBytes); + return keys.kemPublicKey; + } + + #getKemSecretKey(): Uint8Array { + if (!this.#credentials?.keyringBytes?.length) { + throw new Error("No keyring available"); + } + const keys = keyringToKeys(this.#credentials.keyringBytes); + return keys.kemSecretKey; + } + + async sendEncrypted( + type: number | string, + data: Record, + options: MTPSendOptions & { + recipientClientId: bigint | number | string; + recipientPublicKey: string | MTPBytesInput; + senderUserId?: string; + recipientUserId?: string; + recipientDeviceId?: string; + }, + ): Promise { + const ownId = this.#credentials?.clientId; + if (ownId == null) { + throw new Error("Client not registered"); + } + if (options.recipientClientId == null) { + throw new Error("recipientClientId is required"); + } + const recipientClientId = BigInt(options.recipientClientId); + + const plaintext = this.raw.bindings.build_frame(type as string, data, { + sender: ownId, + receiver: recipientClientId, + ...options, + }); + + let session = await this.sessionManager.getSession( + ownId, + recipientClientId, + ); + let kemCiphertext: Uint8Array | undefined; + + if (!session) { + if (options.recipientPublicKey == null) { + throw new Error("recipientPublicKey is required for new encrypted sessions"); + } + const recipientPublicKey = publicKeyBundleToKeys( + options.recipientPublicKey, + ); + const enc = bindings.wasm_kem_encapsulate( + recipientPublicKey.kemPublicKey, + ); + kemCiphertext = enc.ciphertext; + const conversationId = getConversationId(ownId, recipientClientId); + + session = await this.sessionManager.createSession({ + ownClientId: ownId, + peerClientId: recipientClientId, + peerPublicKey: recipientPublicKey.kemPublicKey, + sharedSecret: enc.shared_secret, + role: "initiator", + transcriptContext: { + senderUserId: (options as { senderUserId?: string }).senderUserId, + senderClientId: ownId, + recipientUserId: options.recipientUserId, + recipientClientId, + recipientPublicKey: recipientPublicKey.kemPublicKey, + kemCiphertext, + conversationId, + }, + }); + enc.shared_secret.fill(0); + } + + const { payload, session: newSession } = await encryptPayload({ + plaintext, + session, + kemCiphertext, + }); + + await this.sessionManager.saveSession(newSession); + + const messageId = String(Date.now()); + const createdAt = Date.now(); + const senderUserId = (options as { senderUserId?: string }).senderUserId; + const frame = this.raw.bindings.build_frame( + "EncryptedMessage", + { + MessageId: messageId, + ConversationId: session.conversationId, + SenderClientId: ownId, + RecipientClientId: recipientClientId, + SenderUserId: senderUserId, + RecipientUserId: options.recipientUserId, + CreatedAt: createdAt, + EncryptionVersion: 1, + EncryptedPayload: payload, + }, + { + sender: ownId, + receiver: recipientClientId, + }, + ); + await this.raw.client.send(frame); + + if (senderUserId && options.recipientUserId) { + const ownKemPublicKey = this.#getKemPublicKey(); + const archiveEnc = bindings.wasm_kem_encapsulate(ownKemPublicKey); + const archiveSession = await this.sessionManager.createSession({ + ownClientId: ownId, + peerClientId: ownId, + peerPublicKey: ownKemPublicKey, + sharedSecret: archiveEnc.shared_secret, + role: "initiator", + transcriptContext: { + senderUserId, + senderClientId: ownId, + recipientUserId: options.recipientUserId, + recipientClientId: ownId, + recipientPublicKey: ownKemPublicKey, + kemCiphertext: archiveEnc.ciphertext, + conversationId: `archive:${session.conversationId}:${messageId}`, + }, + }); + archiveEnc.shared_secret.fill(0); + const { payload: archivePayload } = await encryptPayload({ + plaintext, + session: archiveSession, + kemCiphertext: archiveEnc.ciphertext, + }); + const archiveFrame = this.raw.bindings.build_frame( + "EncryptedMessage", + { + MessageId: `${messageId}:sender`, + ConversationId: session.conversationId, + SenderClientId: ownId, + RecipientClientId: ownId, + SenderUserId: senderUserId, + RecipientUserId: options.recipientUserId, + CreatedAt: createdAt, + EncryptionVersion: 1, + EncryptedPayload: archivePayload, + }, + { + sender: ownId, + receiver: ownId, + }, + ); + await this.raw.client.send(archiveFrame); + } + } + + subscribeEncrypted( + type: number | string, + handler: (data: unknown, meta: ParsedFrame) => void | Promise, + ): Unsubscribe; + subscribeEncrypted( + handler: (data: { + type: string; + data: Record; + sender?: bigint; + receiver?: bigint; + }) => void | Promise, + ): Unsubscribe; + subscribeEncrypted( + typeOrHandler: + | number + | string + | ((data: { + type: string; + data: Record; + sender?: bigint; + receiver?: bigint; + }) => void | Promise), + maybeHandler?: (data: unknown, meta: ParsedFrame) => void | Promise, + ): Unsubscribe { + const expectedInnerType = + typeof typeOrHandler === "function" ? null : String(typeOrHandler); + const legacyHandler = + typeof typeOrHandler === "function" ? typeOrHandler : null; + const sub = this.raw.client.subscribe( + "EncryptedMessage", + async (frame: ParsedFrame) => { + const raw = + frame.data?.["encryptedPayload"] ?? + frame.data?.["EncryptedPayload"] ?? + frame.data?.["encrypted_payload"]; + if (!raw) return; + + let payloadBytes: Uint8Array; + if (raw instanceof Uint8Array) { + payloadBytes = raw; + } else if (Array.isArray(raw)) { + payloadBytes = new Uint8Array(raw); + } else { + return; + } + + try { + const parsed = parseEncryptedMessage(payloadBytes); + + const ownId = this.#credentials?.clientId; + if (ownId == null) return; + if (parsed.recipientClientId !== ownId) return; + + const peerClientId = parsed.senderClientId; + let session = await this.sessionManager.getSession( + ownId, + peerClientId, + ); + + if (!session) { + if (!(parsed.flags & FLAG_INIT) || !parsed.kemCiphertext) { + return; + } + + const ownKemSecret = this.#getKemSecretKey(); + const sharedSecret = bindings.wasm_kem_decapsulate( + ownKemSecret, + parsed.kemCiphertext, + ); + + session = await this.sessionManager.createSession({ + ownClientId: ownId, + peerClientId, + peerPublicKey: new Uint8Array(0), + sharedSecret, + role: "receiver", + transcriptContext: { + senderUserId: String( + frame.data?.["SenderUserId"] ?? + frame.data?.["senderUserId"] ?? + "", + ), + senderClientId: peerClientId, + recipientUserId: String( + frame.data?.["RecipientUserId"] ?? + frame.data?.["recipientUserId"] ?? + "", + ), + recipientClientId: ownId, + recipientPublicKey: this.#getKemPublicKey(), + kemCiphertext: parsed.kemCiphertext, + conversationId: getConversationId(peerClientId, ownId), + }, + }); + sharedSecret.fill(0); + } + + const { plaintext, session: newSession } = await decryptPayload({ + payload: payloadBytes, + session, + expectedRecipientClientId: ownId, + }); + + await this.sessionManager.saveSession(newSession); + + let parsedFrame: ParsedFrame; + try { + parsedFrame = this.raw.bindings.parse_frame(plaintext); + } catch { + return; + } + + if (expectedInnerType && parsedFrame.type !== expectedInnerType) { + return; + } + if (legacyHandler) { + await legacyHandler({ + type: parsedFrame.type, + data: parsedFrame.data, + sender: parsedFrame.sender, + receiver: parsedFrame.receiver, + }); + } else if (maybeHandler) { + await maybeHandler(parsedFrame.data, parsedFrame); + } + } catch (e) { + emit(this.#options.logger, { + hint: "error", + type: "E2EE", + error: String(e), + direction: "recv", + }); + } + }, + ); + + return () => this.raw.client.unsubscribe(sub); + } + + async decryptEncryptedRecord( + frameData: Record, + ): Promise { + const raw = + frameData["encryptedPayload"] ?? + frameData["EncryptedPayload"] ?? + frameData["encrypted_payload"]; + if (!raw) throw new Error("EncryptedPayload is required"); + + const payloadBytes = + raw instanceof Uint8Array + ? raw + : Array.isArray(raw) + ? new Uint8Array(raw) + : bytesFrom(raw as MTPBytesInput, "EncryptedPayload"); + + const parsed = parseEncryptedMessage(payloadBytes); + const ownId = this.#credentials?.clientId; + if (ownId == null) throw new Error("Client not registered"); + if (parsed.recipientClientId !== ownId) { + throw new Error("Encrypted message recipient mismatch"); + } + + const peerClientId = parsed.senderClientId; + let session = await this.sessionManager.getSession(ownId, peerClientId); + + const isSenderArchive = + parsed.senderClientId === ownId && parsed.recipientClientId === ownId; + + if (isSenderArchive && (parsed.flags & FLAG_INIT) && parsed.kemCiphertext) { + const ownKemSecret = this.#getKemSecretKey(); + const sharedSecret = bindings.wasm_kem_decapsulate( + ownKemSecret, + parsed.kemCiphertext, + ); + const archiveMessageId = String( + frameData["MessageId"] ?? frameData["messageId"] ?? "", + ).replace(/:sender$/, ""); + session = await this.sessionManager.createSession({ + ownClientId: ownId, + peerClientId: ownId, + peerPublicKey: this.#getKemPublicKey(), + sharedSecret, + role: "receiver", + transcriptContext: { + senderUserId: String( + frameData["SenderUserId"] ?? frameData["senderUserId"] ?? "", + ), + senderClientId: ownId, + recipientUserId: String( + frameData["RecipientUserId"] ?? frameData["recipientUserId"] ?? "", + ), + recipientClientId: ownId, + recipientPublicKey: this.#getKemPublicKey(), + kemCiphertext: parsed.kemCiphertext, + conversationId: `archive:${String(frameData["ConversationId"] ?? frameData["conversationId"] ?? "")}:${archiveMessageId}`, + }, + }); + sharedSecret.fill(0); + } else if (!session) { + if (!(parsed.flags & FLAG_INIT) || !parsed.kemCiphertext) { + throw new Error("No session for non-init encrypted message"); + } + const ownKemSecret = this.#getKemSecretKey(); + const sharedSecret = bindings.wasm_kem_decapsulate( + ownKemSecret, + parsed.kemCiphertext, + ); + + session = await this.sessionManager.createSession({ + ownClientId: ownId, + peerClientId, + peerPublicKey: new Uint8Array(0), + sharedSecret, + role: "receiver", + transcriptContext: { + senderUserId: String( + frameData["SenderUserId"] ?? frameData["senderUserId"] ?? "", + ), + senderClientId: peerClientId, + recipientUserId: String( + frameData["RecipientUserId"] ?? frameData["recipientUserId"] ?? "", + ), + recipientClientId: ownId, + recipientPublicKey: this.#getKemPublicKey(), + kemCiphertext: parsed.kemCiphertext, + conversationId: getConversationId(peerClientId, ownId), + }, + }); + sharedSecret.fill(0); + } + + const { plaintext, session: newSession } = await decryptPayload({ + payload: payloadBytes, + session, + expectedRecipientClientId: ownId, + }); + if (!isSenderArchive) { + await this.sessionManager.saveSession(newSession); + } + return this.raw.bindings.parse_frame(plaintext); + } + + async setEncryptedDeviceSecret( + record: EncryptedDeviceSecretRecord, + ): Promise { + await this.encryptedDeviceSecretProvider.setEncryptedDeviceSecret(record); + } + + async getEncryptedDeviceSecret(query: { + userId: string; + deviceId?: string; + secretId?: string; + }): Promise { + return this.encryptedDeviceSecretProvider.getEncryptedDeviceSecret(query); + } + disconnect(): void { this.raw.client.stop_protocol_pings(); this.raw.client.disconnect(); @@ -906,3 +1553,37 @@ export class MTPClient { } export { ConnectionState, bindings as raw }; + +// E2EE exports +export type { + MTPSessionState, + MTPSessionStorage, + MTPSessionTranscriptContext, +} from "./session"; +export { + MTPSessionManager, + InMemorySessionStorage, + getConversationId, + deriveSessionKeys, + buildSessionTranscript, +} from "./session.js"; +export { MTPRatchet } from "./ratchet.js"; +export type { RatchetStep } from "./ratchet.js"; +export { + serializeEncryptedMessage, + parseEncryptedMessage, + encryptPayload, + decryptPayload, + MTP_E2EE_VERSION, + FLAG_INIT, + MAX_RATCHET_SKIP, +} from "./encrypted-message.js"; +export type { + EncryptedMessageHeader, + SerializedEncryptedMessage, +} from "./encrypted-message"; +export type { + EncryptedDeviceSecretRecord, + MTPEncryptedDeviceSecretProvider, +} from "./encrypted-device-secret"; +export { InMemoryEncryptedDeviceSecretProvider } from "./encrypted-device-secret.js"; diff --git a/src/sdk/ratchet.ts b/src/sdk/ratchet.ts new file mode 100644 index 0000000..9ae8242 --- /dev/null +++ b/src/sdk/ratchet.ts @@ -0,0 +1,70 @@ +import * as bindings from "mtp/raw"; + +function utf8Encode(text: string): Uint8Array { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(text); + } + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(text, "utf-8")); + } + const bytes = new Uint8Array(text.length * 4); + let len = 0; + for (let i = 0; i < text.length; i += 1) { + const code = text.codePointAt(i) as number; + if (code < 0x80) { + bytes[len++] = code; + } else if (code < 0x800) { + bytes[len++] = 0xc0 | (code >> 6); + bytes[len++] = 0x80 | (code & 0x3f); + } else if (code < 0x10000) { + bytes[len++] = 0xe0 | (code >> 12); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + } else { + bytes[len++] = 0xf0 | (code >> 18); + bytes[len++] = 0x80 | ((code >> 12) & 0x3f); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + i += 1; + } + } + return bytes.subarray(0, len); +} + +const HKDF_MESSAGE_KEY = "mtp-e2ee-v1-message-key"; +const HKDF_NEXT_CHAIN = "mtp-e2ee-v1-next-chain"; + +export interface RatchetStep { + key: Uint8Array; + chainKey: Uint8Array; +} + +export class MTPRatchet { + static async stepSend(chainKey: Uint8Array): Promise { + return this.step(chainKey); + } + + static async stepRecv(chainKey: Uint8Array): Promise { + return this.step(chainKey); + } + + static async step(chainKey: Uint8Array): Promise { + const messageKey = bindings.wasm_hkdf_expand( + chainKey, + new Uint8Array(0), + utf8Encode(HKDF_MESSAGE_KEY), + 32, + ); + const nextChainKey = bindings.wasm_hkdf_expand( + chainKey, + new Uint8Array(0), + utf8Encode(HKDF_NEXT_CHAIN), + 32, + ); + + return { + key: messageKey, + chainKey: nextChainKey, + }; + } +} diff --git a/src/sdk/session.ts b/src/sdk/session.ts new file mode 100644 index 0000000..c1ad0a4 --- /dev/null +++ b/src/sdk/session.ts @@ -0,0 +1,249 @@ +import * as bindings from "mtp/raw"; + +function utf8Encode(text: string): Uint8Array { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(text); + } + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(text, "utf-8")); + } + const bytes = new Uint8Array(text.length * 4); + let len = 0; + for (let i = 0; i < text.length; i += 1) { + const code = text.codePointAt(i) as number; + if (code < 0x80) { + bytes[len++] = code; + } else if (code < 0x800) { + bytes[len++] = 0xc0 | (code >> 6); + bytes[len++] = 0x80 | (code & 0x3f); + } else if (code < 0x10000) { + bytes[len++] = 0xe0 | (code >> 12); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + } else { + bytes[len++] = 0xf0 | (code >> 18); + bytes[len++] = 0x80 | ((code >> 12) & 0x3f); + bytes[len++] = 0x80 | ((code >> 6) & 0x3f); + bytes[len++] = 0x80 | (code & 0x3f); + i += 1; + } + } + return bytes.subarray(0, len); +} + +export const HKDF_SALT_ROOT = "mtp-e2ee-v1-root"; +const HKDF_INITIATOR_SEND = "mtp-e2ee-v1-initiator-send"; +const HKDF_INITIATOR_RECV = "mtp-e2ee-v1-initiator-recv"; + +export interface MTPSessionTranscriptContext { + senderUserId?: string; + senderClientId: bigint; + recipientUserId?: string; + recipientClientId: bigint; + recipientPublicKey: Uint8Array; + kemCiphertext: Uint8Array; + conversationId: string; +} + +export interface MTPSessionState { + version: 1; + conversationId: string; + ownClientId: bigint; + peerClientId: bigint; + peerPublicKey: Uint8Array; + sendChainKey: Uint8Array; + recvChainKey: Uint8Array; + sendCount: number; + recvCount: number; + createdAt: number; + updatedAt: number; +} + +export interface MTPSessionStorage { + getSession(conversationId: string): Promise; + setSession(state: MTPSessionState): Promise; + deleteSession(conversationId: string): Promise; +} + +export class InMemorySessionStorage implements MTPSessionStorage { + private store = new Map(); + + async getSession(conversationId: string): Promise { + return this.store.get(conversationId) ?? null; + } + + async setSession(state: MTPSessionState): Promise { + this.store.set(state.conversationId, { ...state }); + } + + async deleteSession(conversationId: string): Promise { + this.store.delete(conversationId); + } +} + +function writeU32BE(value: number): Uint8Array { + return new Uint8Array([ + (value >>> 24) & 0xff, + (value >>> 16) & 0xff, + (value >>> 8) & 0xff, + value & 0xff, + ]); +} + +function writeU64BE(value: bigint): Uint8Array { + if (value < 0n || value > 0xffff_ffff_ffff_ffffn) + throw new Error("u64 out of range"); + const buf = new Uint8Array(8); + for (let i = 7; i >= 0; i -= 1) { + buf[i] = Number(value & 0xffn); + value >>= 8n; + } + return buf; +} + +function concatBytes(parts: Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +function transcriptField(label: string, value: Uint8Array): Uint8Array { + const labelBytes = utf8Encode(label); + return concatBytes([ + writeU32BE(labelBytes.length), + labelBytes, + writeU32BE(value.length), + value, + ]); +} + +export function buildSessionTranscript( + args: MTPSessionTranscriptContext, +): Uint8Array { + const recipientPublicKeyHash = bindings.wasm_sha256(args.recipientPublicKey); + const kemHash = bindings.wasm_sha256(args.kemCiphertext); + return concatBytes([ + transcriptField("domain", utf8Encode("mtp-e2ee-session-transcript-v1")), + transcriptField("version", utf8Encode("1")), + transcriptField("senderUserId", utf8Encode(args.senderUserId ?? "")), + transcriptField("senderClientId", writeU64BE(args.senderClientId)), + transcriptField("recipientUserId", utf8Encode(args.recipientUserId ?? "")), + transcriptField("recipientClientId", writeU64BE(args.recipientClientId)), + transcriptField("recipientPublicKeyHash", recipientPublicKeyHash), + transcriptField("kemCiphertextHash", kemHash), + transcriptField("conversationId", utf8Encode(args.conversationId)), + ]); +} + +export async function deriveSessionKeys( + sharedSecret: Uint8Array, + transcript: Uint8Array = new Uint8Array(0), +): Promise<{ + root: Uint8Array; + initiatorSend: Uint8Array; + initiatorRecv: Uint8Array; +}> { + const rootInfo = concatBytes([utf8Encode(HKDF_SALT_ROOT), transcript]); + const root = bindings.wasm_hkdf_expand( + sharedSecret, + new Uint8Array(0), + rootInfo, + 32, + ); + const initiatorSend = bindings.wasm_hkdf_expand( + root, + new Uint8Array(0), + utf8Encode(HKDF_INITIATOR_SEND), + 32, + ); + const initiatorRecv = bindings.wasm_hkdf_expand( + root, + new Uint8Array(0), + utf8Encode(HKDF_INITIATOR_RECV), + 32, + ); + return { root, initiatorSend, initiatorRecv }; +} + +export function getConversationId( + ownClientId: bigint, + peerClientId: bigint, +): string { + const ids = [ownClientId, peerClientId].sort((a, b) => + a < b ? -1 : a > b ? 1 : 0, + ); + return `${ids[0].toString(16)}:${ids[1].toString(16)}`; +} + +export class MTPSessionManager { + constructor(private storage: MTPSessionStorage) {} + + getConversationId( + ownClientId: bigint, + peerClientId: bigint, + ): Promise { + return Promise.resolve(getConversationId(ownClientId, peerClientId)); + } + + async getSession( + ownClientId: bigint, + peerClientId: bigint, + ): Promise { + return this.storage.getSession( + getConversationId(ownClientId, peerClientId), + ); + } + + async saveSession(state: MTPSessionState): Promise { + await this.storage.setSession({ ...state, updatedAt: Date.now() }); + } + + async deleteSession( + ownClientId: bigint, + peerClientId: bigint, + ): Promise { + await this.storage.deleteSession( + getConversationId(ownClientId, peerClientId), + ); + } + + async createSession(args: { + ownClientId: bigint; + peerClientId: bigint; + peerPublicKey: Uint8Array; + sharedSecret: Uint8Array; + role: "initiator" | "receiver"; + transcript?: Uint8Array; + transcriptContext?: MTPSessionTranscriptContext; + }): Promise { + const transcript = + args.transcript ?? + (args.transcriptContext + ? buildSessionTranscript(args.transcriptContext) + : undefined); + const { root, initiatorSend, initiatorRecv } = await deriveSessionKeys( + args.sharedSecret, + transcript, + ); + const now = Date.now(); + const state: MTPSessionState = { + version: 1, + conversationId: getConversationId(args.ownClientId, args.peerClientId), + ownClientId: args.ownClientId, + peerClientId: args.peerClientId, + peerPublicKey: args.peerPublicKey, + sendChainKey: args.role === "initiator" ? initiatorSend : initiatorRecv, + recvChainKey: args.role === "initiator" ? initiatorRecv : initiatorSend, + sendCount: 0, + recvCount: 0, + createdAt: now, + updatedAt: now, + }; + root.fill(0); + return state; + } +} diff --git a/test/e2ee.mjs b/test/e2ee.mjs new file mode 100644 index 0000000..fe05930 --- /dev/null +++ b/test/e2ee.mjs @@ -0,0 +1,503 @@ +import { initSync } from "../dist/raw/index.js"; +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const wasmPath = path.resolve(__dirname, "../wasm/pkg/mtp_wasm_bg.wasm"); +const wasmBytes = fs.readFileSync(wasmPath); +const wasmModule = new WebAssembly.Module(wasmBytes); +initSync(wasmModule); + +const sdk = await import("../dist/sdk/index.js"); +const { MTPRatchet } = await import("../dist/sdk/ratchet.js"); +const { + serializeEncryptedMessage, + parseEncryptedMessage, + encryptPayload, + decryptPayload, + FLAG_INIT, + MTP_E2EE_VERSION, +} = await import("../dist/sdk/encrypted-message.js"); +const { + MTPSessionManager, + InMemorySessionStorage, + deriveSessionKeys, + getConversationId, +} = await import("../dist/sdk/session.js"); + +const bindings = sdk.raw; + +function concat(...arrays) { + const totalLen = arrays.reduce((sum, a) => sum + a.length, 0); + const result = new Uint8Array(totalLen); + let offset = 0; + for (const a of arrays) { + result.set(a, offset); + offset += a.length; + } + return result; +} + +function setupSessions(sharedSecret, aliceId = 1n, bobId = 2n) { + const aliceStorage = new InMemorySessionStorage(); + const aliceManager = new MTPSessionManager(aliceStorage); + const bobStorage = new InMemorySessionStorage(); + const bobManager = new MTPSessionManager(bobStorage); + + return { + aliceManager, + aliceStorage, + bobManager, + bobStorage, + async initSessions() { + const { initiatorSend, initiatorRecv } = await deriveSessionKeys( + sharedSecret, + new Uint8Array(0), + ); + const aliceSession = await aliceManager.createSession({ + ownClientId: aliceId, + peerClientId: bobId, + peerPublicKey: new Uint8Array(32), + sharedSecret, + role: "initiator", + }); + const bobSession = await bobManager.createSession({ + ownClientId: bobId, + peerClientId: aliceId, + peerPublicKey: new Uint8Array(32), + sharedSecret, + role: "receiver", + }); + return { aliceSession, bobSession, initiatorSend, initiatorRecv }; + }, + }; +} + +await describe("E2EE Session Derivation", async () => { + await it("Both sides derive same shared secret", async () => { + const sharedSecret = sdk.crypto.sha256(new Uint8Array([1, 2, 3, 4, 5])); + const transcript = new Uint8Array(0); + + const aliceKeys = await deriveSessionKeys(sharedSecret, transcript); + const bobKeys = await deriveSessionKeys(sharedSecret, transcript); + + // Deterministic: same inputs → same outputs + assert.deepEqual(aliceKeys.initiatorSend, bobKeys.initiatorSend); + assert.deepEqual(aliceKeys.initiatorRecv, bobKeys.initiatorRecv); + + // Init and recv keys are different + assert.notDeepEqual(aliceKeys.initiatorSend, aliceKeys.initiatorRecv); + }); + + await it("Session manager assigns correct chain keys per role", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([1])); + const { initSessions } = setupSessions(ss); + const { aliceSession, bobSession, initiatorSend, initiatorRecv } = + await initSessions(); + + // Alice (initiator): send = initiatorSend, recv = initiatorRecv + assert.deepEqual(aliceSession.sendChainKey, initiatorSend); + assert.deepEqual(aliceSession.recvChainKey, initiatorRecv); + + // Bob (receiver): send = initiatorRecv, recv = initiatorSend + assert.deepEqual(bobSession.sendChainKey, initiatorRecv); + assert.deepEqual(bobSession.recvChainKey, initiatorSend); + + // Alice's send chain = Bob's recv chain + assert.deepEqual(aliceSession.sendChainKey, bobSession.recvChainKey); + // Alice's recv chain = Bob's send chain + assert.deepEqual(aliceSession.recvChainKey, bobSession.sendChainKey); + }); + + await it("Different transcripts produce different keys", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([99])); + const aliceKeys1 = await deriveSessionKeys(ss, new Uint8Array(0)); + const aliceKeys2 = await deriveSessionKeys( + ss, + sdk.crypto.sha256(new Uint8Array([42])), + ); + assert.notDeepEqual(aliceKeys1.initiatorSend, aliceKeys2.initiatorSend); + }); +}); + +await describe("E2EE Ratchet", async () => { + await it("Repeated sends produce different message keys", async () => { + const chainKey = sdk.crypto.sha256(new Uint8Array([42])); + const step1 = await MTPRatchet.step(chainKey); + const step2 = await MTPRatchet.step(step1.chainKey); + const step3 = await MTPRatchet.step(step2.chainKey); + + assert.notDeepEqual(step1.key, step2.key); + assert.notDeepEqual(step2.key, step3.key); + assert.notDeepEqual(step1.key, step3.key); + assert.notDeepEqual(chainKey, step1.chainKey); + }); + + await it("Receiver can decrypt messages sent by sender in order", async () => { + const chainKey = sdk.crypto.sha256(new Uint8Array([7])); + + const send1 = await MTPRatchet.step(chainKey); + const send2 = await MTPRatchet.step(send1.chainKey); + const send3 = await MTPRatchet.step(send2.chainKey); + + const recv1 = await MTPRatchet.step(chainKey); + const recv2 = await MTPRatchet.step(recv1.chainKey); + const recv3 = await MTPRatchet.step(recv2.chainKey); + + assert.deepEqual(send1.key, recv1.key); + assert.deepEqual(send2.key, recv2.key); + assert.deepEqual(send3.key, recv3.key); + }); +}); + +await describe("E2EE Serialization", async () => { + await it("Roundtrips a basic message", () => { + const msg = { + header: { + version: 1, + flags: 0, + senderClientId: 0x1234567890abcdefn, + recipientClientId: 0xfedcba0987654321n, + messageNumber: 42, + }, + aeadPayload: new Uint8Array([1, 2, 3, 4, 5]), + }; + const bytes = serializeEncryptedMessage(msg); + const parsed = parseEncryptedMessage(bytes); + assert.equal(parsed.header.version, 1); + assert.equal(parsed.header.flags, 0); + assert.equal(parsed.header.senderClientId, msg.header.senderClientId); + assert.equal(parsed.header.recipientClientId, msg.header.recipientClientId); + assert.equal(parsed.header.messageNumber, 42); + assert.equal(parsed.header.kemCiphertext, undefined); + assert.deepEqual(parsed.aeadPayload, msg.aeadPayload); + }); + + await it("Roundtrips an init message with KEM ciphertext", () => { + const msg = { + header: { + version: 1, + flags: FLAG_INIT, + senderClientId: 1n, + recipientClientId: 2n, + messageNumber: 0, + kemCiphertext: new Uint8Array([0xde, 0xad, 0xbe, 0xef]), + }, + aeadPayload: new Uint8Array([10, 20, 30]), + }; + const bytes = serializeEncryptedMessage(msg); + const parsed = parseEncryptedMessage(bytes); + assert.equal(parsed.header.flags & FLAG_INIT, FLAG_INIT); + assert.deepEqual(parsed.header.kemCiphertext, msg.header.kemCiphertext); + }); + + await it("Roundtrip: serialize(parse(x)) === x", () => { + const msg = { + header: { + version: 1, + flags: 0, + senderClientId: 0xaaaabbbbccccddddn, + recipientClientId: 0xffff000011112222n, + messageNumber: 65535, + }, + aeadPayload: new Uint8Array(100).fill(0x42), + }; + const bytes = serializeEncryptedMessage(msg); + const parsed = parseEncryptedMessage(bytes); + const bytes2 = serializeEncryptedMessage(parsed); + assert.deepEqual(bytes, bytes2); + }); + + await it("Rejects malformed payloads", () => { + assert.throws(() => parseEncryptedMessage(new Uint8Array(0))); + assert.throws(() => parseEncryptedMessage(new Uint8Array([0x01]))); + assert.throws(() => + parseEncryptedMessage( + new Uint8Array([0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), + ), + ); + }); + + await it("Rejects unsupported version", () => { + const msg = { + header: { + version: 1, + flags: 0, + senderClientId: 0n, + recipientClientId: 0n, + messageNumber: 0, + }, + aeadPayload: new Uint8Array([1]), + }; + const bytes = serializeEncryptedMessage(msg); + bytes[0] = 99; + assert.throws(() => parseEncryptedMessage(bytes)); + }); + + await it("Rejects trailing data", () => { + const msg = { + header: { + version: 1, + flags: 0, + senderClientId: 0n, + recipientClientId: 0n, + messageNumber: 0, + }, + aeadPayload: new Uint8Array([1]), + }; + const bytes = concat( + serializeEncryptedMessage(msg), + new Uint8Array([0xff]), + ); + assert.throws(() => parseEncryptedMessage(bytes)); + }); +}); + +await describe("E2EE Encrypt/Decrypt", async () => { + await it("Alice encrypts and Bob decrypts successfully", async () => { + const keyring = sdk.crypto.generateKeyring(); + const bobKeys = sdk.crypto.keyringToKeys(keyring); + + // Alice encapsulates to Bob's KEM public key + const enc = sdk.crypto.encapsulate(bobKeys.kemPublicKey); + // Bob decapsulates the ciphertext + const bobSS = sdk.crypto.decapsulate(bobKeys.kemSecretKey, enc.ciphertext); + assert.deepEqual(bobSS, enc.shared_secret); + + const ss = enc.shared_secret; + const { initSessions } = setupSessions(ss); + const { aliceSession, bobSession } = await initSessions(); + + // Alice encrypts a message to Bob + const plaintext = sdk.codec.encode( + "Ping", + { Version: "hello from Alice" }, + { sender: 1n, receiver: 2n }, + ); + + const { payload, session: aliceNewSession } = await encryptPayload({ + plaintext, + session: aliceSession, + kemCiphertext: enc.ciphertext, + }); + + // Bob decrypts the message + const { plaintext: decrypted, session: bobNewSession } = + await decryptPayload({ + payload, + session: bobSession, + }); + + const frame = sdk.codec.decode(decrypted); + assert.equal(frame.type, "Ping"); + assert.equal(frame.data["Version"], "hello from Alice"); + + // Chain keys advanced correctly + assert.deepEqual(aliceNewSession.sendChainKey, bobNewSession.recvChainKey); + assert.equal(aliceNewSession.sendCount, 1); + assert.equal(bobNewSession.recvCount, 1); + assert.notDeepEqual( + aliceNewSession.sendChainKey, + aliceSession.sendChainKey, + ); + }); + + await it("Encrypt-decrypt multiple messages with chain advance", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([1, 2, 3])); + const { initSessions } = setupSessions(ss); + let { aliceSession, bobSession } = await initSessions(); + + // Message 1 + const { payload: p1, session: aliceAfter1 } = await encryptPayload({ + plaintext: sdk.codec.encode("Ping", { Version: "msg1" }), + session: aliceSession, + }); + const { plaintext: d1, session: bobAfter1 } = await decryptPayload({ + payload: p1, + session: bobSession, + }); + assert.equal(sdk.codec.decode(d1).data["Version"], "msg1"); + assert.equal(bobAfter1.recvCount, 1); + + // Message 2 + const { payload: p2, session: aliceAfter2 } = await encryptPayload({ + plaintext: sdk.codec.encode("Ping", { Version: "msg2" }), + session: aliceAfter1, + }); + const { plaintext: d2, session: bobAfter2 } = await decryptPayload({ + payload: p2, + session: bobAfter1, + }); + assert.equal(sdk.codec.decode(d2).data["Version"], "msg2"); + assert.equal(bobAfter2.recvCount, 2); + + // Chain keys match after two messages + assert.deepEqual(aliceAfter2.sendChainKey, bobAfter2.recvChainKey); + assert.equal(aliceAfter2.sendCount, 2); + }); +}); + +await describe("E2EE Public Key Bundle", async () => { + await it("parses public key bundles from GetUserData.PublicKey", () => { + const keyring = sdk.crypto.generateKeyring(); + const keys = sdk.crypto.keyringToKeys(keyring); + const bundle = concat( + new Uint8Array([keys.kemPublicKey.length >> 8, keys.kemPublicKey.length & 0xff]), + keys.kemPublicKey, + new Uint8Array([keys.sigPqPublicKey.length >> 8, keys.sigPqPublicKey.length & 0xff]), + keys.sigPqPublicKey, + new Uint8Array([keys.sigClPublicKey.length >> 8, keys.sigClPublicKey.length & 0xff]), + keys.sigClPublicKey, + ); + + const parsed = sdk.crypto.publicKeyBundleToKeys(bundle); + assert.deepEqual(parsed.kemPublicKey, keys.kemPublicKey); + assert.deepEqual(parsed.sigPqPublicKey, keys.sigPqPublicKey); + assert.deepEqual(parsed.sigClPublicKey, keys.sigClPublicKey); + }); +}); + +await describe("E2EE Session Manager", async () => { + await it("getConversationId is consistent regardless of order", () => { + const id1 = getConversationId(5n, 10n); + const id2 = getConversationId(10n, 5n); + assert.equal(id1, id2); + }); + + await it("MTPSessionManager creates, retrieves, and deletes sessions", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([1, 2, 3])); + const storage = new InMemorySessionStorage(); + const manager = new MTPSessionManager(storage); + + assert.equal(await manager.getSession(1n, 2n), null); + + const session = await manager.createSession({ + ownClientId: 1n, + peerClientId: 2n, + peerPublicKey: new Uint8Array(32), + sharedSecret: ss, + role: "initiator", + }); + assert.equal(session.version, 1); + assert.equal(session.sendCount, 0); + assert.equal(session.recvCount, 0); + + await manager.saveSession(session); + const retrieved = await manager.getSession(1n, 2n); + assert.notEqual(retrieved, null); + assert.equal(retrieved.conversationId, session.conversationId); + + await manager.deleteSession(1n, 2n); + assert.equal(await manager.getSession(1n, 2n), null); + }); +}); + +await describe("E2EE Full Flow: KEM + Session + Ratchet + AEAD", async () => { + await it("Alice encapsulates to Bob, both derive matching sessions, encrypt-decrypt works", async () => { + // Bob generates keyring + const bobKeyring = sdk.crypto.generateKeyring(); + const bobKeys = sdk.crypto.keyringToKeys(bobKeyring); + + // Alice encapsulates to Bob's KEM public key + const enc = sdk.crypto.encapsulate(bobKeys.kemPublicKey); + + // Bob decapsulates + const bobSharedSecret = sdk.crypto.decapsulate( + bobKeys.kemSecretKey, + enc.ciphertext, + ); + assert.deepEqual(bobSharedSecret, enc.shared_secret); + + const ss = enc.shared_secret; + const { initSessions } = setupSessions(ss); + const { aliceSession, bobSession } = await initSessions(); + + // Alice sends encrypted init message with KEM ciphertext + const msg1 = new Uint8Array([0x48, 0x65, 0x6c, 0x6c, 0x6f]); // "Hello" + const { payload: p1, session: aliceAfter1 } = await encryptPayload({ + plaintext: msg1, + session: aliceSession, + kemCiphertext: enc.ciphertext, + }); + + // Bob receives and decrypts + const { plaintext: d1, session: bobAfter1 } = await decryptPayload({ + payload: p1, + session: bobSession, + }); + assert.deepEqual(d1, msg1); + assert.deepEqual(aliceAfter1.sendChainKey, bobAfter1.recvChainKey); + assert.equal(aliceAfter1.sendCount, 1); + assert.equal(bobAfter1.recvCount, 1); + + // Second message (no KEM ciphertext) + const msg2 = new Uint8Array([0x57, 0x6f, 0x72, 0x6c, 0x64]); // "World" + const { payload: p2, session: aliceAfter2 } = await encryptPayload({ + plaintext: msg2, + session: aliceAfter1, + }); + + const { plaintext: d2, session: bobAfter2 } = await decryptPayload({ + payload: p2, + session: bobAfter1, + }); + assert.deepEqual(d2, msg2); + assert.deepEqual(aliceAfter2.sendChainKey, bobAfter2.recvChainKey); + assert.equal(aliceAfter2.sendCount, 2); + assert.equal(bobAfter2.recvCount, 2); + }); +}); + +await describe("E2EE Tamper Detection", async () => { + await it("Rejects modified ciphertext", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([42])); + const { initSessions } = setupSessions(ss); + const { aliceSession, bobSession } = await initSessions(); + + const plaintext = new Uint8Array([0x01, 0x02, 0x03]); + const { payload } = await encryptPayload({ + plaintext, + session: aliceSession, + }); + + // Tamper with AEAD payload + const tampered = new Uint8Array(payload); + tampered[tampered.length - 1] ^= 0xff; + + await assert.rejects( + () => decryptPayload({ payload: tampered, session: bobSession }), + /decrypt failed/, + ); + }); + + await it("Rejects out-of-order message numbers", async () => { + const ss = sdk.crypto.sha256(new Uint8Array([7])); + const { initSessions } = setupSessions(ss); + const { aliceSession, bobSession } = await initSessions(); + + // Send two messages + const { payload: p1, session: aliceAfter1 } = await encryptPayload({ + plaintext: sdk.codec.encode("Ping", { Version: "a" }), + session: aliceSession, + }); + await encryptPayload({ + plaintext: sdk.codec.encode("Ping", { Version: "b" }), + session: aliceAfter1, + }); + + // Bob decrypts p1 + const { session: bobAfter1 } = await decryptPayload({ + payload: p1, + session: bobSession, + }); + + // Now bob expects msgNumber 1, but we try to replay msgNumber 0 + await assert.rejects( + () => decryptPayload({ payload: p1, session: bobAfter1 }), + /replay|out of order/, + ); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 5a697e9..3d94fd4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,17 +12,17 @@ "ignoreDeprecations": "6.0", "paths": { "mtp/raw": ["src/raw/index.ts"], - "mtp/type-map": ["src/type-map/index.ts"] + "mtp/type-map": ["src/type-map/index.ts"], }, "strict": false, "skipLibCheck": true, "isolatedModules": true, - "verbatimModuleSyntax": true + "verbatimModuleSyntax": true, }, "include": [ "src/raw/**/*.ts", "src/sdk/**/*.ts", "src/type-map/**/*.ts", - "src/vite/**/*.ts" - ] + "src/vite/**/*.ts", + ], } diff --git a/type-map/build.rs b/type-map/build.rs index 9a0fafb..41fd144 100755 --- a/type-map/build.rs +++ b/type-map/build.rs @@ -185,10 +185,23 @@ fn main() { serde_yaml::from_str(&content).expect("Failed to parse type-maps.yaml") } Err(_) => { - eprint!("warning: MTP_TYPE_MAPS not set; generating types with reserved entries only"); - Config { - protocol_version: String::new(), - type_maps: BTreeMap::new(), + let manifest_dir = + std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let default_path = manifest_dir.join("../example/type-maps.yaml"); + if default_path.exists() { + println!("cargo:rerun-if-changed={}", default_path.display()); + let content = std::fs::read_to_string(&default_path) + .expect("Failed to read default example/type-maps.yaml"); + serde_yaml::from_str(&content) + .expect("Failed to parse default example/type-maps.yaml") + } else { + eprint!( + "warning: MTP_TYPE_MAPS not set; generating types with reserved entries only" + ); + Config { + protocol_version: String::new(), + type_maps: BTreeMap::new(), + } } } }; @@ -317,6 +330,7 @@ fn generate_comm_type_enum(out: &mut String, user_names: &BTreeSet<&str>) { "#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]" ) .unwrap(); + writeln!(out, "#[allow(clippy::enum_variant_names)]").unwrap(); writeln!(out, "pub enum CommunicationType {{").unwrap(); for entry in RESERVED_COMM_TYPES { @@ -401,6 +415,7 @@ fn generate_data_type_enum(out: &mut String, user_names: &BTreeSet<&str>) { "#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]" ) .unwrap(); + writeln!(out, "#[allow(clippy::enum_variant_names)]").unwrap(); writeln!(out, "pub enum DataType {{").unwrap(); for entry in RESERVED_DATA_TYPES { From 515244ce6664b355f0a89b00c8c18c26c8b95dbd Mon Sep 17 00:00:00 2001 From: Alois Date: Sun, 5 Jul 2026 22:08:32 +0200 Subject: [PATCH 29/97] (fix): build stuff --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a87082..5eb8ab0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1126,9 +1126,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", From 40e942337f4847a94936a6479b0ab0216ece6488 Mon Sep 17 00:00:00 2001 From: Alois Date: Mon, 6 Jul 2026 15:58:33 +0200 Subject: [PATCH 30/97] (fix): thing --- src/sdk/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/index.ts b/src/sdk/index.ts index b7a7331..e890959 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -142,7 +142,7 @@ export const crypto: MTPCrypto = { decryptText: async (key, ciphertext) => { const cipher = new bindings.WasmChaCha20Poly1305(key); try { - const decoded = bytesFromString(ciphertext, "ciphertext"); + const decoded = base64ToBytes(ciphertext); const plaintext = cipher.decrypt(decoded, new Uint8Array(0)); return utf8Decode(plaintext); } finally { From 2126a142f44634ccfdcaa297633fdb63b1d9b46d Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:13:34 +0200 Subject: [PATCH 31/97] insecure clients --- client/src/lib.rs | 2 +- transport/Cargo.toml | 2 +- transport/src/client.rs | 79 ++++++++++++++++++++++++++++++++++++++++- transport/src/host.rs | 21 ++++++++++- 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index 008f77d..e764f55 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -168,7 +168,7 @@ impl MTPClient { */ pub async fn connect(config: ClientConfig) -> Result { let (sender, receiver) = - mtp_transport::connect(&config.url, config.server_cert(), Policy::default()).await?; + mtp_transport::connect(&config.url, config.server_cert(), config.policy).await?; let version_str = format!("{}", PROTOCOL_VERSION); let mut ident = CommunicationValue::new(mtp_codec::CommunicationType::Identification) diff --git a/transport/Cargo.toml b/transport/Cargo.toml index 0af879a..5ff48ac 100644 --- a/transport/Cargo.toml +++ b/transport/Cargo.toml @@ -15,9 +15,9 @@ rustls = { version = "0.23.41" } tokio = { version = "1", features = ["full"] } rustls-native-certs = "0.8.4" log = "0.4" +rcgen = "0.14" [dev-dependencies] -rcgen = "0.14" [[test]] name = "integration" diff --git a/transport/src/client.rs b/transport/src/client.rs index 115b37e..f356ea5 100644 --- a/transport/src/client.rs +++ b/transport/src/client.rs @@ -1,11 +1,65 @@ use std::sync::Arc; use mtp_common::CommunicationError; -use rustls::{ClientConfig as RustlsClientConfig, RootCertStore, pki_types::pem::PemObject}; +use rustls::{ + ClientConfig as RustlsClientConfig, RootCertStore, + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + pki_types::{ServerName, UnixTime, pem::PemObject}, + DigitallySignedStruct, SignatureScheme, +}; use wtransport::{ClientConfig, Endpoint}; use crate::{ConnectionHandle, Policy, Receiver, Sender}; +#[derive(Debug)] +struct NoopCertVerifier; + +impl ServerCertVerifier for NoopCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + vec![ + SignatureScheme::RSA_PKCS1_SHA1, + SignatureScheme::RSA_PKCS1_SHA256, + SignatureScheme::RSA_PKCS1_SHA384, + SignatureScheme::RSA_PKCS1_SHA512, + SignatureScheme::ECDSA_NISTP256_SHA256, + SignatureScheme::ECDSA_NISTP384_SHA384, + SignatureScheme::RSA_PSS_SHA256, + SignatureScheme::RSA_PSS_SHA384, + SignatureScheme::RSA_PSS_SHA512, + SignatureScheme::ED25519, + ] + } +} + pub async fn connect( url: &str, server_cert: Option>, @@ -55,7 +109,30 @@ fn configure_client_with_cert( client_config_from_roots(root_store, policy) } +fn client_config_insecure(policy: &Policy) -> Result { + let mut tls_config = RustlsClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoopCertVerifier)) + .with_no_client_auth(); + + tls_config.alpn_protocols = vec![b"h3".to_vec()]; + + Ok(ClientConfig::builder() + .with_bind_default() + .with_custom_tls(tls_config) + .keep_alive_interval(policy.keep_alive_interval) + .max_idle_timeout(policy.max_idle_timeout) + .map_err(|e| CommunicationError::Other(e.to_string()))? + .build()) +} + fn configure_client_system_roots(policy: &Policy) -> Result { + // Check if insecure mode is enabled via env variable MTP_INSECURE + let insecure = std::env::var("MTP_INSECURE").is_ok(); + if insecure { + // Insecure mode: skip certificate verification entirely + return client_config_insecure(policy); + } let mut root_store = RootCertStore::empty(); // Load native certs diff --git a/transport/src/host.rs b/transport/src/host.rs index 0927aa0..2198770 100644 --- a/transport/src/host.rs +++ b/transport/src/host.rs @@ -5,6 +5,16 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use wtransport::{Connection as WTConnection, Endpoint, ServerConfig}; +fn generate_self_signed_cert() -> (Vec, Vec) { + let key_pair = rcgen::KeyPair::generate().unwrap(); + let params = + rcgen::CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]).unwrap(); + let cert = params.self_signed(&key_pair).unwrap(); + let cert_pem = cert.pem(); + let key_pem = key_pair.serialize_pem(); + (cert_pem.into_bytes(), key_pem.into_bytes()) +} + pub struct Host { incoming: tokio::sync::mpsc::Receiver<(Sender, Receiver)>, local_addr: std::net::SocketAddr, @@ -46,7 +56,16 @@ pub async fn host( ) -> Result { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let server_config = configure_server(ip, port, cert_pem, key_pem, &policy).await?; + // When MTP_INSECURE is set, generate a self-signed cert so the host can + // run without externally-provided TLS credentials. + let is_insecure = std::env::var("MTP_INSECURE").is_ok(); + let (use_cert_pem, use_key_pem) = if is_insecure { + generate_self_signed_cert() + } else { + (cert_pem, key_pem) + }; + + let server_config = configure_server(ip, port, use_cert_pem, use_key_pem, &policy).await?; let endpoint = Endpoint::server(server_config) .map_err(|e| CommunicationError::Other(format!("Endpoint creation failed: {}", e)))?; From c14831474260847c680b4fc5c1c5846aef111cd7 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:14:53 +0200 Subject: [PATCH 32/97] [WIP] Pings, Pongs & Streams --- Cargo.lock | 1 + Cargo.toml | 18 +++- client/Cargo.toml | 4 +- client/src/lib.rs | 194 ++++++++++++++++++++++++++++++++---- codec/src/registry.rs | 38 ++++++- docs/NATIVE-CLIENT.md | 49 ++++++++- docs/NATIVE-HOST.md | 30 ++++++ docs/WASM-CLIENT.md | 33 ++++++ host/Cargo.toml | 3 + host/src/lib.rs | 74 +++++++++----- transport/Cargo.toml | 6 ++ transport/src/client.rs | 3 +- transport/src/connection.rs | 126 +++++++++++++++++++++-- wasm/src/client.rs | 13 +-- wasm/src/config.rs | 7 ++ wasm/src/lib.rs | 3 + wasm/src/transport.rs | 9 +- 17 files changed, 541 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5eb8ab0..a507421 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -985,6 +985,7 @@ dependencies = [ "mtp-crypto", "mtp-files", "mtp-host", + "mtp-transport", "mtp-type-map", "rand 0.8.6", "rcgen", diff --git a/Cargo.toml b/Cargo.toml index daac4c7..8c0dbdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ edition = "2024" mtp-common = { version = "0.1.0", path = "common" } mtp-type-map = { version = "0.1.0", path = "type-map" } mtp-codec = { version = "0.1.0", path = "codec" } +mtp-transport = { version = "0.1.0", path = "transport", optional = true } # --- optional, behind features --- mtp-crypto = { version = "0.1.0", path = "crypto", optional = true, features = [ "serde", @@ -76,10 +77,23 @@ crypto = [ ] # MTP server host - version negotiation, Registry, incoming QUIC connections. -host = ["dep:mtp-host", "mtp-codec/registry"] +host = ["dep:mtp-host", "mtp-codec/registry", "transport"] # MTP client - outgoing QUIC connections to a host. -client = ["dep:mtp-client"] +client = ["dep:mtp-client", "transport"] + +# Opt into stream-specific host/client facade APIs. The transport itself is +# always framed over QUIC/WebTransport streams for compatibility. +streaming = [ + "transport", + "mtp-transport/streaming", + "mtp-host?/streaming", + "mtp-client?/streaming", +] + +# Direct access to the framed QUIC transport. Host/client features enable it +# automatically; this feature is useful for low-level integrations. +transport = ["dep:mtp-transport"] # On-disk storage for keyrings (`.mk`) and public key bundles (`.mpkb`). # Pulls in `crypto` so the `Keyring` / `PublicKeyBundle` types are in scope. diff --git a/client/Cargo.toml b/client/Cargo.toml index 94f8fa2..b913851 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -9,7 +9,9 @@ mtp-codec = { version = "0.1.0", path = "../codec" } mtp-transport = { version = "0.1.0", path = "../transport" } mtp-crypto = { version = "0.1.0", path = "../crypto", optional = true } rand = "0.8" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["rt", "sync", "time"] } [features] crypto = ["dep:mtp-crypto", "mtp-codec/crypto"] +# Enables stream-specific convenience exports and configuration. +streaming = ["mtp-transport/streaming"] diff --git a/client/src/lib.rs b/client/src/lib.rs index e764f55..9c8b49b 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,12 +1,15 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Version}; use mtp_common::CommunicationError; -#[cfg(feature = "crypto")] -use tokio::time::Duration; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{mpsc, Mutex}; +use tokio::time::{Duration, Instant}; pub use MTPClient as Client; pub use MTPConnection as Connection; pub use mtp_transport::Policy; pub use mtp_transport::Receiver; +#[cfg(feature = "streaming")] pub use mtp_transport::SendMode; pub use mtp_transport::Sender; @@ -30,6 +33,9 @@ pub struct ClientConfig { pub client_id: u64, pub description: Option, pub policy: Policy, + pub ping_interval: Duration, + pub max_missed_pings: usize, + pub ping_timestamp: bool, #[cfg(feature = "crypto")] pub auth_timeout: Duration, } @@ -48,6 +54,9 @@ impl ClientConfig { client_id: 0, description: None, policy: Policy::default(), + ping_interval: Duration::ZERO, + max_missed_pings: 3, + ping_timestamp: true, #[cfg(feature = "crypto")] auth_timeout: Duration::from_secs(30), } @@ -77,6 +86,21 @@ impl ClientConfig { self } + pub fn with_ping_interval(mut self, interval: Duration) -> Self { + self.ping_interval = interval; + self + } + + pub fn with_max_missed_pings(mut self, max_missed_pings: usize) -> Self { + self.max_missed_pings = max_missed_pings; + self + } + + pub fn with_ping_timestamp(mut self, ping_timestamp: bool) -> Self { + self.ping_timestamp = ping_timestamp; + self + } + #[cfg(feature = "crypto")] pub fn with_auth_timeout(mut self, timeout: Duration) -> Self { self.auth_timeout = timeout; @@ -97,13 +121,36 @@ pub struct MTPConnection { pub sender: Sender, pub receiver: Receiver, pub description: Option, + ping: Option, #[cfg(feature = "crypto")] pub auth_state: AuthState, #[cfg(feature = "crypto")] pub client_id: u64, } +struct PingSession { + last_ping: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl PingSession { + fn get_ping(&self) -> Option { + self.last_ping.try_lock().ok().and_then(|ping| *ping) + } +} + +impl Drop for PingSession { + fn drop(&mut self) { + self.task.abort(); + } +} + impl MTPConnection { + /* Returns the round-trip time for the latest Ping/Pong exchange. */ + pub fn get_ping(&self) -> Option { + self.ping.as_ref().and_then(PingSession::get_ping) + } + /* * Send a request frame and wait for the response with the same frame id. * Any expected response type is validated after the id match. Frames with @@ -148,6 +195,97 @@ impl MTPConnection { } } +fn start_ping_session( + config: &ClientConfig, + sender: Sender, + receiver: &Receiver, +) -> Option { + if config.ping_interval.is_zero() { + return None; + } + + let (pong_tx, mut pong_rx) = mpsc::unbounded_channel(); + receiver.observe_pongs(pong_tx); + let last_ping = Arc::new(Mutex::new(None)); + let ping_state = last_ping.clone(); + let interval = config.ping_interval; + let max_missed_pings = config.max_missed_pings; + let ping_timestamp = config.ping_timestamp; + let mut close_rx = receiver.handle().subscribe_close(); + + let task = tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.tick().await; + let mut pending = HashMap::new(); + + loop { + tokio::select! { + _ = close_rx.changed() => { + if close_rx.borrow().is_some() { + break; + } + } + _ = ticker.tick() => { + if max_missed_pings > 0 && !pending.is_empty() && pending.len() >= max_missed_pings { + sender.close(); + break; + } + + let mut ping = CommunicationValue::new(mtp_codec::CommunicationType::Ping); + if ping_timestamp { + let sent_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + ping = ping.add_typed_default( + DataType::Timestamp, + DataValue::UnsignedNumber(sent_at), + ); + } + let id = ping.get_id(); + if sender.send(&ping).await.is_err() { + sender.close(); + break; + } + pending.insert(id, Instant::now()); + } + pong = pong_rx.recv() => match pong { + Some(pong) => { + if let Some(sent_at) = pending.remove(&pong.get_id()) { + let mut last_ping = ping_state.lock().await; + *last_ping = Some(sent_at.elapsed()); + } + } + None => break, + }, + } + } + }); + + Some(PingSession { last_ping, task }) +} + +fn connection_from_parts( + config: ClientConfig, + sender: Sender, + receiver: Receiver, + #[cfg(feature = "crypto")] auth_state: AuthState, + #[cfg(feature = "crypto")] client_id: u64, +) -> MTPConnection { + let ping = start_ping_session(&config, sender.clone(), &receiver); + MTPConnection { + version: PROTOCOL_VERSION, + sender, + receiver, + description: config.description, + ping, + #[cfg(feature = "crypto")] + auth_state, + #[cfg(feature = "crypto")] + client_id, + } +} + #[cfg(feature = "crypto")] #[derive(Debug, Clone, PartialEq, Eq)] pub enum AuthState { @@ -183,16 +321,18 @@ impl MTPClient { sender.send(&ident).await?; - Ok(MTPConnection { - version: PROTOCOL_VERSION, + #[cfg(feature = "crypto")] + let client_id = config.client_id; + #[cfg(feature = "crypto")] + return Ok(connection_from_parts( + config, sender, receiver, - description: config.description, - #[cfg(feature = "crypto")] - auth_state: AuthState::Unauthenticated, - #[cfg(feature = "crypto")] - client_id: config.client_id, - }) + AuthState::Unauthenticated, + client_id, + )); + #[cfg(not(feature = "crypto"))] + Ok(connection_from_parts(config, sender, receiver)) } } @@ -488,14 +628,14 @@ impl MTPClient { return Err(e); } - Ok(MTPConnection { - version: PROTOCOL_VERSION, + let client_id = config.client_id; + Ok(connection_from_parts( + config, sender, receiver, - description: config.description, - auth_state: AuthState::Authenticated, - client_id: config.client_id, - }) + AuthState::Authenticated, + client_id, + )) } pub async fn auth_register( @@ -636,14 +776,13 @@ impl MTPClient { return Err(e); } - Ok(MTPConnection { - version: PROTOCOL_VERSION, + Ok(connection_from_parts( + config, sender, receiver, - description: config.description, - auth_state: AuthState::Authenticated, - client_id: assigned_id, - }) + AuthState::Authenticated, + assigned_id, + )) } } @@ -671,6 +810,17 @@ mod tests { assert_eq!(config.client_id, 42); } + #[test] + fn test_ping_config() { + let config = ClientConfig::new("https://localhost:4433") + .with_ping_interval(Duration::from_secs(5)) + .with_max_missed_pings(2) + .with_ping_timestamp(false); + assert_eq!(config.ping_interval, Duration::from_secs(5)); + assert_eq!(config.max_missed_pings, 2); + assert!(!config.ping_timestamp); + } + #[cfg(feature = "crypto")] #[test] fn test_auth_state_unauthenticated_is_not_authenticated() { diff --git a/codec/src/registry.rs b/codec/src/registry.rs index 78757bf..1d06860 100644 --- a/codec/src/registry.rs +++ b/codec/src/registry.rs @@ -1,4 +1,7 @@ -use mtp_type_map::Version; +use mtp_common::CodecError; +use mtp_type_map::{PROTOCOL_VERSION, TypeMap, Version}; + +use crate::CommunicationValue; pub use mtp_type_map::Registry; @@ -9,11 +12,42 @@ pub use mtp_type_map::Registry; #[derive(Clone, Debug)] pub struct VersionedCodec { registry: Registry, + type_map: TypeMap, } impl VersionedCodec { pub fn new(registry: Registry) -> Self { - Self { registry } + let type_map = registry + .latest() + .cloned() + .unwrap_or_else(|| TypeMap::new(PROTOCOL_VERSION)); + Self { registry, type_map } + } + + /// Create a codec bound to a negotiated protocol version. + pub fn for_version(registry: Registry, version: Version) -> Option { + let type_map = registry.get(&version)?.clone(); + Some(Self { registry, type_map }) + } + + /// Return the type map used by this codec. + pub fn type_map(&self) -> &TypeMap { + &self.type_map + } + + /// Return the protocol version used by this codec. + pub fn version(&self) -> &Version { + &self.type_map.version + } + + /// Encode a value using the codec's negotiated framing rules. + pub fn encode(&self, value: &CommunicationValue) -> Result, CodecError> { + value.to_bytes() + } + + /// Decode a frame and retain the negotiated type map for typed access. + pub fn decode(&self, bytes: &[u8]) -> Result { + CommunicationValue::from_bytes_with(bytes, &self.type_map) } pub fn negotiate(&self, client_versions: &[Version]) -> Option { diff --git a/docs/NATIVE-CLIENT.md b/docs/NATIVE-CLIENT.md index 027aa29..421828e 100644 --- a/docs/NATIVE-CLIENT.md +++ b/docs/NATIVE-CLIENT.md @@ -18,10 +18,14 @@ mtp = { path = "/path/to/mtp", features = ["client", "crypto"] } ```rust use mtp::client::{ClientConfig, ClientTlsConfig}; +use std::time::Duration; let config = ClientConfig::new("https://host.example.com:4433") .with_tls(ClientTlsConfig::SystemRoots) - .with_client_id(0); + .with_client_id(0) + .with_ping_interval(Duration::from_secs(5)) + .with_max_missed_pings(3) + .with_ping_timestamp(true); ``` | Field | Type | Description | @@ -30,6 +34,9 @@ let config = ClientConfig::new("https://host.example.com:4433") | `tls` | `ClientTlsConfig` | `SystemRoots` or `PinnedPem(pem_bytes)` | | `client_id` | `u64` | Client identifier (ignored during `auth_register`) | | `description` | `Option` | Optional label sent during handshake (e.g. `"phone"`) | +| `ping_interval` | `Duration` | Interval between Ping frames; zero disables pings | +| `max_missed_pings` | `usize` | Unanswered Ping frames allowed before the connection closes | +| `ping_timestamp` | `bool` | Adds a `Timestamp` entry to each Ping frame | | `auth_timeout` | `Duration` (crypto) | Authentication handshake timeout (default 30s) | ### TLS Certificate Handling @@ -74,6 +81,46 @@ pub struct MTPConnection { - `description` -- the label sent during handshake (set via `ClientConfig::with_description`) - `client_id` -- the confirmed/assigned client identifier (crypto only) +When `ping_interval` is non-zero, MTP sends Ping frames in the background and +consumes their Pong responses before application message handling. `get_ping()` +returns the round-trip duration of the latest matched Pong, or `None` until a +Pong arrives. A connection closes when the configured unanswered Ping limit is +reached. + +### Ping-Pong + +Ping/Pong is part of the protocol, not just a transport keepalive. Each Ping +frame is matched against a Pong with the same frame id, and the client uses the +response to update `get_ping()`. If the host does not answer within the +configured limit, the connection closes. + +Enable it in `ClientConfig`, then inspect the latest round-trip time on the +connection. Pings start after the connection has been established; `None` is +normal until the first matching Pong arrives. + +```rust +use mtp::client::{ClientConfig, MTPClient}; +use std::time::Duration; + +let config = ClientConfig::new("https://host.example.com:4433") + .with_client_id(42) + .with_ping_interval(Duration::from_secs(5)) + .with_max_missed_pings(3) + .with_ping_timestamp(true); + +let conn = MTPClient::connect(config).await?; + +if let Some(round_trip) = conn.get_ping() { + println!("latest MTP round trip: {round_trip:?}"); +} +``` + +The client consumes the Pong frames used by this loop, so they are not returned +by `conn.receiver.receive()`. Set `ping_interval` to `Duration::ZERO` (the +default) to disable protocol pings. `max_missed_pings` is the number of +outstanding Ping frames allowed before the client closes the connection; use a +host with automatic Pong responses, or provide an equivalent responder. + ### Unauthenticated Connect ```rust diff --git a/docs/NATIVE-HOST.md b/docs/NATIVE-HOST.md index 631c1ea..324cd66 100644 --- a/docs/NATIVE-HOST.md +++ b/docs/NATIVE-HOST.md @@ -48,6 +48,7 @@ let config = HostConfig::new( | `port` | `u16` | Listen port | | `tls_fullchain` | `Vec` | PEM-encoded TLS certificate chain | | `tls_key` | `Vec` | PEM-encoded TLS private key | +| `send_pongs` | `bool` | Sends a Pong for each received Ping (default `true`) | | `authentication_policy` | `AuthenticationPolicy` (crypto) | `ForceAuthentication`, `AllowAuthentication`, or `Unauthenticated` | | `host_keyring` | `Keyring` (crypto) | Host's signing and KEM keys | | `get_existing_user` | `Fn(u64) -> Pin> + Send>> + Send + Sync` (crypto) | Async lookup callback for login | @@ -77,6 +78,35 @@ let config = HostConfig::new(ip, port, cert, key); The host requires a TLS certificate. For development, generate a self-signed certificate using `rcgen`. For production, use a CA-signed certificate. +### Ping-Pong + +The host handles protocol Ping/Pong automatically unless you disable it with +`with_pongs(false)`. Enable the default responder explicitly when constructing +the host if you want to make the choice visible in application configuration: + +```rust +let config = HostConfig::new(ip, port, cert, key) + .with_pongs(true); +``` + +For every received Ping, the responder sends a Pong with the same frame id and +copies the optional `Timestamp` data entry. Ping and Pong frames handled this +way are not delivered by `conn.receiver.receive()`. This lets native clients +use `ClientConfig::with_ping_interval` and `MTPConnection::get_ping()` without +adding application-level handlers. + +Disable it only when the application needs to handle Ping frames itself: + +```rust +let config = HostConfig::new(ip, port, cert, key) + .with_pongs(false); +``` + +With automatic responses disabled, Ping frames are delivered through the normal +receiver and the application is responsible for sending a compatible Pong (the +same frame id, and normally the Ping's `Timestamp`) if it wants clients to +continue their protocol ping loop. + ## Accepting Connections ```rust diff --git a/docs/WASM-CLIENT.md b/docs/WASM-CLIENT.md index 3cd4699..2802036 100644 --- a/docs/WASM-CLIENT.md +++ b/docs/WASM-CLIENT.md @@ -149,6 +149,39 @@ If hashes are omitted, the browser uses its normal TLS root store. `maxMessageSize` caps inbound and outbound MTP frames before buffering/sending. `authTimeoutMs` bounds connect/login/register promises at the SDK layer. +## Streams + +The browser client uses one WebTransport session per `MTPClient` instance. +`send()`, `request()`, and `subscribe()` all operate over that session; the SDK +does not expose browser stream objects directly. + +Use the normal message APIs to send and receive over that session: + +```typescript +const client = await MTPClient.create({ url, hostPublicKey }); +await client.connect(); + +const unsubscribe = client.subscribe("SomeType", (message) => { + console.log(message.data); +}); + +await client.send("SomeType", { value: "hello" }); +unsubscribe(); +``` + +Internally, each outbound MTP frame is written to a new WebTransport +unidirectional stream as a four-byte big-endian length followed by the frame, +then that stream is closed. Incoming frames are read from the session's +incoming unidirectional streams. The reader accepts both one-frame streams and +native peers that place several frames on a persistent stream, so browser and +native clients interoperate without stream configuration. + +The SDK deliberately owns stream lifetime and framing. Do not create browser +streams for MTP frames yourself through the SDK. For direct generated bindings, +use `client.raw.client` or import `WasmClient` from `mtp/raw`; a `WasmClient` +still owns one active WebTransport session, so create another instance for an +independent connection. + ## Sending, Requests, Subscriptions, And Pings `send` accepts either a typed message or a prebuilt raw frame: diff --git a/host/Cargo.toml b/host/Cargo.toml index ea4e661..71cb38c 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -13,3 +13,6 @@ tokio = { version = "1", features = ["time"] } [features] crypto = ["dep:mtp-crypto", "mtp-codec/crypto"] +# Stream mode is optional at the facade boundary. The underlying transport +# remains framed/stream based so the default API stays backwards compatible. +streaming = ["mtp-transport/streaming"] diff --git a/host/src/lib.rs b/host/src/lib.rs index 3650385..44d73a9 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -14,6 +14,7 @@ pub use MTPConnection as Connection; pub use MTPHost as Host; pub use mtp_transport::Policy; pub use mtp_transport::Receiver; +#[cfg(feature = "streaming")] pub use mtp_transport::SendMode; pub use mtp_transport::Sender; @@ -55,6 +56,7 @@ pub struct HostConfig { pub tls_key: Vec, pub policy: Policy, + pub send_pongs: bool, #[cfg(feature = "crypto")] pub authentication_policy: AuthenticationPolicy, @@ -76,6 +78,7 @@ impl HostConfig { tls_fullchain, tls_key, policy: Policy::default(), + send_pongs: true, #[cfg(feature = "crypto")] authentication_policy: AuthenticationPolicy::Unauthenticated, #[cfg(feature = "crypto")] @@ -101,6 +104,11 @@ impl HostConfig { self } + pub fn with_pongs(mut self, send_pongs: bool) -> Self { + self.send_pongs = send_pongs; + self + } + #[cfg(feature = "crypto")] pub fn with_authentication( mut self, @@ -186,7 +194,6 @@ pub struct MTPConnection { pub struct MTPHost { transport: mtp_transport::Host, registry: Registry, - #[cfg(feature = "crypto")] config: HostConfig, } @@ -206,7 +213,6 @@ impl MTPHost { Ok(Self { transport, registry, - #[cfg(feature = "crypto")] config, }) } @@ -228,7 +234,7 @@ impl MTPHost { match self.config.authentication_policy { AuthenticationPolicy::ForceAuthentication => { let timeout = self.config.auth_timeout; - return match tokio::time::timeout( + let connection = match tokio::time::timeout( timeout, self.accept_authenticated(sender, receiver), ) @@ -237,9 +243,11 @@ impl MTPHost { Ok(result) => result, Err(_) => Err(AcceptError::AuthenticationTimedOut), }; + return Ok(self.configure_pongs(connection?)); } AuthenticationPolicy::AllowAuthentication => { - return self.accept_allow_auth(sender, receiver).await; + let connection = self.accept_allow_auth(sender, receiver).await?; + return Ok(self.configure_pongs(connection)); } AuthenticationPolicy::Unauthenticated => { let first_msg = match receiver.receive().await { @@ -265,12 +273,13 @@ impl MTPHost { Some(v) => v, None => return Err(AcceptError::UnsupportedVersion(client_version)), }; - let codec = VersionedCodec::new(self.registry.clone()); + let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) + .expect("negotiated version must be registered"); let description = match first_msg.get_data(DataType::Description) { DataValue::Str(s) => Some(s.clone()), _ => None, }; - Ok(Some(MTPConnection { + Ok(self.configure_pongs(Some(MTPConnection { version: negotiated, codec, sender, @@ -282,7 +291,7 @@ impl MTPHost { client_id: rand::random(), #[cfg(feature = "crypto")] client_public_key: None, - })) + }))) } } @@ -304,18 +313,19 @@ impl MTPHost { Some(v) => v, None => return Err(AcceptError::UnsupportedVersion(client_version)), }; - let codec = VersionedCodec::new(self.registry.clone()); + let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) + .expect("negotiated version must be registered"); let description = match first_msg.get_data(DataType::Description) { DataValue::Str(s) => Some(s.clone()), _ => None, }; - return Ok(Some(MTPConnection { + return Ok(self.configure_pongs(Some(MTPConnection { version: negotiated, codec, sender, receiver, description, - })); + }))); } } @@ -326,6 +336,19 @@ impl MTPHost { pub fn registry(&self) -> &Registry { &self.registry } + + fn configure_pongs(&self, connection: Option) -> Option { + if let Some(connection) = connection { + if self.config.send_pongs { + connection + .receiver + .respond_to_pings(connection.sender.clone()); + } + Some(connection) + } else { + None + } + } } #[cfg(feature = "crypto")] @@ -480,6 +503,13 @@ impl MTPHost { }; let tm = mtp_codec::TypeMap::latest(); + let negotiated = self + .registry + .negotiate(std::slice::from_ref(&client_version)) + .ok_or_else(|| { + sender.close(); + AcceptError::UnsupportedVersion(client_version.clone()) + })?; let pq_enabled = !self .config .host_keyring @@ -636,18 +666,8 @@ impl MTPHost { return Err(AcceptError::Send(e)); } - // ===== Version negotiation ===== - let negotiated = match self - .registry - .negotiate(std::slice::from_ref(&client_version)) - { - Some(v) => v, - None => { - sender.close(); - return Err(AcceptError::UnsupportedVersion(client_version)); - } - }; - let codec = VersionedCodec::new(self.registry.clone()); + let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) + .expect("negotiated version must be registered"); Ok(Some(MTPConnection { version: negotiated, @@ -761,7 +781,8 @@ impl MTPHost { Some(v) => v, None => return Err(AcceptError::UnsupportedVersion(client_version)), }; - let codec = VersionedCodec::new(self.registry.clone()); + let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) + .expect("negotiated version must be registered"); return Ok(Some(MTPConnection { version: negotiated, codec, @@ -842,4 +863,11 @@ mod tests { assert_ne!(AuthState::Unauthenticated, AuthState::Authenticated); assert_ne!(AuthState::Pending, AuthState::Authenticated); } + + #[test] + fn host_config_pongs_default_to_enabled() { + let config = HostConfig::new("127.0.0.1".parse().unwrap(), 4433, Vec::new(), Vec::new()); + assert!(config.send_pongs); + assert!(!config.with_pongs(false).send_pongs); + } } diff --git a/transport/Cargo.toml b/transport/Cargo.toml index 5ff48ac..69c85b4 100644 --- a/transport/Cargo.toml +++ b/transport/Cargo.toml @@ -27,3 +27,9 @@ required-features = ["host"] default = [] # Enables hosting a MTP server host = [] +# Documents and exposes the framed uni-directional stream transport used by +# the host and client facades. The transport is stream based by design, so +# keeping this as a marker feature lets facade crates opt into their +# stream-specific convenience exports without making the core transport +# unusable for existing consumers. +streaming = [] diff --git a/transport/src/client.rs b/transport/src/client.rs index f356ea5..e0a7b68 100644 --- a/transport/src/client.rs +++ b/transport/src/client.rs @@ -2,10 +2,9 @@ use std::sync::Arc; use mtp_common::CommunicationError; use rustls::{ - ClientConfig as RustlsClientConfig, RootCertStore, + ClientConfig as RustlsClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme, client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, pki_types::{ServerName, UnixTime, pem::PemObject}, - DigitallySignedStruct, SignatureScheme, }; use wtransport::{ClientConfig, Endpoint}; diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 8a88fc3..b4d6e8a 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -2,7 +2,7 @@ use crate::ConnectionHandle; use mtp_codec::CommunicationValue; use mtp_common::CommunicationError; use std::sync::Arc; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::{mpsc, Mutex, RwLock, Semaphore}; use tokio::time::{Duration, sleep, timeout}; use wtransport::Connection; @@ -29,7 +29,10 @@ pub struct Policy { pub force_close_delay: Duration, pub max_transient_recv_errors: usize, pub transient_recv_backoff: Duration, + pub persistent_stream_max_retries: usize, + pub persistent_stream_retry_backoff: Duration, pub receiver_queue_capacity: usize, + pub max_concurrent_stream_tasks: usize, } impl Default for Policy { @@ -48,7 +51,10 @@ impl Default for Policy { force_close_delay: Duration::from_millis(300), max_transient_recv_errors: 20, transient_recv_backoff: Duration::from_millis(100), + persistent_stream_max_retries: 4, + persistent_stream_retry_backoff: Duration::from_millis(20), receiver_queue_capacity: 1000, + max_concurrent_stream_tasks: 128, } } } @@ -90,6 +96,24 @@ impl Policy { self.receiver_queue_capacity = receiver_queue_capacity; self } + + pub fn with_persistent_stream_retries( + mut self, + persistent_stream_max_retries: usize, + persistent_stream_retry_backoff: Duration, + ) -> Self { + self.persistent_stream_max_retries = persistent_stream_max_retries; + self.persistent_stream_retry_backoff = persistent_stream_retry_backoff; + self + } + + pub fn with_max_concurrent_stream_tasks( + mut self, + max_concurrent_stream_tasks: usize, + ) -> Self { + self.max_concurrent_stream_tasks = max_concurrent_stream_tasks; + self + } } enum ReceivedFrame { @@ -98,8 +122,9 @@ enum ReceivedFrame { Idle, } +#[derive(Clone)] pub struct Sender { - send_guard: Mutex<()>, + send_guard: Arc>, stream_guard: Arc>>, handle: Arc, connection: Connection, @@ -109,7 +134,7 @@ pub struct Sender { impl Sender { pub fn new(connection: Connection, handle: Arc, policy: Arc) -> Self { Self { - send_guard: Mutex::new(()), + send_guard: Arc::new(Mutex::new(())), stream_guard: Arc::new(Mutex::new(None)), handle, connection, @@ -208,23 +233,24 @@ impl Sender { return Err(CommunicationError::StreamClosed); } - let res = { - let stream = Self::ensure_stream(conn, stream_opt, policy).await?; - Self::write_frame(stream, data, policy).await + let res = match Self::ensure_stream(conn, stream_opt, policy).await { + Ok(stream) => Self::write_frame(stream, data, policy).await, + Err(e) => Err(e), }; if res.is_ok() { return Ok(()); } + let err = res.err().unwrap_or(CommunicationError::StreamError); *stream_opt = None; tries += 1; - if tries >= 4 { - let stream = Self::ensure_stream(conn, stream_opt, policy).await?; - return Self::write_frame(stream, data, policy).await; + if tries > policy.persistent_stream_max_retries { + return Err(err); } - tokio::time::sleep(std::time::Duration::from_millis(20 * tries as u64)).await; + let backoff = policy.persistent_stream_retry_backoff * tries as u32; + tokio::time::sleep(backoff).await; } } @@ -433,6 +459,13 @@ pub struct Receiver { rx: Mutex>>, _accept_task: tokio::task::JoinHandle<()>, handle: Arc, + ping_control: Arc>, +} + +#[derive(Clone, Default)] +struct PingControl { + pong_sender: Option, + pong_observer: Option>, } impl Drop for Receiver { @@ -456,6 +489,10 @@ impl Receiver { let conn_handle = handle.clone(); let accept_connection = connection.clone(); let accept_policy = policy.clone(); + let ping_control = Arc::new(RwLock::new(PingControl::default())); + let accept_ping_control = ping_control.clone(); + let stream_limit = Arc::new(Semaphore::new(policy.max_concurrent_stream_tasks.max(1))); + let accept_stream_limit = stream_limit.clone(); let accept_task = tokio::spawn(async move { let mut close_rx = conn_handle.subscribe_close(); @@ -474,15 +511,62 @@ impl Receiver { ) => { match accepted { Ok(Ok(stream)) => { + let permit = match accept_stream_limit.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => break, + }; let tx_stream = tx.clone(); let stream_handle = conn_handle.clone(); let stream_policy = accept_policy.clone(); + let stream_ping_control = accept_ping_control.clone(); tokio::spawn(async move { + let _permit = permit; let mut s = stream; loop { match Self::read_one_frame(&mut s, &stream_policy).await { Ok(ReceivedFrame::Message(msg)) => { + let ping_type = mtp_codec::CommunicationType::Ping + .to_id(&mtp_codec::TypeMap::latest()); + let pong_type = mtp_codec::CommunicationType::Pong + .to_id(&mtp_codec::TypeMap::latest()); + let control = { + let control = stream_ping_control.read().await; + if msg.get_type() == ping_type { + control + .pong_sender + .clone() + .map(|sender| (Some(sender), None)) + } else if msg.get_type() == pong_type { + control + .pong_observer + .clone() + .map(|observer| (None, Some(observer))) + } else { + None + } + }; + + if let Some((Some(sender), _)) = control { + let mut pong = CommunicationValue::new(mtp_codec::CommunicationType::Pong) + .with_id(msg.get_id()); + if let Some(timestamp) = msg.get_data_opt(mtp_codec::DataType::Timestamp) { + pong = pong.add_typed_default( + mtp_codec::DataType::Timestamp, + timestamp.clone(), + ); + } + if let Err(e) = sender.send(&pong).await { + log::warn!("[Receiver] failed to send Pong: {e}"); + } + continue; + } + + if let Some((_, Some(observer))) = control { + let _ = observer.send(msg); + continue; + } + if tx_stream.send(Ok(msg)).await.is_err() { break; } @@ -545,6 +629,25 @@ impl Receiver { rx: Mutex::new(rx), _accept_task: accept_task, handle, + ping_control, + } + } + + /* Respond to reserved Ping frames without exposing them to application I/O. */ + pub fn respond_to_pings(&self, sender: Sender) { + if let Ok(mut control) = self.ping_control.try_write() { + control.pong_sender = Some(sender); + } else { + log::warn!("[Receiver] could not register Ping responder: control lock busy"); + } + } + + /* Route reserved Pong frames to a connection-level observer. */ + pub fn observe_pongs(&self, observer: mpsc::UnboundedSender) { + if let Ok(mut control) = self.ping_control.try_write() { + control.pong_observer = Some(observer); + } else { + log::warn!("[Receiver] could not register Pong observer: control lock busy"); } } @@ -689,7 +792,10 @@ mod tests { assert_eq!(p.force_close_delay, Duration::from_millis(300)); assert_eq!(p.max_transient_recv_errors, 20); assert_eq!(p.transient_recv_backoff, Duration::from_millis(100)); + assert_eq!(p.persistent_stream_max_retries, 4); + assert_eq!(p.persistent_stream_retry_backoff, Duration::from_millis(20)); assert_eq!(p.receiver_queue_capacity, 1000); + assert_eq!(p.max_concurrent_stream_tasks, 128); } #[test] diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 280bbf1..503d1e9 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -267,16 +267,17 @@ pub struct WasmClient { impl WasmClient { #[wasm_bindgen(constructor)] pub fn new( - on_state_change: &js_sys::Function, - on_message: &js_sys::Function, - on_error: &js_sys::Function, + on_state_change: Option, + on_message: Option, + on_error: Option, ) -> Self { + let noop = || js_sys::Function::new_no_args(""); Self { transport: Rc::new(RefCell::new(None)), state: Rc::new(Cell::new(ConnectionState::Disconnected)), - on_state_change: on_state_change.clone(), - on_message: on_message.clone(), - on_error: on_error.clone(), + on_state_change: on_state_change.unwrap_or_else(noop), + on_message: on_message.unwrap_or_else(noop), + on_error: on_error.unwrap_or_else(noop), subscriptions: Rc::new(RefCell::new(HashMap::new())), next_subscription_id: Rc::new(Cell::new(1)), pending_requests: Rc::new(RefCell::new(HashMap::new())), diff --git a/wasm/src/config.rs b/wasm/src/config.rs index 8ac13b5..46cdf5e 100644 --- a/wasm/src/config.rs +++ b/wasm/src/config.rs @@ -9,6 +9,13 @@ pub struct ConnectionConfig { pub(crate) description: Option, } +/// Newer API name for the browser connection configuration. +/// +/// `ConnectionConfig` remains the concrete wasm-bindgen class for backwards +/// compatibility with the existing raw JavaScript bindings. The alias keeps +/// Rust consumers aligned with the native/WASM naming used by the public API. +pub type WasmClientConfig = ConnectionConfig; + #[wasm_bindgen] impl ConnectionConfig { #[wasm_bindgen(constructor)] diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 5b49dc6..4c9f783 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -7,6 +7,9 @@ pub mod logging; pub mod subscription; pub mod transport; +pub use client::WasmClient; +pub use config::{ConnectionConfig, WasmClientConfig}; + #[cfg(not(test))] use wasm_bindgen::prelude::*; diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index 6860552..4ccb14c 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -402,7 +402,14 @@ impl WasmTransport { if let Some(reader) = self.stream_reader.borrow_mut().take() { release_reader_lock(&reader); } - self.buffer.borrow_mut().clear(); + // A frame is never allowed to span stream boundaries. The + // native persistent-stream sender packs frames on one + // stream, while the WASM sender uses one stream per frame; + // either mode must reject a truncated frame instead of + // silently dropping its prefix. + if !self.buffer.borrow().is_empty() { + return Err(js_error("stream ended in the middle of a frame")); + } } } } From a6dd73a41f4d7d2a23d90ff2039216782c2f1612 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:55:31 +0200 Subject: [PATCH 33/97] [add] ping jitter, receiver backpressure, stream frame limits --- Cargo.lock | 270 ++++++++++----------------------- client/Cargo.toml | 6 + client/src/lib.rs | 19 ++- client/tests/ping.rs | 80 ++++++++++ transport/Cargo.toml | 1 + transport/src/connection.rs | 118 +++++++++++++- transport/tests/integration.rs | 146 ++++++++++++++++++ 7 files changed, 450 insertions(+), 190 deletions(-) create mode 100644 client/tests/ping.rs diff --git a/Cargo.lock b/Cargo.lock index a507421..d84909c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,9 +176,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cast" @@ -188,9 +188,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -221,6 +221,17 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chacha20poly1305" version = "0.10.1" @@ -228,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -402,9 +413,9 @@ dependencies = [ [[package]] name = "der" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "const-oid 0.10.2", "zeroize", @@ -592,20 +603,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -615,7 +612,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi 6.0.0", + "r-efi", "rand_core 0.10.1", "wasm-bindgen", ] @@ -806,11 +803,11 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -891,9 +888,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minicov" @@ -913,9 +910,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -987,7 +984,7 @@ dependencies = [ "mtp-host", "mtp-transport", "mtp-type-map", - "rand 0.8.6", + "rand 0.8.7", "rcgen", "tokio", ] @@ -999,9 +996,12 @@ dependencies = [ "mtp-codec", "mtp-common", "mtp-crypto", + "mtp-host", "mtp-transport", - "rand 0.8.6", + "rand 0.8.7", + "rcgen", "tokio", + "tracing", ] [[package]] @@ -1013,7 +1013,7 @@ dependencies = [ "mtp-common", "mtp-crypto", "mtp-type-map", - "rand 0.8.6", + "rand 0.8.7", ] [[package]] @@ -1061,7 +1061,7 @@ dependencies = [ "mtp-common", "mtp-crypto", "mtp-transport", - "rand 0.8.6", + "rand 0.8.7", "tokio", ] @@ -1076,6 +1076,7 @@ dependencies = [ "rustls", "rustls-native-certs", "tokio", + "tracing", "wtransport", ] @@ -1269,7 +1270,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.0", + "der 0.8.1", "spki 0.8.0", ] @@ -1357,15 +1358,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -1379,16 +1381,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1400,12 +1402,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -1414,23 +1410,24 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", - "rand_chacha 0.3.1", + "rand_chacha", "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.9.4" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1443,16 +1440,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -1462,21 +1449,21 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - [[package]] name = "rand_core" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rcgen" version = "0.14.8" @@ -1541,9 +1528,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -1591,9 +1578,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -1802,9 +1789,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -1827,7 +1814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.0", + "der 0.8.1", ] [[package]] @@ -1952,9 +1939,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -2104,15 +2091,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -2238,16 +2216,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -2265,31 +2234,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -2298,102 +2250,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - [[package]] name = "writeable" version = "0.6.3" @@ -2502,18 +2400,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -2596,6 +2494,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/client/Cargo.toml b/client/Cargo.toml index b913851..693378e 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -10,6 +10,12 @@ mtp-transport = { version = "0.1.0", path = "../transport" } mtp-crypto = { version = "0.1.0", path = "../crypto", optional = true } rand = "0.8" tokio = { version = "1", features = ["rt", "sync", "time"] } +tracing = "0.1" + +[dev-dependencies] +mtp-host = { version = "0.1.0", path = "../host" } +mtp-transport = { version = "0.1.0", path = "../transport", features = ["host"] } +rcgen = "0.14" [features] crypto = ["dep:mtp-crypto", "mtp-codec/crypto"] diff --git a/client/src/lib.rs b/client/src/lib.rs index 9c8b49b..860ae85 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -2,14 +2,14 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Versi use mtp_common::CommunicationError; use std::collections::HashMap; use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; +use rand::Rng; +use tokio::sync::{Mutex, mpsc}; use tokio::time::{Duration, Instant}; pub use MTPClient as Client; pub use MTPConnection as Connection; pub use mtp_transport::Policy; pub use mtp_transport::Receiver; -#[cfg(feature = "streaming")] pub use mtp_transport::SendMode; pub use mtp_transport::Sender; @@ -34,6 +34,7 @@ pub struct ClientConfig { pub description: Option, pub policy: Policy, pub ping_interval: Duration, + pub ping_jitter: Option, pub max_missed_pings: usize, pub ping_timestamp: bool, #[cfg(feature = "crypto")] @@ -55,6 +56,7 @@ impl ClientConfig { description: None, policy: Policy::default(), ping_interval: Duration::ZERO, + ping_jitter: None, max_missed_pings: 3, ping_timestamp: true, #[cfg(feature = "crypto")] @@ -91,6 +93,11 @@ impl ClientConfig { self } + pub fn with_ping_jitter(mut self, jitter: Option) -> Self { + self.ping_jitter = jitter; + self + } + pub fn with_max_missed_pings(mut self, max_missed_pings: usize) -> Self { self.max_missed_pings = max_missed_pings; self @@ -209,6 +216,7 @@ fn start_ping_session( let last_ping = Arc::new(Mutex::new(None)); let ping_state = last_ping.clone(); let interval = config.ping_interval; + let ping_jitter = config.ping_jitter; let max_missed_pings = config.max_missed_pings; let ping_timestamp = config.ping_timestamp; let mut close_rx = receiver.handle().subscribe_close(); @@ -231,6 +239,12 @@ fn start_ping_session( break; } + if let Some(jitter) = ping_jitter && !jitter.is_zero() { + let max_ms = jitter.as_millis() as u64; + let extra = rand::thread_rng().gen_range(0..=max_ms); + tokio::time::sleep(Duration::from_millis(extra)).await; + } + let mut ping = CommunicationValue::new(mtp_codec::CommunicationType::Ping); if ping_timestamp { let sent_at = std::time::SystemTime::now() @@ -817,6 +831,7 @@ mod tests { .with_max_missed_pings(2) .with_ping_timestamp(false); assert_eq!(config.ping_interval, Duration::from_secs(5)); + assert_eq!(config.ping_jitter, None); assert_eq!(config.max_missed_pings, 2); assert!(!config.ping_timestamp); } diff --git a/client/tests/ping.rs b/client/tests/ping.rs new file mode 100644 index 0000000..b071864 --- /dev/null +++ b/client/tests/ping.rs @@ -0,0 +1,80 @@ +use std::net::{IpAddr, Ipv4Addr}; + +use mtp_client::{ClientConfig, MTPClient}; +use mtp_host::{HostConfig, MTPHost}; + +fn generate_self_signed_cert() -> (Vec, Vec) { + let key_pair = rcgen::KeyPair::generate().unwrap(); + let params = + rcgen::CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]).unwrap(); + let cert = params.self_signed(&key_pair).unwrap(); + let cert_pem = cert.pem(); + let key_pem = key_pair.serialize_pem(); + (cert_pem.into_bytes(), key_pem.into_bytes()) +} + +async fn start_host(send_pongs: bool) -> (MTPHost, Vec) { + let (cert_pem, key_pem) = generate_self_signed_cert(); + let host = MTPHost::new( + HostConfig::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0, cert_pem.clone(), key_pem) + .with_pongs(send_pongs), + ) + .await + .unwrap(); + (host, cert_pem) +} + +#[tokio::test] +async fn test_ping_rtt_and_missed_ping_teardown() { + let (mut host, cert_pem) = start_host(true).await; + let url = format!("https://127.0.0.1:{}", host.local_addr().port()); + + let client = MTPClient::connect( + ClientConfig::new(url) + .with_pinned_pem(cert_pem) + .with_ping_interval(std::time::Duration::from_millis(25)) + .with_max_missed_pings(3), + ) + .await + .unwrap(); + + let _accepted = host.accept().await.unwrap().unwrap(); + + let ping = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if let Some(ping) = client.get_ping() { + return ping; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + + assert!(ping > std::time::Duration::ZERO); + + let (mut silent_host, silent_cert_pem) = start_host(false).await; + let silent_url = format!("https://127.0.0.1:{}", silent_host.local_addr().port()); + let silent_client = MTPClient::connect( + ClientConfig::new(silent_url) + .with_pinned_pem(silent_cert_pem) + .with_ping_interval(std::time::Duration::from_millis(25)) + .with_max_missed_pings(2), + ) + .await + .unwrap(); + + let _accepted = silent_host.accept().await.unwrap().unwrap(); + + let closed = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if silent_client.sender.is_closed() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await; + + assert!(closed.is_ok(), "client should close after missed pings"); +} diff --git a/transport/Cargo.toml b/transport/Cargo.toml index 69c85b4..6a72928 100644 --- a/transport/Cargo.toml +++ b/transport/Cargo.toml @@ -16,6 +16,7 @@ tokio = { version = "1", features = ["full"] } rustls-native-certs = "0.8.4" log = "0.4" rcgen = "0.14" +tracing = "0.1" [dev-dependencies] diff --git a/transport/src/connection.rs b/transport/src/connection.rs index b4d6e8a..a726a74 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -2,9 +2,10 @@ use crate::ConnectionHandle; use mtp_codec::CommunicationValue; use mtp_common::CommunicationError; use std::sync::Arc; -use tokio::sync::{mpsc, Mutex, RwLock, Semaphore}; +use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; use tokio::time::{Duration, sleep, timeout}; use wtransport::Connection; +use tracing::{debug, info, instrument, trace}; const APPLICATION_CLOSE_REASON: &str = "mtp-close"; @@ -33,6 +34,7 @@ pub struct Policy { pub persistent_stream_retry_backoff: Duration, pub receiver_queue_capacity: usize, pub max_concurrent_stream_tasks: usize, + pub max_frames_per_stream: Option, } impl Default for Policy { @@ -55,6 +57,7 @@ impl Default for Policy { persistent_stream_retry_backoff: Duration::from_millis(20), receiver_queue_capacity: 1000, max_concurrent_stream_tasks: 128, + max_frames_per_stream: None, } } } @@ -114,6 +117,11 @@ impl Policy { self.max_concurrent_stream_tasks = max_concurrent_stream_tasks; self } + + pub fn with_max_frames_per_stream(mut self, max_frames_per_stream: Option) -> Self { + self.max_frames_per_stream = max_frames_per_stream; + self + } } enum ReceivedFrame { @@ -142,6 +150,7 @@ impl Sender { } } + #[instrument(skip(stream, data, policy), level = "trace")] async fn write_frame( stream: &mut wtransport::SendStream, data: &CommunicationValue, @@ -189,6 +198,7 @@ impl Sender { } } + #[instrument(skip(conn, policy), level = "trace")] async fn open_uni_stream( conn: &Connection, policy: &Policy, @@ -221,6 +231,7 @@ impl Sender { } } + #[instrument(skip(conn, stream_opt, data, policy), level = "trace")] async fn send_on_persistent_stream( conn: &Connection, stream_opt: &mut Option, @@ -254,6 +265,7 @@ impl Sender { } } + #[instrument(skip(conn, data, policy), level = "trace")] async fn send_on_single_stream( conn: &Connection, data: &CommunicationValue, @@ -279,6 +291,7 @@ impl Sender { } } + #[instrument(skip(conn, policy), level = "trace")] async fn send_close_frame( conn: &Connection, policy: &Policy, @@ -319,6 +332,7 @@ impl Sender { Ok(()) } + #[instrument(skip(self, data), level = "trace")] pub async fn send(&self, data: &CommunicationValue) -> Result<(), CommunicationError> { if self.handle.is_closed() { return Err(self @@ -374,6 +388,7 @@ impl Sender { } } + #[instrument(skip(self), level = "trace")] pub async fn finish_stream(&self) -> Result<(), CommunicationError> { let _send_lock = self.send_guard.lock().await; let mut stream_opt = self.stream_guard.lock().await; @@ -403,7 +418,9 @@ impl Sender { &self.handle } + #[instrument(skip(self), level = "trace")] pub fn close(&self) { + info!(target = "mtp.transport", "fire-and-forget close requested"); let connection = self.connection.clone(); let handle = self.handle.clone(); let policy = self.policy.clone(); @@ -431,6 +448,7 @@ impl Sender { let _ = Self::send_close_frame(&connection, &policy).await; handle.close(Some(CommunicationError::StreamClosed)); + info!(target = "mtp.transport", "connection closed"); sleep(policy.force_close_delay).await; if connection.quic_connection().close_reason().is_none() { @@ -442,6 +460,51 @@ impl Sender { }); } + #[instrument(skip(self), level = "trace")] + /// Initiate a best-effort graceful close and wait for the configured force-close delay. + pub async fn close_and_wait(&self) { + info!(target = "mtp.transport", "graceful close initiated"); + let connection = self.connection.clone(); + let handle = self.handle.clone(); + let policy = self.policy.clone(); + let mut stream_opt = self.stream_guard.lock().await; + + if connection.quic_connection().close_reason().is_some() || handle.is_closed() { + handle.close(Some(CommunicationError::StreamClosed)); + return; + } + + if let Some(mut stream) = stream_opt.take() { + let close_bytes = policy.close_frame_len.to_be_bytes(); + let close_write = async { + stream.write_all(&close_bytes).await?; + stream.finish().await + }; + + match timeout(policy.write_timeout, close_write).await { + Ok(Ok(())) => {} + Ok(Err(wtransport::error::StreamWriteError::Stopped(code))) => log::warn!( + "[Sender] close_and_wait failed: peer sent STOP_SENDING (error code {code})" + ), + Ok(Err(e)) => log::warn!("[Sender] close_and_wait failed: {e}"), + Err(_) => log::warn!("[Sender] close_and_wait timed out"), + } + } else { + let _ = Self::send_close_frame(&connection, &policy).await; + } + + handle.close(Some(CommunicationError::StreamClosed)); + info!(target = "mtp.transport", "connection closed"); + + sleep(policy.force_close_delay).await; + if connection.quic_connection().close_reason().is_none() { + connection.quic_connection().close( + policy.application_close_code.into(), + APPLICATION_CLOSE_REASON.as_bytes(), + ); + } + } + pub fn is_open(&self) -> bool { self.handle.is_open() } @@ -455,11 +518,17 @@ impl Sender { } } +/// Single-consumer framed message receiver. +/// +/// `receive()` is intended to be driven by one task at a time. Internally the +/// underlying `mpsc::Receiver` is protected by a mutex so the type remains +/// `Sync`, but it is not a multi-consumer queue. pub struct Receiver { rx: Mutex>>, _accept_task: tokio::task::JoinHandle<()>, handle: Arc, ping_control: Arc>, + queue_notify: Arc, } #[derive(Clone, Default)] @@ -491,13 +560,40 @@ impl Receiver { let accept_policy = policy.clone(); let ping_control = Arc::new(RwLock::new(PingControl::default())); let accept_ping_control = ping_control.clone(); + let queue_notify = Arc::new(Notify::new()); + let accept_queue_notify = queue_notify.clone(); let stream_limit = Arc::new(Semaphore::new(policy.max_concurrent_stream_tasks.max(1))); let accept_stream_limit = stream_limit.clone(); + debug!( + target = "mtp.transport", + max_concurrent_stream_tasks = policy.max_concurrent_stream_tasks, + receiver_queue_capacity = policy.receiver_queue_capacity, + "receiver accept loop started" + ); + info!( + target = "mtp.transport", + max_concurrent_stream_tasks = policy.max_concurrent_stream_tasks, + receiver_queue_capacity = policy.receiver_queue_capacity, + "connection accepted" + ); let accept_task = tokio::spawn(async move { let mut close_rx = conn_handle.subscribe_close(); loop { + if tx.capacity() == 0 { + trace!(target = "mtp.transport", "accept loop paused: receiver queue full"); + tokio::select! { + _ = close_rx.changed() => { + if close_rx.borrow().is_some() { + break; + } + } + _ = accept_queue_notify.notified() => {} + } + continue; + } + tokio::select! { _ = close_rx.changed() => { if close_rx.borrow().is_some() { @@ -523,9 +619,20 @@ impl Receiver { tokio::spawn(async move { let _permit = permit; let mut s = stream; + let mut frame_count = 0usize; loop { + if let Some(max_frames) = stream_policy.max_frames_per_stream + && frame_count >= max_frames + { + let close_error = CommunicationError::StreamError; + let _ = tx_stream.send(Err(close_error.clone())).await; + stream_handle.close(Some(close_error)); + break; + } + match Self::read_one_frame(&mut s, &stream_policy).await { Ok(ReceivedFrame::Message(msg)) => { + frame_count += 1; let ping_type = mtp_codec::CommunicationType::Ping .to_id(&mtp_codec::TypeMap::latest()); let pong_type = mtp_codec::CommunicationType::Pong @@ -630,6 +737,7 @@ impl Receiver { _accept_task: accept_task, handle, ping_control, + queue_notify, } } @@ -651,6 +759,7 @@ impl Receiver { } } + #[instrument(skip(stream, policy), level = "trace")] async fn read_one_frame( stream: &mut wtransport::RecvStream, policy: &Policy, @@ -726,6 +835,7 @@ impl Receiver { Ok(ReceivedFrame::Message(message)) } + #[instrument(skip(self), level = "trace")] pub async fn receive(&self) -> Result { if self.handle.is_closed() { return Err(self @@ -736,7 +846,10 @@ impl Receiver { let mut rx = self.rx.lock().await; match rx.recv().await { - Some(result) => result, + Some(result) => { + self.queue_notify.notify_one(); + result + } _ => Err(self .handle .close_reason() @@ -796,6 +909,7 @@ mod tests { assert_eq!(p.persistent_stream_retry_backoff, Duration::from_millis(20)); assert_eq!(p.receiver_queue_capacity, 1000); assert_eq!(p.max_concurrent_stream_tasks, 128); + assert_eq!(p.max_frames_per_stream, None); } #[test] diff --git a/transport/tests/integration.rs b/transport/tests/integration.rs index 1099b9e..276c975 100644 --- a/transport/tests/integration.rs +++ b/transport/tests/integration.rs @@ -199,3 +199,149 @@ async fn test_drop_receiver_keeps_sender_alive() { client_tx.close(); host_tx.close(); } + +#[tokio::test] +async fn test_persistent_stream_reopens_after_local_finish() { + let (_h, client_tx, client_rx, host_tx, host_rx) = connected_pair().await; + + let tm = TypeMap::latest(); + let msg1 = numbered_message(CommunicationType::Ping, 11, &tm); + client_tx.send(&msg1).await.unwrap(); + let received1 = host_rx.receive().await.unwrap(); + assert_numbered_message(&received1, CommunicationType::Ping, 11, &tm); + + client_tx.finish_stream().await.unwrap(); + + let msg2 = numbered_message(CommunicationType::Pong, 22, &tm); + client_tx.send(&msg2).await.unwrap(); + let received2 = host_rx.receive().await.unwrap(); + assert_numbered_message(&received2, CommunicationType::Pong, 22, &tm); + + client_tx.close(); + host_tx.close(); + drop(client_rx); +} + +#[tokio::test] +async fn test_receiver_backpressure_with_small_queue() { + let (cert_pem, key_pem) = generate_self_signed_cert(); + let mut h = start_test_host(cert_pem.clone(), key_pem).await; + let url = format!("https://127.0.0.1:{}", h.local_addr().port()); + let policy = Policy::default().with_receiver_queue_capacity(1); + let (client_tx, client_rx) = connect(&url, Some(cert_pem), policy.clone()) + .await + .unwrap(); + let (_host_tx, host_rx) = h.next().await.unwrap(); + + let tm = TypeMap::latest(); + for i in 0..8u128 { + client_tx + .send(&numbered_message(CommunicationType::Ping, i, &tm)) + .await + .unwrap(); + } + + for i in 0..8u128 { + let received = tokio::time::timeout( + std::time::Duration::from_secs(5), + host_rx.receive(), + ) + .await + .unwrap() + .unwrap(); + assert_numbered_message(&received, CommunicationType::Ping, i, &tm); + } + + client_tx.close(); + drop(client_rx); + h.shutdown(); +} + +#[tokio::test] +async fn test_max_frames_per_stream_enforced() { + let (cert_pem, key_pem) = generate_self_signed_cert(); + let policy = Policy::default().with_max_frames_per_stream(Some(1)); + let mut h = host( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 0, + cert_pem.clone(), + key_pem, + policy, + ) + .await + .unwrap(); + let url = format!("https://127.0.0.1:{}", h.local_addr().port()); + let (client_tx, _client_rx) = connect(&url, Some(cert_pem), Policy::default()) + .await + .unwrap(); + let (_host_tx, host_rx) = h.next().await.unwrap(); + + let tm = TypeMap::latest(); + client_tx + .send(&numbered_message(CommunicationType::Ping, 1, &tm)) + .await + .unwrap(); + let first = host_rx.receive().await.unwrap(); + assert_numbered_message(&first, CommunicationType::Ping, 1, &tm); + + client_tx + .send(&numbered_message(CommunicationType::Ping, 2, &tm)) + .await + .unwrap(); + let second = host_rx.receive().await; + assert!(second.is_err(), "stream should be closed after frame limit"); + + client_tx.close(); + h.shutdown(); +} + +#[tokio::test] +async fn test_semaphore_saturation_with_concurrent_streams() { + let (cert_pem, key_pem) = generate_self_signed_cert(); + let policy = Policy::default() + .with_send_mode(mtp_transport::SendMode::SingleStreamPerMessage) + .with_receiver_queue_capacity(1) + .with_max_concurrent_stream_tasks(1); + let mut h = host( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 0, + cert_pem.clone(), + key_pem, + policy, + ) + .await + .unwrap(); + let url = format!("https://127.0.0.1:{}", h.local_addr().port()); + let (client_tx, _client_rx) = connect(&url, Some(cert_pem), Policy::default()) + .await + .unwrap(); + let (_host_tx, host_rx) = h.next().await.unwrap(); + + let tm = TypeMap::latest(); + let mut joins = Vec::new(); + for i in 0..6u128 { + let tx = client_tx.clone(); + let msg = numbered_message(CommunicationType::Ping, i, &tm); + joins.push(tokio::spawn(async move { tx.send(&msg).await })); + } + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + for join in joins { + join.await.unwrap().unwrap(); + } + + for i in 0..6u128 { + let received = tokio::time::timeout( + std::time::Duration::from_secs(5), + host_rx.receive(), + ) + .await + .unwrap() + .unwrap(); + assert_numbered_message(&received, CommunicationType::Ping, i, &tm); + } + + client_tx.close(); + h.shutdown(); +} From 69be9f7acaf95bc031f82ce0e9ef68adbd3b17f8 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:59:41 +0200 Subject: [PATCH 34/97] [Fix] export Sendmode in Host (not a stream feat) --- host/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/host/src/lib.rs b/host/src/lib.rs index 44d73a9..1477d33 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -14,7 +14,6 @@ pub use MTPConnection as Connection; pub use MTPHost as Host; pub use mtp_transport::Policy; pub use mtp_transport::Receiver; -#[cfg(feature = "streaming")] pub use mtp_transport::SendMode; pub use mtp_transport::Sender; From 089def45d111a5cd77cf1f39c6ef1cfb3f27a0b6 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:42:54 +0200 Subject: [PATCH 35/97] [Add] Pipes (experimental) --- Cargo.lock | 4 +- Cargo.toml | 15 +- client/Cargo.toml | 3 +- client/src/lib.rs | 294 +++++++++++++++++++- codec/Cargo.toml | 2 +- common/Cargo.toml | 3 + common/src/lib.rs | 91 +++++++ docs/NATIVE-CLIENT.md | 130 ++++++++- docs/NATIVE-HOST.md | 130 +++++++++ docs/WASM-CLIENT.md | 120 +++++++++ example/.gitignore | 1 + example/Cargo.lock | 6 + example/client.id | 1 + example/client/Cargo.toml | 3 +- example/client/src/main.rs | 5 + example/client/src/messages.rs | 3 +- example/client/src/pipes.rs | 137 ++++++++++ example/server/Cargo.toml | 3 +- example/server/src/main.rs | 106 ++++++-- example/server/src/tls.rs | 30 ++- example/web-client/index.html | 116 +++++--- example/web-client/src/main.ts | 284 ++++++++++++++++--- example/web-client/vite.config.ts | 31 ++- host/Cargo.toml | 8 +- host/src/lib.rs | 434 +++++++++++++++++++++++++++--- src/sdk/index.ts | 99 +++++++ transport/Cargo.toml | 9 +- transport/src/connection.rs | 338 ++++++++++++++++++++--- transport/src/lib.rs | 8 + transport/src/pipe.rs | 69 +++++ type-map/Cargo.toml | 3 +- type-map/build.rs | 19 ++ wasm/Cargo.toml | 6 +- wasm/src/client.rs | 229 +++++++++++++++- wasm/src/lib.rs | 1 + wasm/src/pipe.rs | 140 ++++++++++ wasm/src/transport.rs | 142 +++++++++- 37 files changed, 2795 insertions(+), 228 deletions(-) create mode 100644 example/client.id create mode 100644 example/client/src/pipes.rs create mode 100644 transport/src/pipe.rs create mode 100644 wasm/src/pipe.rs diff --git a/Cargo.lock b/Cargo.lock index d84909c..875b98f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1163,9 +1163,9 @@ dependencies = [ [[package]] name = "octets" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8311fa8ab7a57759b4ff1f851a3048d9ef0effaa0130726426b742d26d8a88e7" +checksum = "866cb5af6f3aa3c1b44c3c2d79d22165fbb1b102e1b3fb499864bfe34736ec4b" [[package]] name = "oid-registry" diff --git a/Cargo.toml b/Cargo.toml index 8c0dbdf..02a9b96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,8 +62,6 @@ mtp-client = { version = "0.1.0", path = "client", optional = true } mtp-files = { version = "0.1.0", path = "files", optional = true } [features] -default = [] - # Serialization serde = ["mtp-crypto/serde"] @@ -82,19 +80,14 @@ host = ["dep:mtp-host", "mtp-codec/registry", "transport"] # MTP client - outgoing QUIC connections to a host. client = ["dep:mtp-client", "transport"] -# Opt into stream-specific host/client facade APIs. The transport itself is -# always framed over QUIC/WebTransport streams for compatibility. -streaming = [ - "transport", - "mtp-transport/streaming", - "mtp-host?/streaming", - "mtp-client?/streaming", -] - # Direct access to the framed QUIC transport. Host/client features enable it # automatically; this feature is useful for low-level integrations. transport = ["dep:mtp-transport"] +# Direct access to the pipes. Pipes can be used to send raw binary +# without after creation overhead. +pipes = ["mtp-common/pipes", "mtp-codec/pipes", "mtp-transport?/pipes", "mtp-host?/pipes", "mtp-client?/pipes"] + # On-disk storage for keyrings (`.mk`) and public key bundles (`.mpkb`). # Pulls in `crypto` so the `Keyring` / `PublicKeyBundle` types are in scope. files = ["dep:mtp-files", "crypto"] diff --git a/client/Cargo.toml b/client/Cargo.toml index 693378e..cdf04e8 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -19,5 +19,4 @@ rcgen = "0.14" [features] crypto = ["dep:mtp-crypto", "mtp-codec/crypto"] -# Enables stream-specific convenience exports and configuration. -streaming = ["mtp-transport/streaming"] +pipes = ["mtp-common/pipes", "mtp-transport/pipes"] diff --git a/client/src/lib.rs b/client/src/lib.rs index 860ae85..b9ea6ac 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,11 +1,16 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Version}; use mtp_common::CommunicationError; +#[cfg(feature = "pipes")] +pub use mtp_common::PipeError; use std::collections::HashMap; use std::sync::Arc; use rand::Rng; use tokio::sync::{Mutex, mpsc}; use tokio::time::{Duration, Instant}; +#[cfg(feature = "pipes")] +use mtp_transport::PipeReader; + pub use MTPClient as Client; pub use MTPConnection as Connection; pub use mtp_transport::Policy; @@ -13,6 +18,9 @@ pub use mtp_transport::Receiver; pub use mtp_transport::SendMode; pub use mtp_transport::Sender; +#[cfg(feature = "pipes")] +pub use mtp_transport::PipeWriter; + #[cfg(feature = "crypto")] fn unexpected_response_type_error( context: &str, @@ -27,6 +35,162 @@ fn unexpected_response_type_error( )) } +#[cfg(feature = "pipes")] +pub struct PipeHandle { + pipe_id: u32, + description: String, + sender: Sender, + response_rx: tokio::sync::oneshot::Receiver>, +} + +#[cfg(feature = "pipes")] +impl PipeHandle { + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } + + pub fn description(&self) -> &str { + &self.description + } + + pub async fn wait(self) -> Result, PipeError> { + match self.response_rx.await { + Ok(Ok(true)) => { + let writer = self + .sender + .open_pipe(self.pipe_id, &self.description) + .await + .map_err(PipeError::from)?; + Ok(Some(writer)) + } + Ok(Ok(false)) => Ok(None), + Ok(Err(e)) => Err(e), + Err(_) => Err(PipeError::StreamClosed), + } + } +} + +#[cfg(feature = "pipes")] +pub struct PipeRequest { + pipe_id: u32, + description: String, + sender: Sender, + dispatcher: Arc, +} + +#[cfg(feature = "pipes")] +impl PipeRequest { + pub fn id(&self) -> u32 { + self.pipe_id + } + + pub fn description(&self) -> &str { + &self.description + } + + pub async fn accept(self) -> Result { + let (pipe_tx, pipe_rx) = tokio::sync::oneshot::channel(); + { + let mut pending = self.dispatcher.pending_pipes.lock().await; + pending.insert(self.pipe_id, pipe_tx); + } + + let resp = CommunicationValue::new(mtp_codec::CommunicationType::PipeResponse) + .with_id(self.pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolTrue); + self.sender.send(&resp).await.map_err(PipeError::from)?; + + let timeout = self.dispatcher.policy.read_timeout; + tokio::time::timeout(timeout, pipe_rx) + .await + .map_err(|_| PipeError::HandshakeTimeout)? + .map_err(|_| PipeError::StreamClosed) + } + + pub async fn deny(self) -> Result<(), PipeError> { + let resp = CommunicationValue::new(mtp_codec::CommunicationType::PipeResponse) + .with_id(self.pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolFalse); + self.sender.send(&resp).await.map_err(PipeError::from)?; + Ok(()) + } +} + +#[cfg(feature = "pipes")] +struct PipeDispatcher { + pending_creations: Mutex>>>, + pending_pipes: Mutex>>, + policy: Arc, +} + +#[cfg(not(feature = "pipes"))] +struct PipeDispatcher; + +#[cfg(not(feature = "pipes"))] +pub(crate) struct PipeRequest; + +#[cfg(feature = "pipes")] +async fn run_dispatcher( + receiver: Receiver, + sender: Sender, + app_tx: mpsc::Sender>, + pipe_req_tx: mpsc::Sender, + dispatcher: Arc, +) { + let pipe_req_type = + mtp_codec::CommunicationType::PipeRequest.to_id(&mtp_codec::TypeMap::latest()); + let pipe_resp_type = + mtp_codec::CommunicationType::PipeResponse.to_id(&mtp_codec::TypeMap::latest()); + + loop { + match receiver.receive_event().await { + Ok(mtp_transport::TransportEvent::Message(msg)) => { + if msg.get_type() == pipe_req_type { + let pipe_id = msg.get_id(); + let description = msg + .get_str(DataType::Description) + .unwrap_or("") + .to_string(); + let req = PipeRequest { + pipe_id, + description, + sender: sender.clone(), + dispatcher: dispatcher.clone(), + }; + let _ = pipe_req_tx.send(req).await; + continue; + } + + if msg.get_type() == pipe_resp_type { + let pipe_id = msg.get_id(); + let accepted = msg.get_bool(DataType::Accepted).unwrap_or(false); + let mut pending = dispatcher.pending_creations.lock().await; + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(Ok(accepted)); + } + continue; + } + + if app_tx.send(Ok(msg)).await.is_err() { + break; + } + } + Ok(mtp_transport::TransportEvent::Pipe(reader)) => { + let pipe_id = reader.pipe_id(); + let mut pending = dispatcher.pending_pipes.lock().await; + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(reader); + } + } + Err(e) => { + if app_tx.send(Err(e)).await.is_err() { + break; + } + } + } + } +} + pub struct ClientConfig { pub url: String, pub tls: ClientTlsConfig, @@ -129,6 +293,10 @@ pub struct MTPConnection { pub receiver: Receiver, pub description: Option, ping: Option, + app_rx: Mutex>>, + pipe_req_rx: Mutex>, + pipe_dispatcher: Arc, + _dispatcher_task: tokio::task::JoinHandle<()>, #[cfg(feature = "crypto")] pub auth_state: AuthState, #[cfg(feature = "crypto")] @@ -180,7 +348,7 @@ impl MTPConnection { let tm = mtp_codec::TypeMap::latest(); loop { - let response = self.receiver.receive().await?; + let response = self.receive().await?; if response.get_id() != request_id { continue; } @@ -200,6 +368,55 @@ impl MTPConnection { return Ok(response); } } + + pub async fn receive(&self) -> Result { + #[cfg(feature = "pipes")] + { + let mut rx = self.app_rx.lock().await; + match rx.recv().await { + Some(result) => result, + None => Err(CommunicationError::StreamClosed), + } + } + #[cfg(not(feature = "pipes"))] + { + self.receiver.receive().await + } + } +} + +#[cfg(feature = "pipes")] +impl MTPConnection { + pub async fn create_pipe(&self, description: &str) -> Result { + let pipe_id = rand::random::(); + let (tx, rx) = tokio::sync::oneshot::channel(); + + { + let mut pending = self.pipe_dispatcher.pending_creations.lock().await; + pending.insert(pipe_id, tx); + } + + let request = CommunicationValue::new(mtp_codec::CommunicationType::PipeRequest) + .with_id(pipe_id) + .add_typed_default(DataType::Description, DataValue::Str(description.into())); + + self.sender.send(&request).await.map_err(PipeError::from)?; + + Ok(PipeHandle { + pipe_id, + description: description.to_string(), + sender: self.sender.clone(), + response_rx: rx, + }) + } + + pub async fn receive_pipe(&self) -> Result { + let mut rx = self.pipe_req_rx.lock().await; + match rx.recv().await { + Some(req) => Ok(req), + None => Err(CommunicationError::StreamClosed), + } + } } fn start_ping_session( @@ -287,16 +504,71 @@ fn connection_from_parts( #[cfg(feature = "crypto")] client_id: u64, ) -> MTPConnection { let ping = start_ping_session(&config, sender.clone(), &receiver); - MTPConnection { - version: PROTOCOL_VERSION, - sender, - receiver, - description: config.description, - ping, - #[cfg(feature = "crypto")] - auth_state, - #[cfg(feature = "crypto")] - client_id, + + #[cfg(feature = "pipes")] + { + let (app_tx, app_rx) = mpsc::channel::>( + config.policy.receiver_queue_capacity, + ); + let (pipe_req_tx, pipe_req_rx) = mpsc::channel::( + config.policy.receiver_queue_capacity, + ); + + let dispatcher = Arc::new(PipeDispatcher { + pending_creations: Mutex::new(HashMap::new()), + pending_pipes: Mutex::new(HashMap::new()), + policy: Arc::new(config.policy), + }); + + let dispatcher_clone = dispatcher.clone(); + let sender_clone = sender.clone(); + let dispatcher_task = tokio::spawn(run_dispatcher( + receiver.clone(), + sender_clone, + app_tx, + pipe_req_tx, + dispatcher_clone, + )); + + MTPConnection { + version: PROTOCOL_VERSION, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description: config.description, + ping, + _dispatcher_task: dispatcher_task, + #[cfg(feature = "crypto")] + auth_state, + #[cfg(feature = "crypto")] + client_id, + } + } + + #[cfg(not(feature = "pipes"))] + { + let (_, app_rx) = mpsc::channel::>(1); + let (_, pipe_req_rx) = mpsc::channel::(1); + let dispatcher = Arc::new(PipeDispatcher); + let task = tokio::spawn(async {}); + + MTPConnection { + version: PROTOCOL_VERSION, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description: config.description, + ping, + _dispatcher_task: task, + #[cfg(feature = "crypto")] + auth_state, + #[cfg(feature = "crypto")] + client_id, + } } } diff --git a/codec/Cargo.toml b/codec/Cargo.toml index 51a5cde..8b4ea98 100644 --- a/codec/Cargo.toml +++ b/codec/Cargo.toml @@ -12,6 +12,6 @@ byteorder = "1.5" rand = { version = "0.8", features = ["std", "std_rng"] } [features] -default = [] registry = ["mtp-type-map/registry"] crypto = ["dep:mtp-crypto", "mtp-crypto/mlkem-tls"] +pipes = ["mtp-type-map/pipes"] diff --git a/common/Cargo.toml b/common/Cargo.toml index fe1a703..460e724 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -6,6 +6,9 @@ edition = "2024" [dependencies] thiserror = "2.0.18" +[features] +pipes = [] + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] wtransport = { version = "0.7.1", default-features = false, features = [ "aws-lc-rs", diff --git a/common/src/lib.rs b/common/src/lib.rs index 71dea88..773bfca 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -244,6 +244,45 @@ impl Eq for CommunicationError {} #[cfg(target_arch = "wasm32")] impl Eq for CommunicationError {} +/* ================================ PipeError ================================ */ + +#[cfg(feature = "pipes")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PipeError { + Rejected, + HandshakeTimeout, + StreamClosed, + IoError(String), + ConnectionClosed, +} + +#[cfg(feature = "pipes")] +impl std::fmt::Display for PipeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PipeError::Rejected => write!(f, "pipe request was rejected"), + PipeError::HandshakeTimeout => write!(f, "pipe handshake timed out"), + PipeError::StreamClosed => write!(f, "pipe stream closed unexpectedly"), + PipeError::IoError(s) => write!(f, "pipe I/O error: {s}"), + PipeError::ConnectionClosed => write!(f, "connection closed"), + } + } +} + +#[cfg(feature = "pipes")] +impl std::error::Error for PipeError {} + +#[cfg(feature = "pipes")] +impl From for PipeError { + fn from(e: CommunicationError) -> Self { + match e { + CommunicationError::StreamClosed => PipeError::StreamClosed, + CommunicationError::ConnectionError(_) => PipeError::ConnectionClosed, + other => PipeError::IoError(other.to_string()), + } + } +} + /* ================================ TESTS ================================ */ #[cfg(test)] mod communication_error_tests { @@ -291,3 +330,55 @@ mod communication_error_tests { assert!(format!("{}", e).contains("refused")); } } + +/* ================================ PipeError TESTS ================================ */ +#[cfg(feature = "pipes")] +#[cfg(test)] +mod pipe_error_tests { + use super::*; + + #[test] + fn test_pipe_error_display() { + assert_eq!( + format!("{}", PipeError::Rejected), + "pipe request was rejected" + ); + assert_eq!( + format!("{}", PipeError::HandshakeTimeout), + "pipe handshake timed out" + ); + assert_eq!( + format!("{}", PipeError::StreamClosed), + "pipe stream closed unexpectedly" + ); + assert_eq!( + format!("{}", PipeError::ConnectionClosed), + "connection closed" + ); + assert_eq!( + format!("{}", PipeError::IoError("boom".into())), + "pipe I/O error: boom" + ); + } + + #[test] + fn test_pipe_error_from_stream_closed() { + let pe: PipeError = CommunicationError::StreamClosed.into(); + assert_eq!(pe, PipeError::StreamClosed); + } + + #[test] + fn test_pipe_error_from_connection_error() { + let pe: PipeError = CommunicationError::ConnectionError( + wtransport::error::ConnectionError::TimedOut, + ) + .into(); + assert_eq!(pe, PipeError::ConnectionClosed); + } + + #[test] + fn test_pipe_error_from_other() { + let pe: PipeError = CommunicationError::StreamError.into(); + assert_eq!(pe, PipeError::IoError("Stream Error".into())); + } +} diff --git a/docs/NATIVE-CLIENT.md b/docs/NATIVE-CLIENT.md index 421828e..c304575 100644 --- a/docs/NATIVE-CLIENT.md +++ b/docs/NATIVE-CLIENT.md @@ -12,6 +12,9 @@ mtp = { path = "/path/to/mtp", features = ["client"] } # Add crypto for auth_connect / auth_register: mtp = { path = "/path/to/mtp", features = ["client", "crypto"] } + +# Add pipes for raw binary streams: +mtp = { path = "/path/to/mtp", features = ["client", "pipes"] } ``` ## ClientConfig @@ -28,16 +31,18 @@ let config = ClientConfig::new("https://host.example.com:4433") .with_ping_timestamp(true); ``` -| Field | Type | Description | -|---------------|--------------------|-----------------------------------------------------| -| `url` | `String` | `https://host:port` address of the MTP host | -| `tls` | `ClientTlsConfig` | `SystemRoots` or `PinnedPem(pem_bytes)` | -| `client_id` | `u64` | Client identifier (ignored during `auth_register`) | -| `description` | `Option` | Optional label sent during handshake (e.g. `"phone"`) | -| `ping_interval` | `Duration` | Interval between Ping frames; zero disables pings | -| `max_missed_pings` | `usize` | Unanswered Ping frames allowed before the connection closes | -| `ping_timestamp` | `bool` | Adds a `Timestamp` entry to each Ping frame | -| `auth_timeout` | `Duration` (crypto) | Authentication handshake timeout (default 30s) | +| Field | Type | Default | Description | +|-------------------------|--------------------|------------------|---------------------------------------------| +| `url` | `String` | required | Host URL (`https://host:port`) | +| `tls` | `ClientTlsConfig` | `SystemRoots` | `SystemRoots` or `PinnedPem(Vec)` | +| `client_id` | `u64` | `0` | Client identifier (for login) | +| `description` | `Option` | `None` | Optional label sent to host | +| `policy` | `Policy` | default | Transport policy (timeouts, send mode) | +| `ping_interval` | `Duration` | `Duration::ZERO` | Interval between protocol Ping frames | +| `ping_jitter` | `Option` | `None` | Random jitter added to each interval | +| `max_missed_pings` | `usize` | `3` | Disconnect after this many unanswered Pings | +| `ping_timestamp` | `bool` | `true` | Include a `Timestamp` data entry in Ping | +| `auth_timeout` (crypto) | `Duration` | `30s` | Max time for auth handshake | ### TLS Certificate Handling @@ -310,6 +315,111 @@ Sends a close frame and signals the peer. The `Sender::close()` spawns an async task that sends the frame, waits for `force_close_delay` (default 300ms), then force-closes the QUIC connection if the peer has not already done so. +## Pipes + +With the `pipes` feature enabled, the client can open **raw binary streams** +to the host. A Pipe is a unidirectional QUIC stream that carries a lightweight +`PipeRequest` handshake frame, then transitions to raw bytes with zero per-frame +overhead. + +### Enabling Pipes + +Add the `pipes` feature to your dependency: + +```toml +[dependencies] +mtp = { path = "/path/to/mtp", features = ["client", "pipes"] } +``` + +### Creating a Pipe + +```rust +use mtp::client::MTPClient; +use tokio::io::AsyncWriteExt; + +let conn = MTPClient::connect(config).await?; + +// Initiate a pipe request +let handle = conn.create_pipe("file-transfer").await?; + +// Wait for the host to accept or reject +match handle.wait().await? { + Some(mut writer) => { + writer.write_all(b"raw binary data").await?; + writer.finish().await?; // graceful close + } + None => { + println!("host rejected the pipe"); + } +} +``` + +### PipeHandle + +```rust +pub struct PipeHandle { + pipe_id: u32, + description: String, +} +``` + +| Method | Returns | Description | +|--------|---------|-------------| +| `wait()` | `Result, PipeError>` | Block until the host responds. `Some(writer)` if accepted, `None` if rejected. | + +`PipeHandle` consumes itself on `wait()`, so you cannot poll it multiple times. + +### PipeWriter + +```rust +pub struct PipeWriter { + // wraps a QUIC SendStream +} +``` + +`PipeWriter` implements `tokio::io::AsyncWrite`. After the handshake succeeds, +writes go directly to the QUIC stream with no framing overhead. + +| Method | Returns | Description | +|--------|---------|-------------| +| `finish()` | `Result<(), CommunicationError>` | Gracefully close the stream (sends FIN) | +| `abort()` | `Result<(), ClosedStream>` | Abruptly reset the stream | + +```rust +use tokio::io::AsyncWriteExt; + +let mut writer = handle.wait().await?.unwrap(); +writer.write_all(b"chunk 1").await?; +writer.write_all(b"chunk 2").await?; +writer.finish().await?; +``` + +### PipeError + +```rust +pub enum PipeError { + Rejected, // pipe request was rejected + HandshakeTimeout, // pipe handshake timed out + StreamClosed, // pipe stream closed unexpectedly + IoError(String), // pipe I/O error + ConnectionClosed, // connection closed +} +``` + +`PipeError` implements `std::error::Error` and can be converted from +`CommunicationError` via `PipeError::from()`. + +### Do Not Use `receiver.receive()` for Pipes + +When the `pipes` feature is active, `conn.receiver.receive()` will **skip** +`PipeResponse` frames and may return them as ordinary messages if called from +the wrong task. Use the facade methods: + +- `conn.receive()` to receive normal `CommunicationValue` messages +- `conn.create_pipe(description)` to initiate a new pipe + +These methods are internally synchronised and safe to call from separate tasks. + ## Crypto Containers With the `crypto` feature, `DataValue` supports encrypted, signed, and diff --git a/docs/NATIVE-HOST.md b/docs/NATIVE-HOST.md index 324cd66..eb46607 100644 --- a/docs/NATIVE-HOST.md +++ b/docs/NATIVE-HOST.md @@ -13,6 +13,9 @@ mtp = { path = "/path/to/mtp", features = ["host"] } # Add crypto for authenticated connections: mtp = { path = "/path/to/mtp", features = ["host", "crypto"] } + +# Add pipes for raw binary streams: +mtp = { path = "/path/to/mtp", features = ["host", "pipes"] } ``` ## HostConfig @@ -306,6 +309,133 @@ let desc_id = DataTypeId(tm.data_id_enum(DataType::Description).unwrap()); let value = msg.get_data(desc_id); ``` +## Pipes + +With the `pipes` feature enabled, the host can accept **raw binary streams** +from clients. A Pipe is a unidirectional QUIC stream opened by the client that +carries a lightweight `PipeRequest` handshake frame, then transitions to raw +bytes with zero per-frame overhead. + +### Enabling Pipes + +Add the `pipes` feature to your dependency: + +```toml +[dependencies] +mtp = { path = "/path/to/mtp", features = ["host", "pipes"] } +``` + +### Receiving Pipe Requests + +When `pipes` is enabled, **do not call `conn.receiver.receive()` directly**. +Instead, use `conn.receive()` for normal messages and `conn.receive_pipe()` +for incoming pipe requests. A background dispatcher task routes events +internally so the two channels do not race. + +```rust +use mtp::host::{MTPHost, PipeRequest}; +use tokio::io::AsyncReadExt; + +while let Some(conn) = host.accept().await? { + tokio::spawn(async move { + loop { + tokio::select! { + Ok(msg) = conn.receive() => { + // handle normal CommunicationValue + } + Ok(req) = conn.receive_pipe() => { + handle_pipe(req).await; + } + else => break, + } + } + }); +} + +async fn handle_pipe(req: PipeRequest) { + println!("Pipe {} requested: {}", req.id(), req.description()); + // Accept or deny... +} +``` + +### PipeRequest + +```rust +pub struct PipeRequest { + // pipe_id assigned by the creator + // description provided by the creator +} +``` + +| Method | Returns | Description | +|--------|---------|-------------| +| `id()` | `u32` | The pipe ID chosen by the creator | +| `description()` | `&str` | Creator-provided label (e.g. `"file-transfer"`) | +| `accept()` | `Result` | Accept the pipe; returns an `AsyncRead` stream | +| `deny()` | `Result<(), PipeError>` | Reject the pipe | + +### Accepting a Pipe + +```rust +use tokio::io::AsyncReadExt; + +async fn handle_pipe(req: PipeRequest) { + match req.accept().await { + Ok(mut reader) => { + let mut buf = Vec::new(); + if let Err(e) = reader.read_to_end(&mut buf).await { + eprintln!("pipe read error: {e}"); + } + println!("received {} bytes", buf.len()); + } + Err(e) => { + eprintln!("pipe accept failed: {e}"); + } + } +} +``` + +`PipeReader` implements `tokio::io::AsyncRead`. The stream reads until the +creator calls `PipeWriter::finish()` or the connection closes. + +### Rejecting a Pipe + +```rust +async fn handle_pipe(req: PipeRequest) { + if !should_allow(&req) { + req.deny().await.ok(); + return; + } + // ... accept +} +``` + +### PipeError + +```rust +pub enum PipeError { + Rejected, // pipe request was rejected + HandshakeTimeout, // pipe handshake timed out + StreamClosed, // pipe stream closed unexpectedly + IoError(String), // pipe I/O error + ConnectionClosed, // connection closed +} +``` + +`PipeError` implements `std::error::Error` and can be converted from +`CommunicationError` via `PipeError::from()`. + +### Important: Do Not Use `receiver.receive()` with Pipes + +When the `pipes` feature is active, `conn.receiver.receive()` will **skip** +`PipeRequest` frames and may return them as ordinary messages if called from +the wrong task. Always use the facade methods: + +- `conn.receive()` -- normal `CommunicationValue` messages +- `conn.receive_pipe()` -- incoming `PipeRequest` objects + +These methods are internally synchronised and safe to call from separate tasks. + ## Host Callbacks ### get_existing_user diff --git a/docs/WASM-CLIENT.md b/docs/WASM-CLIENT.md index 2802036..c21334e 100644 --- a/docs/WASM-CLIENT.md +++ b/docs/WASM-CLIENT.md @@ -231,6 +231,82 @@ await MTPClient.create({ Use `pings: true` for the default interval. +## Pipes + +Pipes are raw binary streams over QUIC. A pipe starts with a lightweight `PipeRequest` handshake frame, then the stream carries raw bytes with zero per-frame overhead. Pipes are unidirectional; the peer that initiates the pipe writes, and the peer that accepts it reads. + +### Outgoing Pipes + +`createPipe` sends a `PipeRequest` frame and returns a handle. Call `wait()` to block until the remote peer accepts or denies: + +```typescript +const handle = await client.createPipe("file-transfer"); + +const writer = await handle.wait(); +if (writer == null) { + console.log("host denied the pipe"); + return; +} + +await writer.write(new Uint8Array([0x01, 0x02, 0x03])); +await writer.write(chunk); +await writer.close(); +``` + +`writer.close()` sends a QUIC stream FIN. `writer.abort()` resets the stream abruptly. Each `write` resolves when the chunk has been handed to the transport; it does not wait for the peer to consume it. + +The handle and writer expose `pipeId` and `description`: + +```typescript +console.log(handle.pipeId, handle.description); +console.log(writer.pipeId); +``` + +### Incoming Pipes + +Set a handler to receive pipe requests from the remote peer: + +```typescript +client.setOnPipeRequest((request) => { + console.log("incoming pipe", request.pipeId, request.description); + // accept or deny asynchronously +}); +``` + +Accept a request to receive a `PipeReader`: + +```typescript +client.setOnPipeRequest(async (request) => { + if (request.description === "file-transfer") { + const reader = await client.acceptPipe(request.pipeId); + + while (true) { + const chunk = await reader.read(); + if (chunk == null) break; // stream closed by peer + processChunk(chunk); + } + } else { + await client.denyPipe(request.pipeId); + } +}); +``` + +`reader.read()` resolves with a `Uint8Array` or `null` when the peer closes the stream. The reader exposes `pipeId` and `description`: + +```typescript +console.log(reader.pipeId, reader.description); +``` + +### Pipe Handshake + +1. The initiator calls `createPipe(description)`; the SDK sends a `PipeRequest` frame with a random `pipeId` and the description. +2. The receiver's `setOnPipeRequest` callback fires with `{ pipeId, description }`. +3. The receiver calls `acceptPipe(pipeId)`; the SDK sends a `PipeResponse` with `Accepted = true` and opens a new unidirectional stream for raw data. +4. The initiator's `handle.wait()` resolves with a `PipeWriter` bound to that stream. +5. If the receiver calls `denyPipe(pipeId)`, `handle.wait()` resolves with `null`. + +Pipes share the same WebTransport session as message frames; they do not need a separate connection. + ## Logger Events The SDK logger receives parsed events: @@ -333,4 +409,48 @@ const confirmedId = await rawClient.auth_connect( ); ``` +### Raw Pipes + +The raw `WasmClient` exposes the same pipe operations as the SDK wrapper: + +```typescript +// Incoming pipe requests +rawClient.set_on_pipe_request((event) => { + const { pipeId, description } = event; + // accept or deny +}); + +// Outgoing pipe +const handle = await rawClient.create_pipe("file-transfer"); +const writer = await handle.wait(); +if (writer) { + await writer.write(new Uint8Array([0x01, 0x02])); + await writer.close(); +} + +// Accept incoming pipe +const reader = await rawClient.accept_pipe(pipeId); +const chunk = await reader.read(); + +// Deny incoming pipe +await rawClient.deny_pipe(pipeId); +``` + +Raw `PipeWriter` and `PipeReader` have the same interface as the SDK types: + +```typescript +interface PipeWriter { + write(data: Uint8Array): Promise; + close(): Promise; + abort(): void; + readonly pipeId: number; +} + +interface PipeReader { + read(): Promise; + readonly pipeId: number; + readonly description: string; +} +``` + A `WasmClient` manages one active WebTransport session. Create a new instance for independent connections, and call `free()` or `[Symbol.dispose]()` on raw WASM objects when you want to release memory eagerly. diff --git a/example/.gitignore b/example/.gitignore index 94b9ca3..c8a5b0c 100644 --- a/example/.gitignore +++ b/example/.gitignore @@ -11,5 +11,6 @@ web-client/public/host_public_key_bundle.hex web-client/public/mtp_dev_cert_hash.txt web-client/dist/ +client.id *.mk *.mpkb diff --git a/example/Cargo.lock b/example/Cargo.lock index a5b5094..700252a 100644 --- a/example/Cargo.lock +++ b/example/Cargo.lock @@ -208,6 +208,7 @@ name = "client" version = "0.1.0" dependencies = [ "mtp", + "rand 0.8.6", "tokio", ] @@ -902,6 +903,7 @@ dependencies = [ "mtp-crypto", "mtp-files", "mtp-host", + "mtp-transport", "mtp-type-map", ] @@ -915,6 +917,7 @@ dependencies = [ "mtp-transport", "rand 0.8.6", "tokio", + "tracing", ] [[package]] @@ -984,9 +987,11 @@ dependencies = [ "log", "mtp-codec", "mtp-common", + "rcgen", "rustls", "rustls-native-certs", "tokio", + "tracing", "wtransport", ] @@ -1574,6 +1579,7 @@ dependencies = [ "mtp", "rcgen", "serde_json", + "time", "tokio", ] diff --git a/example/client.id b/example/client.id new file mode 100644 index 0000000..e37d32a --- /dev/null +++ b/example/client.id @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/example/client/Cargo.toml b/example/client/Cargo.toml index d68f33e..b9c303f 100644 --- a/example/client/Cargo.toml +++ b/example/client/Cargo.toml @@ -8,5 +8,6 @@ name = "client" path = "src/main.rs" [dependencies] -mtp = { version = "0.1.0", path = "../../", features = ["client", "crypto", "files"] } +mtp = { version = "0.1.0", path = "../../", features = ["client", "crypto", "files", "pipes"] } tokio = { version = "1", features = ["full"] } +rand = "0.8" diff --git a/example/client/src/main.rs b/example/client/src/main.rs index 1a67cac..23a0174 100644 --- a/example/client/src/main.rs +++ b/example/client/src/main.rs @@ -1,5 +1,6 @@ mod auth; mod messages; +mod pipes; use std::fs; use std::path::Path; @@ -38,6 +39,10 @@ async fn main() -> Result<(), Box> { let (conn, keyring) = auth::connect_or_register(config, host_public_key, "client").await?; messages::send_and_receive(&conn, &keyring, &server_bundle).await?; + println!("\n--- Pipe demo ---"); + pipes::run_pipe_demo(&conn, 1).await?; + + conn.sender.close(); println!("\nDone"); Ok(()) } diff --git a/example/client/src/messages.rs b/example/client/src/messages.rs index 45f1dc0..08501fa 100644 --- a/example/client/src/messages.rs +++ b/example/client/src/messages.rs @@ -96,13 +96,12 @@ pub async fn send_and_receive( println!("Sending: {msg}"); conn.sender.send(&msg).await?; - match conn.receiver.receive().await { + match conn.receive().await { Ok(resp) => { println!("Received: {resp}"); } Err(e) => eprintln!("Receive error: {e}"), } - conn.sender.close(); Ok(()) } diff --git a/example/client/src/pipes.rs b/example/client/src/pipes.rs new file mode 100644 index 0000000..f09ee70 --- /dev/null +++ b/example/client/src/pipes.rs @@ -0,0 +1,137 @@ +use mtp::client::MTPConnection; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::oneshot; +use tokio::time::{Duration, Instant}; + +pub async fn run_pipe_demo( + conn: &MTPConnection, + iterations: usize, +) -> Result<(), Box> { + let sizes = [64, 256, 1024, 4096]; + let mut all_elapsed = Vec::with_capacity(sizes.len() * iterations); + let mut all_data_only = Vec::with_capacity(sizes.len() * iterations); + + for (i, &size) in sizes.iter().enumerate() { + let mut size_elapsed = Vec::with_capacity(iterations); + let mut size_data_only = Vec::with_capacity(iterations); + + for run in 0..iterations { + let random_bytes: Vec = (0..size).map(|_| rand::random::()).collect(); + let description = format!("pipe-demo-{i}-run{run}"); + println!(" [pipe {i}.{run}] creating pipe ({size} bytes): {description}"); + + let handle = conn.create_pipe(&description).await?; + let pipe_id = handle.pipe_id(); + println!(" [pipe {i}.{run}] create_pipe returned (pipe_id={pipe_id})"); + + // Overall timer starts before any I/O + let overall_start = Instant::now(); + + // Channel to capture the instant the writer actually starts writing + let (write_start_tx, write_start_rx) = oneshot::channel(); + + let write_bytes = random_bytes.clone(); + let writer_handle = tokio::spawn(async move { + println!(" [pipe {i}.{run}] writer: waiting for server accept ..."); + match handle.wait().await { + Ok(Some(mut writer)) => { + // Record the instant we begin writing + let _ = write_start_tx.send(Instant::now()); + + println!( + " [pipe {i}.{run}] writer: pipe accepted (pipe_id={pipe_id}), writing {} bytes ...", + write_bytes.len() + ); + writer + .write_all(&write_bytes) + .await + .map_err(|e| mtp::common::PipeError::IoError(e.to_string()))?; + writer + .finish() + .await + .map_err(|e| mtp::common::PipeError::IoError(e.to_string()))?; + println!(" [pipe {i}.{run}] writer: data sent and finished"); + Ok::<(), mtp::common::PipeError>(()) + } + Ok(None) => { + eprintln!(" [pipe {i}.{run}] writer: pipe denied by server"); + Err(mtp::common::PipeError::Rejected) + } + Err(e) => { + eprintln!(" [pipe {i}.{run}] writer: error: {e}"); + Err(e) + } + } + }); + + println!(" [pipe {i}.{run}] waiting for server's return pipe via receive_pipe() ..."); + let pipe_req = conn.receive_pipe().await?; + println!( + " [pipe {i}.{run}] received return pipe: id={} desc={:?}", + pipe_req.id(), + pipe_req.description() + ); + + let mut reader = pipe_req.accept().await?; + println!(" [pipe {i}.{run}] return pipe accepted, reading data ..."); + + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await?; + let overall_elapsed = overall_start.elapsed(); + + // Receive the instant the writer started writing + let data_start = write_start_rx.await?; + let data_only_elapsed = Instant::now() - data_start; + + match writer_handle.await { + Ok(Ok(())) => {} + Ok(Err(e)) => eprintln!(" [pipe {i}.{run}] writer error: {e}"), + Err(e) => eprintln!(" [pipe {i}.{run}] writer task panicked: {e}"), + } + + let matches = buf == random_bytes; + println!( + " [pipe {i}.{run}] round-trip: {} bytes, \ + total={:.3}ms, data-only={:.3}ms, match={matches}", + size, + overall_elapsed.as_secs_f64() * 1000.0, + data_only_elapsed.as_secs_f64() * 1000.0, + ); + + size_elapsed.push(overall_elapsed); + size_data_only.push(data_only_elapsed); + all_elapsed.push(overall_elapsed); + all_data_only.push(data_only_elapsed); + } + + // ---- per-size averages ---- + let avg_total = average_duration(&size_elapsed); + let avg_data = average_duration(&size_data_only); + println!( + " [pipe {i}] AVERAGE for size {size}: \ + total={avg_total:.3}ms, data-only={avg_data:.3}ms \ + (over {iterations} runs)" + ); + } + + // ---- overall averages ---- + let overall_total = average_duration(&all_elapsed); + let overall_data = average_duration(&all_data_only); + println!( + " [summary] OVERALL AVERAGE loopback time: \ + total={overall_total:.3}ms, data-only={overall_data:.3}ms \ + ({} measurements)", + all_elapsed.len() + ); + + Ok(()) +} + +/// Helper: average a slice of Durations without overflowing. +fn average_duration(durations: &[Duration]) -> f64 { + if durations.is_empty() { + return 0.0; + } + let sum_ms: f64 = durations.iter().map(|d| d.as_secs_f64() * 1000.0).sum(); + sum_ms / durations.len() as f64 +} diff --git a/example/server/Cargo.toml b/example/server/Cargo.toml index 494f36f..2f3a5b3 100644 --- a/example/server/Cargo.toml +++ b/example/server/Cargo.toml @@ -8,9 +8,10 @@ name = "server" path = "src/main.rs" [dependencies] -mtp = { version = "0.1.0", path = "../../", features = ["crypto", "host", "files"] } +mtp = { version = "0.1.0", path = "../../", features = ["crypto", "host", "files", "pipes"] } rcgen = "0.14" tokio = { version = "1", features = ["full"] } serde_json = { version = "1" } hex = "0.4" base64 = "0.22" +time = "0.3" diff --git a/example/server/src/main.rs b/example/server/src/main.rs index 86aafdc..4ccd627 100644 --- a/example/server/src/main.rs +++ b/example/server/src/main.rs @@ -4,7 +4,6 @@ mod keys; mod tls; use mtp::host::{AuthenticationPolicy, HostConfig, MTPHost}; - use mtp::type_map::TypeMap; use std::future::Future; use std::path::Path; @@ -28,6 +27,54 @@ fn dev_cert_paths() -> (String, String) { (cert, key) } +async fn handle_pipe_loopback( + conn: &mtp::host::MTPConnection, + req: mtp::host::PipeRequest, +) -> Result<(), Box> { + let pipe_id = req.id(); + println!( + " [loopback] Pipe request: id={pipe_id} description={:?}", + req.description() + ); + + println!(" [loopback] Calling accept() for pipe {pipe_id} ..."); + let mut reader = req.accept().await?; + println!(" [loopback] Pipe {pipe_id} accepted, reading data ..."); + + let mut buf = Vec::new(); + tokio::io::AsyncReadExt::read_to_end(&mut reader, &mut buf).await?; + println!( + " [loopback] Pipe {pipe_id} read {} bytes, creating return pipe ...", + buf.len() + ); + + let handle = conn.create_pipe("loopback").await?; + println!( + " [loopback] Return pipe created (id={}), waiting for client ...", + handle.pipe_id() + ); + + match handle.wait().await? { + Some(mut writer) => { + println!( + " [loopback] Client accepted return pipe, writing {} bytes ...", + buf.len() + ); + tokio::io::AsyncWriteExt::write_all(&mut writer, &buf).await?; + writer.finish().await?; + println!( + " [loopback] Pipe {pipe_id} loopback complete ({} bytes)", + buf.len() + ); + } + None => { + eprintln!(" [loopback] Return pipe denied by client for pipe {pipe_id}"); + } + } + + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), Box> { let (cert_path, key_path) = dev_cert_paths(); @@ -39,8 +86,6 @@ async fn main() -> Result<(), Box> { let (_host_id, host_keyring) = keys::load_or_generate_host_keys("host.mk")?; keys::export_host_public_keys(&host_keyring)?; - // The keyring is moved into the host config; keep a copy for decrypting the - // demo payloads clients encrypt to our KEM public key. let decrypt_keyring = mtp::crypto::Keyring::from_bytes(&host_keyring.to_bytes()) .expect("re-load host keyring for decryption"); @@ -116,20 +161,47 @@ async fn main() -> Result<(), Box> { let tm: &TypeMap = conn.codec.registry().get(&conn.version).unwrap(); - match conn.receiver.receive().await { - Ok(msg) => { - println!("Received: {msg}"); - let response = handlers::process_and_respond( - &msg, - tm, - conn.client_public_key.as_ref(), - &decrypt_keyring, - ); - println!("Sending: {response}"); - conn.sender.send(&response).await?; - } - Err(e) => { - eprintln!("Receive error: {e}"); + println!("Waiting for messages / pipe requests ..."); + loop { + tokio::select! { + biased; + + pipe_req = conn.receive_pipe() => { + match pipe_req { + Ok(req) => { + println!(" Pipe request: id={} desc={:?}", req.id(), req.description()); + if let Err(e) = handle_pipe_loopback(&conn, req).await { + eprintln!(" Pipe loopback error: {e}"); + } + } + Err(e) => { + println!("Pipe channel closed: {e}"); + break; + } + } + } + msg = conn.receive() => { + match msg { + Ok(msg) => { + println!("Received: {msg}"); + let response = handlers::process_and_respond( + &msg, + tm, + conn.client_public_key.as_ref(), + &decrypt_keyring, + ); + println!("Sending: {response}"); + if let Err(e) = conn.sender.send(&response).await { + eprintln!("Send error: {e}"); + break; + } + } + Err(e) => { + println!("Connection ended: {e}"); + break; + } + } + } } } diff --git a/example/server/src/tls.rs b/example/server/src/tls.rs index a119b5d..7dd57cb 100644 --- a/example/server/src/tls.rs +++ b/example/server/src/tls.rs @@ -1,7 +1,9 @@ -use std::fs; -use std::path::Path; - use base64::Engine; +use rcgen::{CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose, SanType}; +use std::fs; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::path::Path; +use time::{Duration, OffsetDateTime}; pub fn load_or_generate_tls( cert_path: &str, @@ -19,8 +21,26 @@ pub fn load_or_generate_tls( if let Some(parent) = Path::new(key_path).parent() { fs::create_dir_all(parent)?; } - let key_pair = rcgen::KeyPair::generate()?; - let params = rcgen::CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()])?; + + let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P256_SHA256)?; + + let mut params = CertificateParams::new(vec!["localhost".into()])?; + params.not_before = OffsetDateTime::now_utc() - Duration::minutes(5); + params.not_after = OffsetDateTime::now_utc() + Duration::days(13); + + params + .subject_alt_names + .push(SanType::IpAddress(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))); + params + .subject_alt_names + .push(SanType::IpAddress(IpAddr::V6(Ipv6Addr::new( + 0, 0, 0, 0, 0, 0, 0, 1, + )))); + + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + params.is_ca = IsCa::NoCa; + let cert = params.self_signed(&key_pair)?; let cert_str = cert.pem(); diff --git a/example/web-client/index.html b/example/web-client/index.html index 66e395b..941b59e 100644 --- a/example/web-client/index.html +++ b/example/web-client/index.html @@ -1,39 +1,89 @@ - + - - - - MTP Web Client - - - -

MTP WebTransport Client

- - + + + + MTP Web Client + + + +

MTP WebTransport Client

+ + - - + + - - + + -
- - - -
+
+ + + +
-
Initializing...
-
- - +
+

Pipe Demo

+
+ + +
+
+ +
Initializing...
+
+ + diff --git a/example/web-client/src/main.ts b/example/web-client/src/main.ts index 2e8da5c..93d4224 100644 --- a/example/web-client/src/main.ts +++ b/example/web-client/src/main.ts @@ -1,14 +1,28 @@ import { MTPClient } from "mtp"; -import type { MTPCredentialStorage, MTPLogEvent, ParsedFrame } from "mtp"; +import type { + MTPCredentialStorage, + MTPLogEvent, + MTPPipeReader, + ParsedFrame, +} from "mtp"; const STATUS = document.getElementById("status")!; const KEY_STATUS = document.getElementById("key-status")!; const SERVER_URL = document.getElementById("server-url") as HTMLInputElement; -const HOST_PUBLIC_KEY = document.getElementById("host-public-key") as HTMLTextAreaElement; -const CLIENT_CREDENTIALS = document.getElementById("client-credentials") as HTMLTextAreaElement; -const GENERATE_KEYPAIR = document.getElementById("generate-keypair") as HTMLButtonElement; +const HOST_PUBLIC_KEY = document.getElementById( + "host-public-key", +) as HTMLTextAreaElement; +const CLIENT_CREDENTIALS = document.getElementById( + "client-credentials", +) as HTMLTextAreaElement; +const GENERATE_KEYPAIR = document.getElementById( + "generate-keypair", +) as HTMLButtonElement; const CONNECT = document.getElementById("connect") as HTMLButtonElement; const CLEAR_KEYS = document.getElementById("clear-keys") as HTMLButtonElement; +const STREAM_MIC = document.getElementById("stream-mic") as HTMLButtonElement; +const STOP_MIC = document.getElementById("stop-mic") as HTMLButtonElement; +const PIPE_STATUS = document.getElementById("pipe-status")!; const CREDENTIALS_KEY = "mtp-web-client-credentials"; const HOST_PUBLIC_KEY_KEY = "mtp-web-client-host-public-key"; @@ -22,6 +36,13 @@ type SavedKeys = { let clientId: bigint | null = null; let devCertHash = ""; +let activeClient: ReturnType extends Promise + ? T + : never; +let micStream: MediaStream | null = null; +let mediaRecorder: MediaRecorder | null = null; +let pipeSendCount = 0; +let pendingPipeReaders: MTPPipeReader[] = []; const credentialStorage: MTPCredentialStorage = { getItem: (key) => localStorage.getItem(key), @@ -36,6 +57,13 @@ function log(msg: string, cls = "") { STATUS.appendChild(line); } +function pipeLog(msg: string, cls = "pipe") { + const line = document.createElement("div"); + line.textContent = msg; + if (cls) line.className = cls; + PIPE_STATUS.appendChild(line); +} + function renderStructured(value: unknown): string { return JSON.stringify(value, (_key, item) => { if (typeof item === "bigint") { @@ -70,13 +98,16 @@ function setKeyStatus(msg: string) { } function bytesToHex(bytes: Uint8Array): string { - return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); } function hexToBytes(value: string): Uint8Array { const hex = value.replace(/[^0-9a-fA-F]/g, ""); if (hex.length === 0) throw new Error("host public key is required"); - if (hex.length % 2 !== 0) throw new Error("host public key hex has an odd length"); + if (hex.length % 2 !== 0) + throw new Error("host public key hex has an odd length"); const bytes = new Uint8Array(hex.length / 2); for (let i = 0; i < bytes.length; i += 1) { @@ -87,7 +118,10 @@ function hexToBytes(value: string): Uint8Array { function saveHostPublicKey() { try { - localStorage.setItem(HOST_PUBLIC_KEY_KEY, bytesToHex(hexToBytes(HOST_PUBLIC_KEY.value))); + localStorage.setItem( + HOST_PUBLIC_KEY_KEY, + bytesToHex(hexToBytes(HOST_PUBLIC_KEY.value)), + ); } catch { localStorage.removeItem(HOST_PUBLIC_KEY_KEY); } @@ -102,7 +136,9 @@ function loadKeys() { if (!raw) { CLIENT_CREDENTIALS.value = ""; - setKeyStatus("No saved SDK credentials. The next connection will generate and store a reusable keyring."); + setKeyStatus( + "No saved SDK credentials. The next connection will generate and store a reusable keyring.", + ); return; } @@ -127,11 +163,13 @@ function loadKeys() { async function loadHostPublicKey() { try { - const response = await fetch("/host_public_key_bundle.hex", { cache: "no-store" }); + const response = await fetch("/host_public_key_bundle.hex", { + cache: "no-store", + }); if (!response.ok) return; const hostPublicKey = (await response.text()).trim(); - if (!hostPublicKey) return; + if (!hostPublicKey || !/^[0-9a-f]+$/i.test(hostPublicKey)) return; HOST_PUBLIC_KEY.value = hostPublicKey; saveHostPublicKey(); @@ -143,11 +181,14 @@ async function loadHostPublicKey() { async function loadDevCertHash() { try { - const response = await fetch("/mtp_dev_cert_hash.txt", { cache: "no-store" }); + const response = await fetch(`/mtp_dev_cert_hash.txt?t=${Date.now()}`, { + cache: "no-store", + }); if (!response.ok) return; - devCertHash = (await response.text()).trim(); - if (devCertHash) { + const hash = (await response.text()).trim(); + if (/^[0-9a-f]{64}$/i.test(hash)) { + devCertHash = hash; log(`Loaded WebTransport certificate hash: ${devCertHash}`); } } catch { @@ -157,14 +198,47 @@ async function loadDevCertHash() { async function initWasm() { log("Loading WASM module..."); - await MTPClient.create({ url: SERVER_URL.value, storage: credentialStorage, credentialsStorageKey: CREDENTIALS_KEY }); + await MTPClient.create({ + url: SERVER_URL.value, + storage: credentialStorage, + credentialsStorageKey: CREDENTIALS_KEY, + }); const supported = MTPClient.isSupported(); log(`WASM loaded. WebTransport supported: ${supported}`); CONNECT.disabled = !supported; } +async function createClient() { + const hostPk = hexToBytes(HOST_PUBLIC_KEY.value); + await loadDevCertHash(); + const serverUrl = SERVER_URL.value.trim(); + const serverCertificateHashes = devCertHash ? [devCertHash] : undefined; + + const client = await MTPClient.create({ + url: serverUrl, + hostPublicKey: hostPk, + storage: credentialStorage, + credentialsStorageKey: CREDENTIALS_KEY, + serverCertificateHashes, + pings: { intervalMs: 30_000 }, + logger(event) { + log( + renderLoggerEvent(event), + event.hint === "error" + ? "error" + : event.type === "state" + ? "state" + : "", + ); + }, + }); + + return client; +} + async function connect() { STATUS.textContent = ""; + PIPE_STATUS.textContent = ""; if (!MTPClient.isSupported()) { log("WebTransport is not supported in this browser.", "error"); @@ -175,25 +249,19 @@ async function connect() { await loadDevCertHash(); const serverUrl = SERVER_URL.value.trim(); - const serverCertificateHashes = devCertHash ? [`sha-256:${devCertHash}`] : undefined; + const serverCertificateHashes = devCertHash ? [devCertHash] : undefined; if (serverCertificateHashes) { log(`Pinning WebTransport certificate hash: ${serverCertificateHashes[0]}`); } else { - log("No WebTransport certificate hash loaded; relying on browser trust store.", "state"); + log( + "No WebTransport certificate hash loaded; relying on browser trust store.", + "state", + ); } try { - const client = await MTPClient.create({ - url: serverUrl, - hostPublicKey: hostPk, - storage: credentialStorage, - credentialsStorageKey: CREDENTIALS_KEY, - serverCertificateHashes, - pings: { intervalMs: 30_000 }, - logger(event) { - log(renderLoggerEvent(event), event.hint === "error" ? "error" : event.type === "state" ? "state" : ""); - }, - }); + const client = await createClient(); + activeClient = client; client.subscribe("Pong", (frame: ParsedFrame) => { log(`Subscribed Pong: ${formatParsedFrame(frame)}`, "received"); @@ -205,31 +273,164 @@ async function connect() { log(`Connected as client ${activeClientId}`); log("\nSending typed Ping..."); - await client.send("Ping", { - Description: "MTP web client send ping", - Timestamp: BigInt(Date.now()), - }, { sender: activeClientId }); + await client.send( + "Ping", + { + Description: "MTP web client send ping", + Timestamp: BigInt(Date.now()), + }, + { sender: activeClientId }, + ); log("Typed Ping sent."); log("\nRequesting Pong by Ping frame id..."); - const response = await client.request("Ping", { - Description: "MTP web client request ping", - Timestamp: BigInt(Date.now()), - }, { sender: activeClientId, responseType: "Pong" }); + const response = await client.request( + "Ping", + { + Description: "MTP web client request ping", + Timestamp: BigInt(Date.now()), + }, + { sender: activeClientId, responseType: "Pong" }, + ); log(`Request response: ${formatParsedFrame(response)}`, "received"); log("\nClient running. Waiting for incoming messages..."); + STREAM_MIC.disabled = false; + log("\nPipe demo ready. Click 'Stream Microphone' to start.", "pipe"); } catch (error) { log(`[error] ${error}`, "error"); } } +async function startMicStreaming() { + if (!activeClient) { + pipeLog("No active client connection.", "error"); + return; + } + + try { + micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch (e) { + pipeLog(`Microphone access denied: ${e}`, "error"); + return; + } + + STREAM_MIC.disabled = true; + STOP_MIC.disabled = false; + pipeLog("Microphone acquired. Creating pipe ..."); + + const handle = await activeClient.createPipe("mic-audio"); + pipeLog( + `Pipe created (id=${handle.pipeId}). Waiting for server to accept ...`, + ); + + // Handle incoming pipe requests from the server (loopback return pipes) + activeClient.setOnPipeRequest(async (request) => { + pipeLog( + `Incoming return pipe: id=${request.pipeId} desc=${request.description}`, + ); + try { + const reader = await activeClient!.acceptPipe(request.pipeId); + pendingPipeReaders.push(reader); + readLoopbackPipe(reader); + } catch (e) { + pipeLog(`Failed to accept return pipe: ${e}`, "error"); + } + }); + + const writer = await handle.wait(); + if (!writer) { + pipeLog("Pipe denied by server.", "error"); + stopMicStreaming(); + return; + } + + pipeLog(`Pipe accepted. Streaming microphone (pipe id=${writer.pipeId}) ...`); + + // Stream microphone audio via MediaRecorder + const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") + ? "audio/webm;codecs=opus" + : "audio/webm"; + mediaRecorder = new MediaRecorder(micStream, { mimeType }); + + mediaRecorder.ondataavailable = async (event) => { + if (event.data.size === 0 || !activeClient) return; + + const startTime = performance.now(); + pipeSendCount++; + const chunkNum = pipeSendCount; + + try { + const buffer = await event.data.arrayBuffer(); + const data = new Uint8Array(buffer); + await writer.write(data); + pipeLog( + ` [chunk ${chunkNum}] sent ${data.length} bytes (${(performance.now() - startTime).toFixed(1)}ms write)`, + ); + } catch (e) { + pipeLog(` [chunk ${chunkNum}] send error: ${e}`, "error"); + } + }; + + mediaRecorder.start(200); // emit data every 200ms + pipeLog("Streaming started (200ms chunks)."); +} + +async function readLoopbackPipe(reader: MTPPipeReader) { + const startTime = performance.now(); + let totalBytes = 0; + let chunkCount = 0; + + try { + while (true) { + const data = await reader.read(); + if (data == null) break; // EOF + totalBytes += data.length; + chunkCount++; + } + } catch (e) { + pipeLog(` Return pipe read error: ${e}`, "error"); + return; + } + + const elapsed = performance.now() - startTime; + pipeLog( + ` Return pipe complete: ${chunkCount} chunks, ${totalBytes} bytes, ` + + `delay=${elapsed.toFixed(1)}ms`, + ); + + // Clean up the reader from the pending list + const idx = pendingPipeReaders.indexOf(reader); + if (idx >= 0) pendingPipeReaders.splice(idx, 1); +} + +async function stopMicStreaming() { + if (mediaRecorder && mediaRecorder.state !== "inactive") { + mediaRecorder.stop(); + mediaRecorder = null; + } + + if (micStream) { + micStream.getTracks().forEach((track) => track.stop()); + micStream = null; + } + + // Close pending pipe readers + pendingPipeReaders = []; + + STREAM_MIC.disabled = false; + STOP_MIC.disabled = true; + pipeLog("Microphone streaming stopped."); +} + GENERATE_KEYPAIR.addEventListener("click", () => { try { clientId = null; localStorage.removeItem(CREDENTIALS_KEY); CLIENT_CREDENTIALS.value = ""; - setKeyStatus("Cleared saved credentials. The next connection will generate a new reusable keyring."); + setKeyStatus( + "Cleared saved credentials. The next connection will generate a new reusable keyring.", + ); log("Cleared saved SDK credentials."); } catch (e) { log(`Credential reset failed: ${e}`, "error"); @@ -257,6 +458,17 @@ CLEAR_KEYS.addEventListener("click", () => { log("Cleared saved SDK credentials and host public key."); }); +STREAM_MIC.addEventListener("click", () => { + startMicStreaming().catch((e) => { + pipeLog(`Pipe streaming error: ${e}`, "error"); + console.error(e); + }); +}); + +STOP_MIC.addEventListener("click", () => { + stopMicStreaming(); +}); + HOST_PUBLIC_KEY.addEventListener("change", saveHostPublicKey); initWasm() diff --git a/example/web-client/vite.config.ts b/example/web-client/vite.config.ts index 6be6c39..340deb2 100644 --- a/example/web-client/vite.config.ts +++ b/example/web-client/vite.config.ts @@ -1,16 +1,41 @@ -import { defineConfig } from 'vite'; +import { defineConfig, type Plugin } from 'vite'; import fs from 'fs'; import path from 'path'; import { mtp } from 'mtp/vite'; -const devCertDir = path.resolve(__dirname, '../dev-cert'); +const exampleDir = path.resolve(__dirname, '..'); +const devCertDir = path.join(exampleDir, 'dev-cert'); const certPath = process.env.MTP_DEV_CERT ?? path.join(devCertDir, 'cert.pem'); const keyPath = process.env.MTP_DEV_KEY ?? path.join(devCertDir, 'key.pem'); const hasDevCert = fs.existsSync(certPath) && fs.existsSync(keyPath); +const devFiles: Record = { + '/host_public_key_bundle.hex': path.join(exampleDir, 'host_public_key_bundle.hex'), + '/mtp_dev_cert_hash.txt': path.join(devCertDir, 'sha256.txt'), +}; + +function devFileServe(): Plugin { + return { + name: 'dev-file-serve', + configureServer(server) { + server.middlewares.use((req, res, next) => { + const target = devFiles[req.url?.split('?')[0] ?? '']; + if (!target) return next(); + + fs.readFile(target, (err, data) => { + if (err) return next(); + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Cache-Control', 'no-store'); + res.end(data); + }); + }); + }, + }; +} + export default defineConfig({ - plugins: [mtp({ typeMaps: '../../example/type-maps.yaml' })], + plugins: [mtp({ typeMaps: '../../example/type-maps.yaml' }), devFileServe()], server: { https: hasDevCert ? { diff --git a/host/Cargo.toml b/host/Cargo.toml index 71cb38c..32ba530 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -9,10 +9,10 @@ mtp-codec = { version = "0.1.0", path = "../codec", features = ["registry"] } mtp-transport = { version = "0.1.0", path = "../transport", features = ["host"] } mtp-crypto = { version = "0.1.0", path = "../crypto", optional = true } rand = "0.8" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["time", "sync"] } [features] + crypto = ["dep:mtp-crypto", "mtp-codec/crypto"] -# Stream mode is optional at the facade boundary. The underlying transport -# remains framed/stream based so the default API stays backwards compatible. -streaming = ["mtp-transport/streaming"] + +pipes = ["mtp-common/pipes", "mtp-transport/pipes"] diff --git a/host/src/lib.rs b/host/src/lib.rs index 1477d33..b251cc6 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -3,13 +3,22 @@ use mtp_codec::{ registry::{Registry, VersionedCodec}, }; use mtp_common::CommunicationError; +#[cfg(feature = "pipes")] +pub use mtp_common::PipeError; use std::net::IpAddr; #[cfg(feature = "crypto")] use std::pin::Pin; use std::{error::Error, fmt}; +#[cfg(feature = "pipes")] +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{Mutex, mpsc}; #[cfg(feature = "crypto")] use tokio::time::Duration; +#[cfg(feature = "pipes")] +use mtp_transport::PipeReader; + pub use MTPConnection as Connection; pub use MTPHost as Host; pub use mtp_transport::Policy; @@ -17,6 +26,9 @@ pub use mtp_transport::Receiver; pub use mtp_transport::SendMode; pub use mtp_transport::Sender; +#[cfg(feature = "pipes")] +pub use mtp_transport::PipeWriter; + /* ---- async callback type aliases ---- */ #[cfg(feature = "crypto")] pub type GetExistingClient = Box< @@ -47,6 +59,162 @@ pub enum AuthenticationPolicy { Unauthenticated, } +#[cfg(feature = "pipes")] +pub struct PipeHandle { + pipe_id: u32, + description: String, + sender: Sender, + response_rx: tokio::sync::oneshot::Receiver>, +} + +#[cfg(feature = "pipes")] +impl PipeHandle { + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } + + pub fn description(&self) -> &str { + &self.description + } + + pub async fn wait(self) -> Result, PipeError> { + match self.response_rx.await { + Ok(Ok(true)) => { + let writer = self + .sender + .open_pipe(self.pipe_id, &self.description) + .await + .map_err(PipeError::from)?; + Ok(Some(writer)) + } + Ok(Ok(false)) => Ok(None), + Ok(Err(e)) => Err(e), + Err(_) => Err(PipeError::StreamClosed), + } + } +} + +#[cfg(feature = "pipes")] +pub struct PipeRequest { + pipe_id: u32, + description: String, + sender: Sender, + dispatcher: Arc, +} + +#[cfg(feature = "pipes")] +impl PipeRequest { + pub fn id(&self) -> u32 { + self.pipe_id + } + + pub fn description(&self) -> &str { + &self.description + } + + pub async fn accept(self) -> Result { + let (pipe_tx, pipe_rx) = tokio::sync::oneshot::channel(); + { + let mut pending = self.dispatcher.pending_pipes.lock().await; + pending.insert(self.pipe_id, pipe_tx); + } + + let resp = CommunicationValue::new(mtp_codec::CommunicationType::PipeResponse) + .with_id(self.pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolTrue); + self.sender.send(&resp).await.map_err(PipeError::from)?; + + let timeout = self.dispatcher.policy.read_timeout; + tokio::time::timeout(timeout, pipe_rx) + .await + .map_err(|_| PipeError::HandshakeTimeout)? + .map_err(|_| PipeError::StreamClosed) + } + + pub async fn deny(self) -> Result<(), PipeError> { + let resp = CommunicationValue::new(mtp_codec::CommunicationType::PipeResponse) + .with_id(self.pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolFalse); + self.sender.send(&resp).await.map_err(PipeError::from)?; + Ok(()) + } +} + +#[cfg(feature = "pipes")] +struct PipeDispatcher { + pending_creations: Mutex>>>, + pending_pipes: Mutex>>, + policy: Arc, +} + +#[cfg(not(feature = "pipes"))] +struct PipeDispatcher; + +#[cfg(not(feature = "pipes"))] +pub(crate) struct PipeRequest; + +#[cfg(feature = "pipes")] +async fn run_dispatcher( + receiver: Receiver, + sender: Sender, + app_tx: mpsc::Sender>, + pipe_req_tx: mpsc::Sender, + dispatcher: Arc, +) { + let pipe_req_type = + mtp_codec::CommunicationType::PipeRequest.to_id(&mtp_codec::TypeMap::latest()); + let pipe_resp_type = + mtp_codec::CommunicationType::PipeResponse.to_id(&mtp_codec::TypeMap::latest()); + + loop { + match receiver.receive_event().await { + Ok(mtp_transport::TransportEvent::Message(msg)) => { + if msg.get_type() == pipe_req_type { + let pipe_id = msg.get_id(); + let description = msg + .get_str(DataType::Description) + .unwrap_or("") + .to_string(); + let req = PipeRequest { + pipe_id, + description, + sender: sender.clone(), + dispatcher: dispatcher.clone(), + }; + let _ = pipe_req_tx.send(req).await; + continue; + } + + if msg.get_type() == pipe_resp_type { + let pipe_id = msg.get_id(); + let accepted = msg.get_bool(DataType::Accepted).unwrap_or(false); + let mut pending = dispatcher.pending_creations.lock().await; + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(Ok(accepted)); + } + continue; + } + + if app_tx.send(Ok(msg)).await.is_err() { + break; + } + } + Ok(mtp_transport::TransportEvent::Pipe(reader)) => { + let pipe_id = reader.pipe_id(); + let mut pending = dispatcher.pending_pipes.lock().await; + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(reader); + } + } + Err(e) => { + if app_tx.send(Err(e)).await.is_err() { + break; + } + } + } + } +} + /* Host configuration. */ pub struct HostConfig { pub ip: IpAddr, @@ -180,7 +348,11 @@ pub struct MTPConnection { pub codec: VersionedCodec, pub sender: Sender, pub receiver: Receiver, + app_rx: Mutex>>, + pipe_req_rx: Mutex>, + pipe_dispatcher: Arc, pub description: Option, + _dispatcher_task: tokio::task::JoinHandle<()>, #[cfg(feature = "crypto")] pub auth_state: AuthState, #[cfg(feature = "crypto")] @@ -242,11 +414,10 @@ impl MTPHost { Ok(result) => result, Err(_) => Err(AcceptError::AuthenticationTimedOut), }; - return Ok(self.configure_pongs(connection?)); + return connection; } AuthenticationPolicy::AllowAuthentication => { - let connection = self.accept_allow_auth(sender, receiver).await?; - return Ok(self.configure_pongs(connection)); + return self.accept_allow_auth(sender, receiver).await; } AuthenticationPolicy::Unauthenticated => { let first_msg = match receiver.receive().await { @@ -278,19 +449,16 @@ impl MTPHost { DataValue::Str(s) => Some(s.clone()), _ => None, }; - Ok(self.configure_pongs(Some(MTPConnection { - version: negotiated, - codec, + Ok(Some(self.connection_from_parts( sender, receiver, + negotiated, + codec, description, - #[cfg(feature = "crypto")] - auth_state: AuthState::Unauthenticated, - #[cfg(feature = "crypto")] - client_id: rand::random(), - #[cfg(feature = "crypto")] - client_public_key: None, - }))) + AuthState::Unauthenticated, + rand::random(), + None, + ))) } } @@ -318,13 +486,13 @@ impl MTPHost { DataValue::Str(s) => Some(s.clone()), _ => None, }; - return Ok(self.configure_pongs(Some(MTPConnection { - version: negotiated, - codec, + return Ok(Some(self.connection_from_parts( sender, receiver, + negotiated, + codec, description, - }))); + ))); } } @@ -336,16 +504,204 @@ impl MTPHost { &self.registry } - fn configure_pongs(&self, connection: Option) -> Option { - if let Some(connection) = connection { + #[cfg(not(feature = "crypto"))] + fn connection_from_parts( + &self, + sender: Sender, + receiver: Receiver, + version: Version, + codec: VersionedCodec, + description: Option, + ) -> MTPConnection { + #[cfg(feature = "pipes")] + { if self.config.send_pongs { - connection - .receiver - .respond_to_pings(connection.sender.clone()); + receiver.respond_to_pings(sender.clone()); } - Some(connection) - } else { - None + + let (app_tx, app_rx) = + mpsc::channel(self.config.policy.receiver_queue_capacity); + let (pipe_req_tx, pipe_req_rx) = + mpsc::channel(self.config.policy.receiver_queue_capacity); + + let dispatcher = Arc::new(PipeDispatcher { + pending_creations: Mutex::new(HashMap::new()), + pending_pipes: Mutex::new(HashMap::new()), + policy: Arc::new(self.config.policy), + }); + + let dispatcher_clone = dispatcher.clone(); + let receiver_clone = receiver.clone(); + let sender_clone = sender.clone(); + let task = tokio::spawn(run_dispatcher( + receiver_clone, + sender_clone, + app_tx, + pipe_req_tx, + dispatcher_clone, + )); + + MTPConnection { + version, + codec, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description, + _dispatcher_task: task, + } + } + + #[cfg(not(feature = "pipes"))] + { + if self.config.send_pongs { + receiver.respond_to_pings(sender.clone()); + } + + let (_, app_rx) = mpsc::channel::>(1); + let (_, pipe_req_rx) = mpsc::channel::(1); + let dispatcher = Arc::new(PipeDispatcher); + let task = tokio::spawn(async {}); + + MTPConnection { + version, + codec, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description, + _dispatcher_task: task, + } + } + } + + #[cfg(feature = "crypto")] + fn connection_from_parts( + &self, + sender: Sender, + receiver: Receiver, + version: Version, + codec: VersionedCodec, + description: Option, + auth_state: AuthState, + client_id: u64, + client_public_key: Option, + ) -> MTPConnection { + #[cfg(feature = "pipes")] + { + if self.config.send_pongs { + receiver.respond_to_pings(sender.clone()); + } + + let (app_tx, app_rx) = + mpsc::channel(self.config.policy.receiver_queue_capacity); + let (pipe_req_tx, pipe_req_rx) = + mpsc::channel(self.config.policy.receiver_queue_capacity); + + let dispatcher = Arc::new(PipeDispatcher { + pending_creations: Mutex::new(HashMap::new()), + pending_pipes: Mutex::new(HashMap::new()), + policy: Arc::new(self.config.policy), + }); + + let dispatcher_clone = dispatcher.clone(); + let receiver_clone = receiver.clone(); + let sender_clone = sender.clone(); + let task = tokio::spawn(run_dispatcher( + receiver_clone, + sender_clone, + app_tx, + pipe_req_tx, + dispatcher_clone, + )); + + MTPConnection { + version, + codec, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description, + _dispatcher_task: task, + auth_state, + client_id, + client_public_key, + } + } + + #[cfg(not(feature = "pipes"))] + { + if self.config.send_pongs { + receiver.respond_to_pings(sender.clone()); + } + + let (_, app_rx) = mpsc::channel::>(1); + let (_, pipe_req_rx) = mpsc::channel::(1); + let dispatcher = Arc::new(PipeDispatcher); + let task = tokio::spawn(async {}); + + MTPConnection { + version, + codec, + sender, + receiver, + app_rx: Mutex::new(app_rx), + pipe_req_rx: Mutex::new(pipe_req_rx), + pipe_dispatcher: dispatcher, + description, + _dispatcher_task: task, + auth_state, + client_id, + client_public_key, + } + } + } +} + +#[cfg(feature = "pipes")] +impl MTPConnection { + pub async fn receive(&self) -> Result { + let mut rx = self.app_rx.lock().await; + match rx.recv().await { + Some(result) => result, + None => Err(CommunicationError::StreamClosed), + } + } + + pub async fn create_pipe(&self, description: &str) -> Result { + let pipe_id = rand::random::(); + let (tx, rx) = tokio::sync::oneshot::channel(); + + { + let mut pending = self.pipe_dispatcher.pending_creations.lock().await; + pending.insert(pipe_id, tx); + } + + let request = CommunicationValue::new(mtp_codec::CommunicationType::PipeRequest) + .with_id(pipe_id) + .add_typed_default(DataType::Description, DataValue::Str(description.into())); + + self.sender.send(&request).await.map_err(PipeError::from)?; + + Ok(PipeHandle { + pipe_id, + description: description.to_string(), + sender: self.sender.clone(), + response_rx: rx, + }) + } + + pub async fn receive_pipe(&self) -> Result { + let mut rx = self.pipe_req_rx.lock().await; + match rx.recv().await { + Some(req) => Ok(req), + None => Err(CommunicationError::StreamClosed), } } } @@ -668,16 +1024,16 @@ impl MTPHost { let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) .expect("negotiated version must be registered"); - Ok(Some(MTPConnection { - version: negotiated, - codec, + Ok(Some(self.connection_from_parts( sender, receiver, + negotiated, + codec, description, - auth_state: AuthState::Authenticated, - client_id: assigned_id, - client_public_key: Some(client_bundle), - })) + AuthState::Authenticated, + assigned_id, + Some(client_bundle), + ))) } /* @@ -782,16 +1138,16 @@ impl MTPHost { }; let codec = VersionedCodec::for_version(self.registry.clone(), negotiated.clone()) .expect("negotiated version must be registered"); - return Ok(Some(MTPConnection { - version: negotiated, - codec, + return Ok(Some(self.connection_from_parts( sender, receiver, + negotiated, + codec, description, - auth_state: AuthState::Unauthenticated, - client_id: rand::random(), - client_public_key: None, - })); + AuthState::Unauthenticated, + rand::random(), + None, + ))); } sender.close(); diff --git a/src/sdk/index.ts b/src/sdk/index.ts index e890959..bb3cd2f 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -2,6 +2,7 @@ import initWasm, { ConnectionConfig, ConnectionState, WasmClient, + WasmPipeHandle, keyring_generate, } from "mtp/raw"; import * as bindings from "mtp/raw"; @@ -287,6 +288,30 @@ export interface MTPRequestOptions extends MTPSendOptions { responseType?: MTPCommunicationType; } +export interface MTPPipeWriter { + write(data: Uint8Array): Promise; + close(): Promise; + abort(): void; + readonly pipeId: number; +} + +export interface MTPPipeReader { + read(): Promise; + readonly pipeId: number; + readonly description: string; +} + +export interface MTPPipeRequest { + pipeId: number; + description: string; +} + +export interface MTPOutgoingPipeHandle { + readonly pipeId: number; + readonly description: string; + wait(): Promise; +} + type InternalCredentials = Omit< MTPCredentials, "clientId" | "keyring" | "hostPublicKey" @@ -1546,6 +1571,80 @@ export class MTPClient { return this.encryptedDeviceSecretProvider.getEncryptedDeviceSecret(query); } + setOnPipeRequest( + handler: ((request: MTPPipeRequest) => void) | null, + ): void { + if (handler == null) { + this.raw.client.set_on_pipe_request(null); + return; + } + this.raw.client.set_on_pipe_request( + (event: { pipeId: number; description: string }) => { + emit(this.#options.logger, { + hint: "info", + type: "PipeRequest", + data: event, + direction: "recv", + }); + handler({ pipeId: event.pipeId, description: event.description }); + }, + ); + } + + async createPipe(description: string): Promise { + if (typeof description !== "string") { + throw new TypeError("description must be a string"); + } + const handle: WasmPipeHandle = await this.raw.client.create_pipe( + description, + ); + const sdk = this; + return { + pipeId: handle.pipeId, + description: handle.description, + async wait(): Promise { + const result = await handle.wait(); + if (result == null) { + return null; + } + emit(sdk.#options.logger, { + hint: "info", + type: "PipeCreated", + data: { pipeId: result.pipeId }, + direction: "send", + }); + return result as unknown as MTPPipeWriter; + }, + }; + } + + async acceptPipe(pipeId: number): Promise { + if (typeof pipeId !== "number" || !Number.isFinite(pipeId)) { + throw new TypeError("pipeId must be a finite number"); + } + const reader = await this.raw.client.accept_pipe(pipeId); + emit(this.#options.logger, { + hint: "info", + type: "PipeAccepted", + data: { pipeId: reader.pipeId, description: reader.description }, + direction: "send", + }); + return reader as unknown as MTPPipeReader; + } + + async denyPipe(pipeId: number): Promise { + if (typeof pipeId !== "number" || !Number.isFinite(pipeId)) { + throw new TypeError("pipeId must be a finite number"); + } + await this.raw.client.deny_pipe(pipeId); + emit(this.#options.logger, { + hint: "info", + type: "PipeDenied", + data: { pipeId }, + direction: "send", + }); + } + disconnect(): void { this.raw.client.stop_protocol_pings(); this.raw.client.disconnect(); diff --git a/transport/Cargo.toml b/transport/Cargo.toml index 6a72928..fec48c9 100644 --- a/transport/Cargo.toml +++ b/transport/Cargo.toml @@ -25,12 +25,7 @@ name = "integration" required-features = ["host"] [features] -default = [] # Enables hosting a MTP server host = [] -# Documents and exposes the framed uni-directional stream transport used by -# the host and client facades. The transport is stream based by design, so -# keeping this as a marker feature lets facade crates opt into their -# stream-specific convenience exports without making the core transport -# unusable for existing consumers. -streaming = [] + +pipes = ["mtp-codec/pipes"] diff --git a/transport/src/connection.rs b/transport/src/connection.rs index a726a74..387b13e 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -1,4 +1,6 @@ use crate::ConnectionHandle; +#[cfg(feature = "pipes")] +use crate::pipe::PipeReader; use mtp_codec::CommunicationValue; use mtp_common::CommunicationError; use std::sync::Arc; @@ -7,6 +9,13 @@ use tokio::time::{Duration, sleep, timeout}; use wtransport::Connection; use tracing::{debug, info, instrument, trace}; +#[cfg(feature = "pipes")] +#[derive(Debug)] +pub enum TransportEvent { + Message(CommunicationValue), + Pipe(PipeReader), +} + const APPLICATION_CLOSE_REASON: &str = "mtp-close"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -418,6 +427,43 @@ impl Sender { &self.handle } + #[cfg(feature = "pipes")] + #[instrument(skip(self, description), level = "trace")] + pub async fn open_pipe( + &self, + pipe_id: u32, + description: &str, + ) -> Result { + if self.handle.is_closed() { + return Err(self + .handle + .close_reason() + .unwrap_or(CommunicationError::UseAfterClosed)); + } + + if self.connection.quic_connection().close_reason().is_some() { + let reason = self + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed); + self.handle.close(Some(reason.clone())); + return Err(reason); + } + + let mut stream = Self::open_uni_stream(&self.connection, &self.policy).await?; + + let request = CommunicationValue::new(mtp_codec::CommunicationType::PipeRequest) + .with_id(pipe_id) + .add_typed_default( + mtp_codec::DataType::Description, + mtp_codec::DataValue::Str(description.to_string()), + ); + + Self::write_frame(&mut stream, &request, &self.policy).await?; + + Ok(crate::pipe::PipeWriter { stream }) + } + #[instrument(skip(self), level = "trace")] pub fn close(&self) { info!(target = "mtp.transport", "fire-and-forget close requested"); @@ -518,12 +564,25 @@ impl Sender { } } -/// Single-consumer framed message receiver. +/// Framed message receiver. /// -/// `receive()` is intended to be driven by one task at a time. Internally the -/// underlying `mpsc::Receiver` is protected by a mutex so the type remains -/// `Sync`, but it is not a multi-consumer queue. +/// When the `pipes` feature is disabled, `receive()` is intended to be driven +/// by one task at a time. When `pipes` is enabled, an internal dispatcher task +/// consumes events from the channel; applications should use the +/// `MTPConnection::receive()` and `MTPConnection::receive_pipe()` methods +/// instead of calling `receiver.receive()` directly. +/// +/// The type is cheaply cloneable: all clones share the same internal channel. pub struct Receiver { + inner: Arc, +} + +struct ReceiverInner { + #[cfg(feature = "pipes")] + msg_rx: Mutex>>, + #[cfg(feature = "pipes")] + pipe_rx: Mutex>, + #[cfg(not(feature = "pipes"))] rx: Mutex>>, _accept_task: tokio::task::JoinHandle<()>, handle: Arc, @@ -531,26 +590,39 @@ pub struct Receiver { queue_notify: Arc, } +impl Clone for Receiver { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + if Arc::strong_count(&self.inner) == 1 { + self.inner._accept_task.abort(); + } + } +} + #[derive(Clone, Default)] struct PingControl { pong_sender: Option, pong_observer: Option>, } -impl Drop for Receiver { - fn drop(&mut self) { - // The accept loop holds clones of the connection and the shared - // ConnectionHandle. Without this, dropping a Receiver without first - // closing the connection would leave that task running forever. Abort - // it directly rather than closing the shared handle, so a still-live - // Sender on the same connection is unaffected. abort() is a no-op if - // the task already finished (e.g. the connection was closed). - self._accept_task.abort(); - } -} - impl Receiver { pub fn new(connection: Connection, handle: Arc, policy: Arc) -> Self { + #[cfg(feature = "pipes")] + let (msg_tx, msg_rx) = mpsc::channel::>( + policy.receiver_queue_capacity, + ); + #[cfg(feature = "pipes")] + let (pipe_tx, pipe_rx) = mpsc::channel::( + policy.receiver_queue_capacity, + ); + #[cfg(not(feature = "pipes"))] let (tx, rx) = mpsc::channel::>( policy.receiver_queue_capacity, ); @@ -581,7 +653,12 @@ impl Receiver { let mut close_rx = conn_handle.subscribe_close(); loop { - if tx.capacity() == 0 { + #[cfg(feature = "pipes")] + let cap_full = msg_tx.capacity() == 0 || pipe_tx.capacity() == 0; + #[cfg(not(feature = "pipes"))] + let cap_full = tx.capacity() == 0; + + if cap_full { trace!(target = "mtp.transport", "accept loop paused: receiver queue full"); tokio::select! { _ = close_rx.changed() => { @@ -611,6 +688,11 @@ impl Receiver { Ok(permit) => permit, Err(_) => break, }; + #[cfg(feature = "pipes")] + let msg_tx_stream = msg_tx.clone(); + #[cfg(feature = "pipes")] + let pipe_tx_stream = pipe_tx.clone(); + #[cfg(not(feature = "pipes"))] let tx_stream = tx.clone(); let stream_handle = conn_handle.clone(); let stream_policy = accept_policy.clone(); @@ -625,6 +707,9 @@ impl Receiver { && frame_count >= max_frames { let close_error = CommunicationError::StreamError; + #[cfg(feature = "pipes")] + let _ = msg_tx_stream.send(Err(close_error.clone())).await; + #[cfg(not(feature = "pipes"))] let _ = tx_stream.send(Err(close_error.clone())).await; stream_handle.close(Some(close_error)); break; @@ -633,6 +718,40 @@ impl Receiver { match Self::read_one_frame(&mut s, &stream_policy).await { Ok(ReceivedFrame::Message(msg)) => { frame_count += 1; + + #[cfg(feature = "pipes")] + { + let pipe_request_type = + mtp_codec::CommunicationType::PipeRequest + .to_id(&mtp_codec::TypeMap::latest()); + if msg.get_type() == pipe_request_type + && frame_count == 1 + { + let pipe_id = msg.get_id(); + let description = msg + .get_str(mtp_codec::DataType::Description) + .unwrap_or("") + .to_string(); + + let pipe_reader = crate::pipe::PipeReader { + stream: s, + description, + pipe_id, + }; + + if pipe_tx_stream + .send(pipe_reader) + .await + .is_err() + { + stream_handle.close(Some( + CommunicationError::StreamClosed, + )); + } + break; + } + } + let ping_type = mtp_codec::CommunicationType::Ping .to_id(&mtp_codec::TypeMap::latest()); let pong_type = mtp_codec::CommunicationType::Pong @@ -674,12 +793,24 @@ impl Receiver { continue; } + #[cfg(feature = "pipes")] + if msg_tx_stream + .send(Ok(msg)) + .await + .is_err() + { + break; + } + #[cfg(not(feature = "pipes"))] if tx_stream.send(Ok(msg)).await.is_err() { break; } } Ok(ReceivedFrame::ClosedByPeer) => { let close_error = CommunicationError::StreamClosed; + #[cfg(feature = "pipes")] + let _ = msg_tx_stream.send(Err(close_error.clone())).await; + #[cfg(not(feature = "pipes"))] let _ = tx_stream.send(Err(close_error.clone())).await; stream_handle.close(Some(close_error)); break; @@ -697,6 +828,9 @@ impl Receiver { other => other, }; + #[cfg(feature = "pipes")] + let _ = msg_tx_stream.send(Err(close_error.clone())).await; + #[cfg(not(feature = "pipes"))] let _ = tx_stream.send(Err(close_error.clone())).await; stream_handle.close(Some(close_error)); break; @@ -709,6 +843,9 @@ impl Receiver { Ok(Err(_e)) => { // A connection error from accept_uni means the connection is permanently closed. let close_error = CommunicationError::StreamClosed; + #[cfg(feature = "pipes")] + let _ = msg_tx.send(Err(close_error.clone())).await; + #[cfg(not(feature = "pipes"))] let _ = tx.send(Err(close_error.clone())).await; conn_handle.close(Some(close_error)); break; @@ -717,6 +854,9 @@ impl Receiver { Err(_) => { if accept_connection.quic_connection().close_reason().is_some() { let close_error = CommunicationError::StreamClosed; + #[cfg(feature = "pipes")] + let _ = msg_tx.send(Err(close_error.clone())).await; + #[cfg(not(feature = "pipes"))] let _ = tx.send(Err(close_error.clone())).await; conn_handle.close(Some(close_error)); break; @@ -733,17 +873,24 @@ impl Receiver { }); Self { - rx: Mutex::new(rx), - _accept_task: accept_task, - handle, - ping_control, - queue_notify, + inner: Arc::new(ReceiverInner { + #[cfg(feature = "pipes")] + msg_rx: Mutex::new(msg_rx), + #[cfg(feature = "pipes")] + pipe_rx: Mutex::new(pipe_rx), + #[cfg(not(feature = "pipes"))] + rx: Mutex::new(rx), + _accept_task: accept_task, + handle, + ping_control, + queue_notify, + }), } } /* Respond to reserved Ping frames without exposing them to application I/O. */ pub fn respond_to_pings(&self, sender: Sender) { - if let Ok(mut control) = self.ping_control.try_write() { + if let Ok(mut control) = self.inner.ping_control.try_write() { control.pong_sender = Some(sender); } else { log::warn!("[Receiver] could not register Ping responder: control lock busy"); @@ -752,7 +899,7 @@ impl Receiver { /* Route reserved Pong frames to a connection-level observer. */ pub fn observe_pongs(&self, observer: mpsc::UnboundedSender) { - if let Ok(mut control) = self.ping_control.try_write() { + if let Ok(mut control) = self.inner.ping_control.try_write() { control.pong_observer = Some(observer); } else { log::warn!("[Receiver] could not register Pong observer: control lock busy"); @@ -837,44 +984,163 @@ impl Receiver { #[instrument(skip(self), level = "trace")] pub async fn receive(&self) -> Result { - if self.handle.is_closed() { + if self.inner.handle.is_closed() { return Err(self + .inner .handle .close_reason() .unwrap_or(CommunicationError::StreamClosed)); } - let mut rx = self.rx.lock().await; - match rx.recv().await { - Some(result) => { - self.queue_notify.notify_one(); - result + #[cfg(feature = "pipes")] + { + let mut rx = self.inner.msg_rx.lock().await; + match rx.recv().await { + Some(Ok(msg)) => { + self.inner.queue_notify.notify_one(); + Ok(msg) + } + Some(Err(e)) => Err(e), + None => Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)), } - _ => Err(self + } + #[cfg(not(feature = "pipes"))] + { + let mut rx = self.inner.rx.lock().await; + match rx.recv().await { + Some(result) => { + self.inner.queue_notify.notify_one(); + result + } + None => Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)), + } + } + } + + #[cfg(feature = "pipes")] + #[instrument(skip(self), level = "trace")] + pub async fn receive_event(&self) -> Result { + if self.inner.handle.is_closed() { + return Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)); + } + + let mut msg_rx = self.inner.msg_rx.lock().await; + let mut pipe_rx = self.inner.pipe_rx.lock().await; + tokio::select! { + msg = msg_rx.recv() => { + match msg { + Some(Ok(val)) => { + self.inner.queue_notify.notify_one(); + Ok(TransportEvent::Message(val)) + } + Some(Err(e)) => Err(e), + None => Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)), + } + } + pipe = pipe_rx.recv() => { + match pipe { + Some(reader) => { + self.inner.queue_notify.notify_one(); + Ok(TransportEvent::Pipe(reader)) + } + None => Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)), + } + } + } + } + + #[cfg(feature = "pipes")] + #[instrument(skip(self), level = "trace")] + pub async fn receive_pipe(&self) -> Result { + if self.inner.handle.is_closed() { + return Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)); + } + + let mut rx = self.inner.pipe_rx.lock().await; + match rx.recv().await { + Some(reader) => { + self.inner.queue_notify.notify_one(); + Ok(reader) + } + None => Err(self + .inner .handle .close_reason() .unwrap_or(CommunicationError::StreamClosed)), } } + #[cfg(feature = "pipes")] + pub fn try_receive_pipe(&self) -> Result, CommunicationError> { + if self.inner.handle.is_closed() { + return Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)); + } + + match self.inner.pipe_rx.try_lock() { + Ok(mut rx) => match rx.try_recv() { + Ok(reader) => { + self.inner.queue_notify.notify_one(); + Ok(Some(reader)) + } + Err(mpsc::error::TryRecvError::Empty) => Ok(None), + Err(mpsc::error::TryRecvError::Disconnected) => { + return Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)); + } + }, + Err(_) => Ok(None), + } + } + pub fn handle(&self) -> &Arc { - &self.handle + &self.inner.handle } pub fn close(&self) { - self.handle.close(None); + self.inner.handle.close(None); } pub fn is_open(&self) -> bool { - self.handle.is_open() + self.inner.handle.is_open() } pub fn is_closed(&self) -> bool { - self.handle.is_closed() + self.inner.handle.is_closed() } pub fn close_reason(&self) -> Option { - self.handle.close_reason() + self.inner.handle.close_reason() } } diff --git a/transport/src/lib.rs b/transport/src/lib.rs index 707e74e..9c9bff3 100644 --- a/transport/src/lib.rs +++ b/transport/src/lib.rs @@ -2,8 +2,16 @@ pub mod client; pub mod connection; pub mod connection_handle; +#[cfg(feature = "pipes")] +pub mod pipe; + pub use connection::{Policy, Receiver, SendMode, Sender}; +#[cfg(feature = "pipes")] +pub use connection::TransportEvent; +#[cfg(feature = "pipes")] +pub use pipe::{PipeReader, PipeWriter}; + pub use client::connect; pub use connection_handle::ConnectionHandle; diff --git a/transport/src/pipe.rs b/transport/src/pipe.rs new file mode 100644 index 0000000..33157e7 --- /dev/null +++ b/transport/src/pipe.rs @@ -0,0 +1,69 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +#[derive(Debug)] +pub struct PipeWriter { + pub(crate) stream: wtransport::SendStream, +} + +impl PipeWriter { + pub async fn finish(mut self) -> Result<(), mtp_common::CommunicationError> { + self.stream + .finish() + .await + .map_err(|e| { + log::warn!("[PipeWriter] finish failed: {e}"); + mtp_common::CommunicationError::StreamWriteError(e) + }) + } + + pub fn abort(&mut self) -> Result<(), wtransport::error::ClosedStream> { + self.stream.reset(wtransport::VarInt::from_u32(0)) + } +} + +impl AsyncWrite for PipeWriter { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stream).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + +#[derive(Debug)] +pub struct PipeReader { + pub(crate) stream: wtransport::RecvStream, + pub(crate) description: String, + pub(crate) pipe_id: u32, +} + +impl PipeReader { + pub fn description(&self) -> &str { + &self.description + } + + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } +} + +impl AsyncRead for PipeReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_read(cx, buf) + } +} diff --git a/type-map/Cargo.toml b/type-map/Cargo.toml index 3dac68f..02051a8 100644 --- a/type-map/Cargo.toml +++ b/type-map/Cargo.toml @@ -5,11 +5,12 @@ edition = "2024" build = "build.rs" [features] -default = [] # Enables multi-version type-map constructors, builtin_type_maps(), and # the Registry struct for version negotiation. Used by host, not client. registry = [] +pipes = [] + [dependencies] [build-dependencies] diff --git a/type-map/build.rs b/type-map/build.rs index 41fd144..201ad1e 100755 --- a/type-map/build.rs +++ b/type-map/build.rs @@ -116,6 +116,21 @@ const RESERVED_COMM_TYPES: &[ReservedEntry] = &[ name: "GatewayTimeout", id: 22, }, + #[cfg(feature = "pipes")] + ReservedEntry { + name: "PipeRequest", + id: 23, + }, + #[cfg(feature = "pipes")] + ReservedEntry { + name: "PipeResponse", + id: 24, + }, + #[cfg(feature = "pipes")] + ReservedEntry { + name: "PipeAbort", + id: 25, + }, ]; const RESERVED_DATA_TYPES: &[ReservedEntry] = &[ @@ -168,6 +183,10 @@ const RESERVED_DATA_TYPES: &[ReservedEntry] = &[ name: "ErrorMessage", id: 12, }, + ReservedEntry { + name: "Accepted", + id: 13, + }, ]; fn main() { diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index b3aa3e2..081d308 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -23,9 +23,13 @@ getrandom-v02 = { package = "getrandom", version = "0.2", features = ["js"] } mtp-common = { version = "0.1.0", path = "../common" } mtp-type-map = { version = "0.1.0", path = "../type-map" } -mtp-codec = { version = "0.1.0", path = "../codec", features = ["crypto"] } +mtp-codec = { version = "0.1.0", path = "../codec", features = ["crypto", "pipes"] } mtp-crypto = { version = "0.1.0", path = "../crypto", features = ["wasm"] } [dev-dependencies] wasm-bindgen-test = "0.3" hex = "0.4" + +[features] +default = [] +pipes = [] diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 503d1e9..6da9156 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -13,6 +13,7 @@ use mtp_crypto::SignatureScheme; use crate::config::ConnectionConfig; use crate::error::js_error; +use crate::pipe::PipeReader; use crate::transport::WasmTransport; struct PendingRequest { @@ -25,6 +26,57 @@ struct PingTimer { closure: Closure, } +#[wasm_bindgen(typescript_custom_section)] +const PIPE_HANDLE_TS: &str = r#" +export interface WasmPipeHandle { + wait(): Promise; + readonly pipeId: number; + readonly description: string; +} +"#; + +#[wasm_bindgen] +pub struct WasmPipeHandle { + pipe_id: u32, + description: String, + transport: WasmTransport, + response_rx: Rc>>>>, +} + +#[wasm_bindgen] +impl WasmPipeHandle { + pub async fn wait(&self) -> Result { + let rx = self + .response_rx + .borrow_mut() + .take() + .ok_or_else(|| js_error("handle already consumed"))?; + + let accepted = rx + .await + .map_err(|_| js_error("pipe handle channel closed"))?; + + match accepted { + Ok(true) => { + let writer = self.transport.open_pipe(self.pipe_id, &self.description).await?; + Ok(JsValue::from(writer)) + } + Ok(false) => Ok(JsValue::NULL), + Err(e) => Err(e), + } + } + + #[wasm_bindgen(getter)] + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } + + #[wasm_bindgen(getter)] + pub fn description(&self) -> String { + self.description.clone() + } +} + fn frame_property(frame: &JsValue, key: &str) -> Option { js_sys::Reflect::get(frame, &JsValue::from_str(key)) .ok() @@ -110,6 +162,22 @@ fn reject_pending_requests( } } +fn reject_pending_pipe_creations( + pending: &Rc>>>>, + message: &str, +) { + let pending = std::mem::take(&mut *pending.borrow_mut()); + for (_, tx) in pending { + let _ = tx.send(Err(js_error(message))); + } +} + +fn random_pipe_id() -> Result { + let mut bytes = [0u8; 4]; + getrandom::fill(&mut bytes).map_err(|_| js_error("rng failed"))?; + Ok(u32::from_be_bytes(bytes)) +} + fn raw_frame_preview(bytes: &[u8]) -> String { let shown = bytes.len().min(256); let mut preview = hex::encode(&bytes[..shown]); @@ -261,6 +329,9 @@ pub struct WasmClient { next_subscription_id: Rc>, pending_requests: Rc>>, ping_timer: Rc>>, + pending_pipe_creations: Rc>>>>, + pending_pipes: Rc>>>, + on_pipe_request: Rc>>, } #[wasm_bindgen] @@ -282,6 +353,9 @@ impl WasmClient { next_subscription_id: Rc::new(Cell::new(1)), pending_requests: Rc::new(RefCell::new(HashMap::new())), ping_timer: Rc::new(RefCell::new(None)), + pending_pipe_creations: Rc::new(RefCell::new(HashMap::new())), + pending_pipes: Rc::new(RefCell::new(HashMap::new())), + on_pipe_request: Rc::new(RefCell::new(None)), } } @@ -686,9 +760,96 @@ impl WasmClient { } self.subscriptions.borrow_mut().clear(); self.reject_pending_requests("disconnected"); + reject_pending_pipe_creations(&self.pending_pipe_creations, "disconnected"); self.set_state(ConnectionState::Disconnected); } + /// Set the callback invoked when a remote peer opens a pipe request. + /// The callback receives a plain JS object `{ pipeId: number, description: string }`. + #[wasm_bindgen] + pub fn set_on_pipe_request(&self, callback: Option) { + *self.on_pipe_request.borrow_mut() = callback; + } + + /// Initiate an outgoing pipe. Returns a `WasmPipeHandle` whose `wait()` + /// method resolves after the remote peer accepts (or denies) the request. + #[wasm_bindgen] + pub async fn create_pipe(&self, description: &str) -> Result { + let transport = self + .transport + .borrow() + .clone() + .ok_or_else(|| js_error("not connected"))?; + + let pipe_id = random_pipe_id()?; + let (tx, rx) = oneshot::channel(); + self.pending_pipe_creations + .borrow_mut() + .insert(pipe_id, tx); + + let request = + CommunicationValue::new(CommunicationType::PipeRequest).with_id(pipe_id).add_typed_default( + DataType::Description, + DataValue::Str(description.to_string()), + ); + let request_bytes = request + .to_bytes() + .map_err(|e| js_error(&format!("encode failed: {}", e)))?; + transport.send_frame(&request_bytes).await?; + + Ok(WasmPipeHandle { + pipe_id, + description: description.to_string(), + transport, + response_rx: Rc::new(RefCell::new(Some(rx))), + }) + } + + /// Accept an incoming pipe request (identified by `pipe_id`). Sends a + /// `PipeResponse` with `Accepted = true` and returns a `PipeReader` once + /// the remote peer opens the pipe stream. + #[wasm_bindgen] + pub async fn accept_pipe(&self, pipe_id: u32) -> Result { + let transport = self + .transport + .borrow() + .clone() + .ok_or_else(|| js_error("not connected"))?; + + let resp = CommunicationValue::new(CommunicationType::PipeResponse) + .with_id(pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolTrue); + let resp_bytes = resp + .to_bytes() + .map_err(|e| js_error(&format!("encode failed: {}", e)))?; + transport.send_frame(&resp_bytes).await?; + + let (tx, rx) = oneshot::channel(); + self.pending_pipes.borrow_mut().insert(pipe_id, tx); + + rx.await + .map_err(|_| js_error("pipe closed before stream arrived")) + } + + /// Deny an incoming pipe request. Sends a `PipeResponse` with + /// `Accepted = false`. + #[wasm_bindgen] + pub async fn deny_pipe(&self, pipe_id: u32) -> Result<(), JsValue> { + let transport = self + .transport + .borrow() + .clone() + .ok_or_else(|| js_error("not connected"))?; + + let resp = CommunicationValue::new(CommunicationType::PipeResponse) + .with_id(pipe_id) + .add_typed_default(DataType::Accepted, DataValue::BoolFalse); + let resp_bytes = resp + .to_bytes() + .map_err(|e| js_error(&format!("encode failed: {}", e)))?; + transport.send_frame(&resp_bytes).await + } + fn set_state(&self, new_state: ConnectionState) { self.state.set(new_state); @@ -738,10 +899,68 @@ impl WasmClient { let pending_requests = self.pending_requests.clone(); let loop_pending_requests = pending_requests.clone(); let ping_timer = self.ping_timer.clone(); + let pending_pipe_creations = self.pending_pipe_creations.clone(); + let pending_pipes = self.pending_pipes.clone(); + let on_pipe_request = self.on_pipe_request.clone(); + let loop_pipe_creations = pending_pipe_creations.clone(); wasm_bindgen_futures::spawn_local(async move { loop_transport - .receive_loop( + .receive_loop_with_pipes( move |frame: JsValue| { + let message_type = frame_type(&frame); + if let Some(ref msg_type) = message_type { + if msg_type == "PipeRequest" { + let pipe_id = frame_id(&frame).unwrap_or(0); + let description = frame_property(&frame, "data") + .and_then(|data| { + let desc = js_sys::Reflect::get( + &data, + &JsValue::from_str("Description"), + ) + .ok()?; + desc.as_string() + }) + .unwrap_or_default(); + + let cb = on_pipe_request.borrow(); + if let Some(ref callback) = *cb { + let obj = js_sys::Object::new(); + let _ = js_sys::Reflect::set( + &obj, + &"pipeId".into(), + &JsValue::from_f64(pipe_id as f64), + ); + let _ = js_sys::Reflect::set( + &obj, + &"description".into(), + &JsValue::from_str(&description), + ); + let _ = callback.call1(&JsValue::NULL, &obj.into()); + } + return; + } + + if msg_type == "PipeResponse" { + let pipe_id = frame_id(&frame).unwrap_or(0); + let accepted = frame_property(&frame, "data") + .and_then(|data| { + let acc = js_sys::Reflect::get( + &data, + &JsValue::from_str("Accepted"), + ) + .ok()?; + acc.as_bool() + }) + .unwrap_or(false); + + let mut pending = loop_pipe_creations.borrow_mut(); + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(Ok(accepted)); + } + return; + } + } + route_incoming_frame( &frame, &on_msg, @@ -750,11 +969,19 @@ impl WasmClient { ); }, on_err.clone(), + move |pipe_reader: PipeReader| { + let pipe_id = pipe_reader.pipe_id(); + let mut pending = pending_pipes.borrow_mut(); + if let Some(tx) = pending.remove(&pipe_id) { + let _ = tx.send(pipe_reader); + } + }, ) .await; state.set(ConnectionState::Disconnected); stop_ping_timer(&ping_timer); reject_pending_requests(&pending_requests, "disconnected"); + reject_pending_pipe_creations(&pending_pipe_creations, "disconnected"); }); } diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 4c9f783..980c28e 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -4,6 +4,7 @@ pub mod crypto; pub mod error; pub mod frame; pub mod logging; +pub mod pipe; pub mod subscription; pub mod transport; diff --git a/wasm/src/pipe.rs b/wasm/src/pipe.rs new file mode 100644 index 0000000..4cf2bfc --- /dev/null +++ b/wasm/src/pipe.rs @@ -0,0 +1,140 @@ +use wasm_bindgen::prelude::*; +use wasm_bindgen::JsCast; +use wasm_bindgen_futures::JsFuture; + +use crate::error::js_error; +use crate::transport::release_writer_lock; + +#[wasm_bindgen(typescript_custom_section)] +const PIPE_TS: &str = r#" +export interface PipeWriter { + write(data: Uint8Array): Promise; + close(): Promise; + abort(): void; + readonly pipeId: number; +} + +export interface PipeReader { + read(): Promise; + readonly pipeId: number; + readonly description: string; +} +"#; + +#[wasm_bindgen] +pub struct PipeWriter { + writer: JsValue, + pipe_id: u32, +} + +impl PipeWriter { + pub fn new(writer: JsValue, pipe_id: u32) -> Self { + Self { writer, pipe_id } + } +} + +#[wasm_bindgen] +impl PipeWriter { + pub async fn write(&mut self, data: &[u8]) -> Result<(), JsValue> { + let chunk = js_sys::Uint8Array::from(data); + let write_fn = js_sys::Reflect::get(&self.writer, &JsValue::from_str("write")) + .map_err(|_| js_error("missing write"))? + .dyn_into::() + .map_err(|_| js_error("write not a function"))?; + let write_promise = write_fn + .call1(&self.writer, &chunk) + .map_err(|e| js_error(&format!("write failed: {:?}", e)))?; + JsFuture::from(write_promise.unchecked_into::()).await?; + Ok(()) + } + + pub async fn close(self) -> Result<(), JsValue> { + let close_fn = js_sys::Reflect::get(&self.writer, &JsValue::from_str("close")) + .map_err(|_| js_error("missing close"))? + .dyn_into::() + .map_err(|_| js_error("close not a function"))?; + let close_promise = close_fn + .call0(&self.writer) + .map_err(|e| js_error(&format!("close failed: {:?}", e)))?; + if let Err(e) = + JsFuture::from(close_promise.unchecked_into::()).await + { + crate::transport::log_stream_error_code(&e, "pipe writer close"); + } + release_writer_lock(&self.writer); + Ok(()) + } + + pub fn abort(&mut self) -> Result<(), JsValue> { + let abort_fn = js_sys::Reflect::get(&self.writer, &JsValue::from_str("abort")) + .map_err(|_| js_error("missing abort"))? + .dyn_into::() + .map_err(|_| js_error("abort not a function"))?; + let _ = abort_fn.call0(&self.writer); + release_writer_lock(&self.writer); + Ok(()) + } + + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } +} + +#[wasm_bindgen] +pub struct PipeReader { + reader: JsValue, + description: String, + pipe_id: u32, + pending: Vec, +} + +impl PipeReader { + pub fn new(reader: JsValue, pipe_id: u32, description: String, pending: Vec) -> Self { + Self { + reader, + pipe_id, + description, + pending, + } + } +} + +#[wasm_bindgen] +impl PipeReader { + pub async fn read(&mut self) -> Result { + if !self.pending.is_empty() { + let data = std::mem::take(&mut self.pending); + return Ok(js_sys::Uint8Array::from(&data[..]).into()); + } + + let read_fn = js_sys::Reflect::get(&self.reader, &JsValue::from_str("read")) + .map_err(|_| js_error("missing read"))? + .dyn_into::() + .map_err(|_| js_error("read not a function"))?; + let promise = read_fn + .call0(&self.reader) + .map_err(|_| js_error("read call failed"))? + .unchecked_into::(); + let result = JsFuture::from(promise).await?; + + let done = js_sys::Reflect::get(&result, &JsValue::from_str("done")) + .ok() + .and_then(|v| v.as_bool()) + .unwrap_or(true); + if done { + return Ok(JsValue::NULL); + } + + let value = js_sys::Reflect::get(&result, &JsValue::from_str("value")) + .map_err(|_| js_error("missing value"))?; + Ok(js_sys::Uint8Array::new(&value).into()) + } + + pub fn pipe_id(&self) -> u32 { + self.pipe_id + } + + pub fn description(&self) -> String { + self.description.clone() + } +} diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index 4ccb14c..ab30cc5 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -1,4 +1,4 @@ -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::rc::Rc; use wasm_bindgen::JsCast; @@ -11,7 +11,7 @@ use crate::frame::parse_frame_value; const CLOSE_FRAME_LEN: u32 = u32::MAX; /// Logs the `streamErrorCode` from a stream-level WebTransportError (STOP_SENDING / RESET_STREAM). Session errors are skipped. -fn log_stream_error_code(error: &JsValue, context: &str) { +pub(crate) fn log_stream_error_code(error: &JsValue, context: &str) { let source = js_sys::Reflect::get(error, &JsValue::from_str("source")) .ok() .and_then(|v| v.as_string()); @@ -70,7 +70,7 @@ fn resolve_stream_readable(recv_stream: &JsValue) -> Result { } /// Releases a writer's lock so an abandoned writer isn't treated as an abort (which sends STOP_SENDING). -fn release_writer_lock(writer: &JsValue) { +pub(crate) fn release_writer_lock(writer: &JsValue) { if let Ok(release) = js_sys::Reflect::get(writer, &JsValue::from_str("releaseLock")) .and_then(|f| f.dyn_into::().map_err(Into::into)) { @@ -79,7 +79,7 @@ fn release_writer_lock(writer: &JsValue) { } /// Releases a reader's lock so an abandoned reader isn't treated as a cancel (which sends STOP_SENDING). -fn release_reader_lock(reader: &JsValue) { +pub(crate) fn release_reader_lock(reader: &JsValue) { if let Ok(release) = js_sys::Reflect::get(reader, &JsValue::from_str("releaseLock")) .and_then(|f| f.dyn_into::().map_err(Into::into)) { @@ -119,6 +119,8 @@ pub struct WasmTransport { stream_reader: Rc>>, /// Bytes already read from the current stream but not yet consumed as a frame. buffer: Rc>>, + /// Set to `true` when `open_next_stream` succeeds; cleared after the first frame is parsed. + new_stream_frame: Rc>, } impl WasmTransport { @@ -177,6 +179,7 @@ impl WasmTransport { streams_reader: Rc::new(RefCell::new(None)), stream_reader: Rc::new(RefCell::new(None)), buffer: Rc::new(RefCell::new(Vec::new())), + new_stream_frame: Rc::new(Cell::new(false)), }) } @@ -309,6 +312,7 @@ impl WasmTransport { .map_err(|_| js_error("stream getReader call failed"))?; *self.stream_reader.borrow_mut() = Some(reader); + self.new_stream_frame.set(true); Ok(true) } @@ -451,6 +455,136 @@ impl WasmTransport { } } + /// Pipe-aware receive loop. Identical to `receive_loop` but detects + /// `PipeRequest` as the first frame on a new incoming stream and routes + /// the stream to `on_pipe` instead of `on_message`. + pub async fn receive_loop_with_pipes( + &self, + mut on_message: F, + on_error: js_sys::Function, + mut on_pipe: G, + ) where + F: FnMut(JsValue), + G: FnMut(crate::pipe::PipeReader), + { + let pipe_request_type = mtp_codec::CommunicationType::PipeRequest + .to_id(&mtp_codec::TypeMap::latest()); + + loop { + match self.next_frame().await { + Ok(FrameOutcome::Frame(frame)) => { + let is_first = self.new_stream_frame.get(); + if is_first { + self.new_stream_frame.set(false); + if let Ok(comm) = mtp_codec::CommunicationValue::from_bytes(&frame) + && comm.get_type() == pipe_request_type + { + let pipe_id = comm.get_id(); + let description = comm + .get_str(mtp_codec::DataType::Description) + .unwrap_or("") + .to_string(); + + let pending = { + let mut buf = self.buffer.borrow_mut(); + std::mem::take(&mut *buf) + }; + + if let Some(reader) = self.stream_reader.borrow_mut().take() { + let pipe_reader = crate::pipe::PipeReader::new( + reader, + pipe_id, + description, + pending, + ); + on_pipe(pipe_reader); + } + continue; + } + } + + match parse_frame_value(&frame) { + Ok(parsed) => { + on_message(parsed); + } + Err(e) => { + let message = e.as_string().unwrap_or_else(|| format!("{:?}", e)); + let _ = + on_error.call1(&JsValue::NULL, &JsValue::from_str(&message)); + } + } + } + Ok(FrameOutcome::Closed) | Ok(FrameOutcome::Ended) => break, + Err(e) => { + let _ = on_error.call1(&JsValue::NULL, &e); + break; + } + } + } + } + + /// Open a new outgoing unidirectional stream and write a `PipeRequest` + /// frame as the first frame. Returns a `PipeWriter` whose underlying + /// `WritableStream` remains open for subsequent raw-data writes. + pub async fn open_pipe( + &self, + pipe_id: u32, + description: &str, + ) -> Result { + let create_stream = js_sys::Reflect::get( + &self.inner, + &JsValue::from_str("createUnidirectionalStream"), + )? + .dyn_into::() + .map_err(|_| js_error("createUnidirectionalStream not a function"))?; + let stream_promise = create_stream + .call0(&self.inner)? + .dyn_into::() + .map_err(|_| js_error("createUnidirectionalStream did not return a Promise"))?; + let stream = JsFuture::from(stream_promise).await?; + + let writable_or_stream = resolve_stream_writable(&stream)?; + let writer_val = js_sys::Reflect::get(&writable_or_stream, &JsValue::from_str("getWriter")) + .map_err(|_| js_error("missing getWriter"))? + .dyn_into::() + .map_err(|_| js_error("getWriter not a function"))? + .call0(&writable_or_stream) + .map_err(|_| js_error("getWriter call failed"))?; + + let request = mtp_codec::CommunicationValue::new(mtp_codec::CommunicationType::PipeRequest) + .with_id(pipe_id) + .add_typed_default( + mtp_codec::DataType::Description, + mtp_codec::DataValue::Str(description.to_string()), + ); + let frame_bytes = request + .to_bytes() + .map_err(|e| js_error(&format!("encode failed: {}", e)))?; + + let len = frame_bytes.len() as u32; + let mut wire = Vec::with_capacity(4 + frame_bytes.len()); + wire.extend_from_slice(&len.to_be_bytes()); + wire.extend_from_slice(&frame_bytes); + + let chunk = js_sys::Uint8Array::from(&wire[..]); + let write_fn = js_sys::Reflect::get(&writer_val, &JsValue::from_str("write")) + .map_err(|_| js_error("missing write"))? + .dyn_into::() + .map_err(|_| js_error("write not a function"))?; + let write_promise = write_fn + .call1(&writer_val, &chunk) + .map_err(|e| js_error(&format!("write failed: {:?}", e)))?; + if let Err(e) = + JsFuture::from(write_promise.unchecked_into::()).await + { + log_stream_error_code(&e, "open_pipe write"); + release_writer_lock(&writer_val); + return Err(e); + } + + Ok(crate::pipe::PipeWriter::new(writer_val, pipe_id)) + } + pub fn close(&self) { // Release reader locks before closing so they aren't treated as cancels. if let Some(reader) = self.stream_reader.borrow_mut().take() { From 203ef1adccb174cb405658128261db99e9ece5b9 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:53:48 +0200 Subject: [PATCH 36/97] format --- client/src/lib.rs | 12 ++++-------- client/tests/ping.rs | 9 +++++++-- common/src/lib.rs | 7 +++---- host/src/lib.rs | 17 ++++++----------- transport/src/connection.rs | 18 ++++++++---------- transport/src/pipe.rs | 11 ++++------- transport/tests/integration.rs | 26 +++++++++----------------- wasm/src/client.rs | 14 ++++++++------ wasm/src/pipe.rs | 6 ++---- wasm/src/transport.rs | 11 ++++------- 10 files changed, 55 insertions(+), 76 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index b9ea6ac..73fd4df 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -2,9 +2,9 @@ use mtp_codec::{CommunicationValue, DataType, DataValue, PROTOCOL_VERSION, Versi use mtp_common::CommunicationError; #[cfg(feature = "pipes")] pub use mtp_common::PipeError; +use rand::Rng; use std::collections::HashMap; use std::sync::Arc; -use rand::Rng; use tokio::sync::{Mutex, mpsc}; use tokio::time::{Duration, Instant}; @@ -147,10 +147,7 @@ async fn run_dispatcher( Ok(mtp_transport::TransportEvent::Message(msg)) => { if msg.get_type() == pipe_req_type { let pipe_id = msg.get_id(); - let description = msg - .get_str(DataType::Description) - .unwrap_or("") - .to_string(); + let description = msg.get_str(DataType::Description).unwrap_or("").to_string(); let req = PipeRequest { pipe_id, description, @@ -510,9 +507,8 @@ fn connection_from_parts( let (app_tx, app_rx) = mpsc::channel::>( config.policy.receiver_queue_capacity, ); - let (pipe_req_tx, pipe_req_rx) = mpsc::channel::( - config.policy.receiver_queue_capacity, - ); + let (pipe_req_tx, pipe_req_rx) = + mpsc::channel::(config.policy.receiver_queue_capacity); let dispatcher = Arc::new(PipeDispatcher { pending_creations: Mutex::new(HashMap::new()), diff --git a/client/tests/ping.rs b/client/tests/ping.rs index b071864..e497db9 100644 --- a/client/tests/ping.rs +++ b/client/tests/ping.rs @@ -16,8 +16,13 @@ fn generate_self_signed_cert() -> (Vec, Vec) { async fn start_host(send_pongs: bool) -> (MTPHost, Vec) { let (cert_pem, key_pem) = generate_self_signed_cert(); let host = MTPHost::new( - HostConfig::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0, cert_pem.clone(), key_pem) - .with_pongs(send_pongs), + HostConfig::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 0, + cert_pem.clone(), + key_pem, + ) + .with_pongs(send_pongs), ) .await .unwrap(); diff --git a/common/src/lib.rs b/common/src/lib.rs index 773bfca..75739f1 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -369,10 +369,9 @@ mod pipe_error_tests { #[test] fn test_pipe_error_from_connection_error() { - let pe: PipeError = CommunicationError::ConnectionError( - wtransport::error::ConnectionError::TimedOut, - ) - .into(); + let pe: PipeError = + CommunicationError::ConnectionError(wtransport::error::ConnectionError::TimedOut) + .into(); assert_eq!(pe, PipeError::ConnectionClosed); } diff --git a/host/src/lib.rs b/host/src/lib.rs index b251cc6..3f51b3d 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -5,13 +5,13 @@ use mtp_codec::{ use mtp_common::CommunicationError; #[cfg(feature = "pipes")] pub use mtp_common::PipeError; +#[cfg(feature = "pipes")] +use std::collections::HashMap; use std::net::IpAddr; #[cfg(feature = "crypto")] use std::pin::Pin; -use std::{error::Error, fmt}; -#[cfg(feature = "pipes")] -use std::collections::HashMap; use std::sync::Arc; +use std::{error::Error, fmt}; use tokio::sync::{Mutex, mpsc}; #[cfg(feature = "crypto")] use tokio::time::Duration; @@ -171,10 +171,7 @@ async fn run_dispatcher( Ok(mtp_transport::TransportEvent::Message(msg)) => { if msg.get_type() == pipe_req_type { let pipe_id = msg.get_id(); - let description = msg - .get_str(DataType::Description) - .unwrap_or("") - .to_string(); + let description = msg.get_str(DataType::Description).unwrap_or("").to_string(); let req = PipeRequest { pipe_id, description, @@ -519,8 +516,7 @@ impl MTPHost { receiver.respond_to_pings(sender.clone()); } - let (app_tx, app_rx) = - mpsc::channel(self.config.policy.receiver_queue_capacity); + let (app_tx, app_rx) = mpsc::channel(self.config.policy.receiver_queue_capacity); let (pipe_req_tx, pipe_req_rx) = mpsc::channel(self.config.policy.receiver_queue_capacity); @@ -597,8 +593,7 @@ impl MTPHost { receiver.respond_to_pings(sender.clone()); } - let (app_tx, app_rx) = - mpsc::channel(self.config.policy.receiver_queue_capacity); + let (app_tx, app_rx) = mpsc::channel(self.config.policy.receiver_queue_capacity); let (pipe_req_tx, pipe_req_rx) = mpsc::channel(self.config.policy.receiver_queue_capacity); diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 387b13e..00ddad5 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -4,10 +4,10 @@ use crate::pipe::PipeReader; use mtp_codec::CommunicationValue; use mtp_common::CommunicationError; use std::sync::Arc; -use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; +use tokio::sync::{Mutex, Notify, RwLock, Semaphore, mpsc}; use tokio::time::{Duration, sleep, timeout}; -use wtransport::Connection; use tracing::{debug, info, instrument, trace}; +use wtransport::Connection; #[cfg(feature = "pipes")] #[derive(Debug)] @@ -119,10 +119,7 @@ impl Policy { self } - pub fn with_max_concurrent_stream_tasks( - mut self, - max_concurrent_stream_tasks: usize, - ) -> Self { + pub fn with_max_concurrent_stream_tasks(mut self, max_concurrent_stream_tasks: usize) -> Self { self.max_concurrent_stream_tasks = max_concurrent_stream_tasks; self } @@ -619,9 +616,7 @@ impl Receiver { policy.receiver_queue_capacity, ); #[cfg(feature = "pipes")] - let (pipe_tx, pipe_rx) = mpsc::channel::( - policy.receiver_queue_capacity, - ); + let (pipe_tx, pipe_rx) = mpsc::channel::(policy.receiver_queue_capacity); #[cfg(not(feature = "pipes"))] let (tx, rx) = mpsc::channel::>( policy.receiver_queue_capacity, @@ -659,7 +654,10 @@ impl Receiver { let cap_full = tx.capacity() == 0; if cap_full { - trace!(target = "mtp.transport", "accept loop paused: receiver queue full"); + trace!( + target = "mtp.transport", + "accept loop paused: receiver queue full" + ); tokio::select! { _ = close_rx.changed() => { if close_rx.borrow().is_some() { diff --git a/transport/src/pipe.rs b/transport/src/pipe.rs index 33157e7..75fbb37 100644 --- a/transport/src/pipe.rs +++ b/transport/src/pipe.rs @@ -9,13 +9,10 @@ pub struct PipeWriter { impl PipeWriter { pub async fn finish(mut self) -> Result<(), mtp_common::CommunicationError> { - self.stream - .finish() - .await - .map_err(|e| { - log::warn!("[PipeWriter] finish failed: {e}"); - mtp_common::CommunicationError::StreamWriteError(e) - }) + self.stream.finish().await.map_err(|e| { + log::warn!("[PipeWriter] finish failed: {e}"); + mtp_common::CommunicationError::StreamWriteError(e) + }) } pub fn abort(&mut self) -> Result<(), wtransport::error::ClosedStream> { diff --git a/transport/tests/integration.rs b/transport/tests/integration.rs index 276c975..326bcea 100644 --- a/transport/tests/integration.rs +++ b/transport/tests/integration.rs @@ -228,9 +228,7 @@ async fn test_receiver_backpressure_with_small_queue() { let mut h = start_test_host(cert_pem.clone(), key_pem).await; let url = format!("https://127.0.0.1:{}", h.local_addr().port()); let policy = Policy::default().with_receiver_queue_capacity(1); - let (client_tx, client_rx) = connect(&url, Some(cert_pem), policy.clone()) - .await - .unwrap(); + let (client_tx, client_rx) = connect(&url, Some(cert_pem), policy.clone()).await.unwrap(); let (_host_tx, host_rx) = h.next().await.unwrap(); let tm = TypeMap::latest(); @@ -242,13 +240,10 @@ async fn test_receiver_backpressure_with_small_queue() { } for i in 0..8u128 { - let received = tokio::time::timeout( - std::time::Duration::from_secs(5), - host_rx.receive(), - ) - .await - .unwrap() - .unwrap(); + let received = tokio::time::timeout(std::time::Duration::from_secs(5), host_rx.receive()) + .await + .unwrap() + .unwrap(); assert_numbered_message(&received, CommunicationType::Ping, i, &tm); } @@ -332,13 +327,10 @@ async fn test_semaphore_saturation_with_concurrent_streams() { } for i in 0..6u128 { - let received = tokio::time::timeout( - std::time::Duration::from_secs(5), - host_rx.receive(), - ) - .await - .unwrap() - .unwrap(); + let received = tokio::time::timeout(std::time::Duration::from_secs(5), host_rx.receive()) + .await + .unwrap() + .unwrap(); assert_numbered_message(&received, CommunicationType::Ping, i, &tm); } diff --git a/wasm/src/client.rs b/wasm/src/client.rs index 6da9156..f186960 100644 --- a/wasm/src/client.rs +++ b/wasm/src/client.rs @@ -58,7 +58,10 @@ impl WasmPipeHandle { match accepted { Ok(true) => { - let writer = self.transport.open_pipe(self.pipe_id, &self.description).await?; + let writer = self + .transport + .open_pipe(self.pipe_id, &self.description) + .await?; Ok(JsValue::from(writer)) } Ok(false) => Ok(JsValue::NULL), @@ -783,12 +786,11 @@ impl WasmClient { let pipe_id = random_pipe_id()?; let (tx, rx) = oneshot::channel(); - self.pending_pipe_creations - .borrow_mut() - .insert(pipe_id, tx); + self.pending_pipe_creations.borrow_mut().insert(pipe_id, tx); - let request = - CommunicationValue::new(CommunicationType::PipeRequest).with_id(pipe_id).add_typed_default( + let request = CommunicationValue::new(CommunicationType::PipeRequest) + .with_id(pipe_id) + .add_typed_default( DataType::Description, DataValue::Str(description.to_string()), ); diff --git a/wasm/src/pipe.rs b/wasm/src/pipe.rs index 4cf2bfc..b87c258 100644 --- a/wasm/src/pipe.rs +++ b/wasm/src/pipe.rs @@ -1,5 +1,5 @@ -use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; +use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use crate::error::js_error; @@ -56,9 +56,7 @@ impl PipeWriter { let close_promise = close_fn .call0(&self.writer) .map_err(|e| js_error(&format!("close failed: {:?}", e)))?; - if let Err(e) = - JsFuture::from(close_promise.unchecked_into::()).await - { + if let Err(e) = JsFuture::from(close_promise.unchecked_into::()).await { crate::transport::log_stream_error_code(&e, "pipe writer close"); } release_writer_lock(&self.writer); diff --git a/wasm/src/transport.rs b/wasm/src/transport.rs index ab30cc5..b0ff763 100644 --- a/wasm/src/transport.rs +++ b/wasm/src/transport.rs @@ -467,8 +467,8 @@ impl WasmTransport { F: FnMut(JsValue), G: FnMut(crate::pipe::PipeReader), { - let pipe_request_type = mtp_codec::CommunicationType::PipeRequest - .to_id(&mtp_codec::TypeMap::latest()); + let pipe_request_type = + mtp_codec::CommunicationType::PipeRequest.to_id(&mtp_codec::TypeMap::latest()); loop { match self.next_frame().await { @@ -509,8 +509,7 @@ impl WasmTransport { } Err(e) => { let message = e.as_string().unwrap_or_else(|| format!("{:?}", e)); - let _ = - on_error.call1(&JsValue::NULL, &JsValue::from_str(&message)); + let _ = on_error.call1(&JsValue::NULL, &JsValue::from_str(&message)); } } } @@ -574,9 +573,7 @@ impl WasmTransport { let write_promise = write_fn .call1(&writer_val, &chunk) .map_err(|e| js_error(&format!("write failed: {:?}", e)))?; - if let Err(e) = - JsFuture::from(write_promise.unchecked_into::()).await - { + if let Err(e) = JsFuture::from(write_promise.unchecked_into::()).await { log_stream_error_code(&e, "open_pipe write"); release_writer_lock(&writer_val); return Err(e); From 3d757e00f2a289028f192497a000bf79fa921bb8 Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:59:11 +0200 Subject: [PATCH 37/97] Formating --- transport/src/connection.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/transport/src/connection.rs b/transport/src/connection.rs index 00ddad5..b8f9d60 100644 --- a/transport/src/connection.rs +++ b/transport/src/connection.rs @@ -1109,13 +1109,11 @@ impl Receiver { Ok(Some(reader)) } Err(mpsc::error::TryRecvError::Empty) => Ok(None), - Err(mpsc::error::TryRecvError::Disconnected) => { - return Err(self - .inner - .handle - .close_reason() - .unwrap_or(CommunicationError::StreamClosed)); - } + Err(mpsc::error::TryRecvError::Disconnected) => Err(self + .inner + .handle + .close_reason() + .unwrap_or(CommunicationError::StreamClosed)), }, Err(_) => Ok(None), } From 6a65e43ca9f7d128b691c8946adaeecccc2cb2ec Mon Sep 17 00:00:00 2001 From: Alex Emmet <111742636+Alex-Emmet@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:41:54 +0200 Subject: [PATCH 38/97] [Fix] Pipes... --- Cargo.lock | 15 ++- example/server/src/main.rs | 124 +++++++++---------- example/web-client/index.html | 7 +- example/web-client/src/main.ts | 214 ++++++++++++++++++++++++++++++--- host/src/lib.rs | 31 +++++ transport/src/host.rs | 3 +- transport/tests/integration.rs | 2 +- wasm/Cargo.toml | 1 + wasm/src/client.rs | 17 +++ wasm/src/transport.rs | 39 +++--- 10 files changed, 351 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 875b98f..ccf28ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,6 +1105,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", + "web-sys", ] [[package]] @@ -1837,9 +1838,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -2185,6 +2186,16 @@ version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c31d56021e873866c968588ed85ccdf56db5c426e44afdb4618c39895104b920" +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" diff --git a/example/server/src/main.rs b/example/server/src/main.rs index 4ccd627..0cb82db 100644 --- a/example/server/src/main.rs +++ b/example/server/src/main.rs @@ -8,6 +8,7 @@ use mtp::type_map::TypeMap; use std::future::Future; use std::path::Path; use std::pin::Pin; +use std::sync::Arc; fn dev_cert_paths() -> (String, String) { let cert = std::env::var("MTP_DEV_CERT").unwrap_or_else(|_| { @@ -41,13 +42,6 @@ async fn handle_pipe_loopback( let mut reader = req.accept().await?; println!(" [loopback] Pipe {pipe_id} accepted, reading data ..."); - let mut buf = Vec::new(); - tokio::io::AsyncReadExt::read_to_end(&mut reader, &mut buf).await?; - println!( - " [loopback] Pipe {pipe_id} read {} bytes, creating return pipe ...", - buf.len() - ); - let handle = conn.create_pipe("loopback").await?; println!( " [loopback] Return pipe created (id={}), waiting for client ...", @@ -56,16 +50,19 @@ async fn handle_pipe_loopback( match handle.wait().await? { Some(mut writer) => { - println!( - " [loopback] Client accepted return pipe, writing {} bytes ...", - buf.len() - ); - tokio::io::AsyncWriteExt::write_all(&mut writer, &buf).await?; + println!(" [loopback] Client accepted return pipe, echoing incoming bytes ..."); + let mut total = 0usize; + let mut buf = [0u8; 16 * 1024]; + loop { + let n = tokio::io::AsyncReadExt::read(&mut reader, &mut buf).await?; + if n == 0 { + break; + } + total += n; + tokio::io::AsyncWriteExt::write_all(&mut writer, &buf[..n]).await?; + } writer.finish().await?; - println!( - " [loopback] Pipe {pipe_id} loopback complete ({} bytes)", - buf.len() - ); + println!(" [loopback] Pipe {pipe_id} loopback complete ({} bytes)", total); } None => { eprintln!(" [loopback] Return pipe denied by client for pipe {pipe_id}"); @@ -86,8 +83,10 @@ async fn main() -> Result<(), Box> { let (_host_id, host_keyring) = keys::load_or_generate_host_keys("host.mk")?; keys::export_host_public_keys(&host_keyring)?; - let decrypt_keyring = mtp::crypto::Keyring::from_bytes(&host_keyring.to_bytes()) - .expect("re-load host keyring for decryption"); + let decrypt_keyring = Arc::new( + mtp::crypto::Keyring::from_bytes(&host_keyring.to_bytes()) + .expect("re-load host keyring for decryption"), + ); let (clients, next_id) = clients::load_client_db("clients.json")?; @@ -152,61 +151,64 @@ async fn main() -> Result<(), Box> { println!("Server listening on {}", host.local_addr()); while let Some(conn) = host.accept().await? { - let desc = conn.description.as_deref().unwrap_or("(no description)"); - println!( - "\n--- New connection (version {}, description: {desc}) ---", - conn.version - ); - println!("Client ID: {}", conn.client_id); + let decrypt_keyring = Arc::clone(&decrypt_keyring); + tokio::spawn(async move { + let desc = conn.description.as_deref().unwrap_or("(no description)"); + println!( + "\n--- New connection (version {}, description: {desc}) ---", + conn.version + ); + println!("Client ID: {}", conn.client_id); - let tm: &TypeMap = conn.codec.registry().get(&conn.version).unwrap(); + let tm: &TypeMap = conn.codec.registry().get(&conn.version).unwrap(); - println!("Waiting for messages / pipe requests ..."); - loop { - tokio::select! { - biased; + println!("Waiting for messages / pipe requests ..."); + loop { + tokio::select! { + biased; - pipe_req = conn.receive_pipe() => { - match pipe_req { - Ok(req) => { - println!(" Pipe request: id={} desc={:?}", req.id(), req.description()); - if let Err(e) = handle_pipe_loopback(&conn, req).await { - eprintln!(" Pipe loopback error: {e}"); + pipe_req = conn.receive_pipe() => { + match pipe_req { + Ok(req) => { + println!(" Pipe request: id={} desc={:?}", req.id(), req.description()); + if let Err(e) = handle_pipe_loopback(&conn, req).await { + eprintln!(" Pipe loopback error: {e}"); + } } - } - Err(e) => { - println!("Pipe channel closed: {e}"); - break; - } - } - } - msg = conn.receive() => { - match msg { - Ok(msg) => { - println!("Received: {msg}"); - let response = handlers::process_and_respond( - &msg, - tm, - conn.client_public_key.as_ref(), - &decrypt_keyring, - ); - println!("Sending: {response}"); - if let Err(e) = conn.sender.send(&response).await { - eprintln!("Send error: {e}"); + Err(e) => { + println!("Pipe channel closed: {e}"); break; } } - Err(e) => { - println!("Connection ended: {e}"); - break; + } + msg = conn.receive() => { + match msg { + Ok(msg) => { + println!("Received: {msg}"); + let response = handlers::process_and_respond( + &msg, + tm, + conn.client_public_key.as_ref(), + &decrypt_keyring, + ); + println!("Sending: {response}"); + if let Err(e) = conn.sender.send(&response).await { + eprintln!("Send error: {e}"); + break; + } + } + Err(e) => { + println!("Connection ended: {e}"); + break; + } } } } } - } - conn.sender.close(); - println!("Connection closed\n"); + conn.sender.close(); + println!("Connection closed\n"); + }); } Ok(()) diff --git a/example/web-client/index.html b/example/web-client/index.html index 941b59e..f2fadda 100644 --- a/example/web-client/index.html +++ b/example/web-client/index.html @@ -51,7 +51,7 @@

MTP WebTransport Client

- +