From 7d885ecd577de1393ca5068c6f250479218e8210 Mon Sep 17 00:00:00 2001 From: samuelzielke Date: Thu, 22 Oct 2026 10:43:09 +0200 Subject: [PATCH] feat(intercom): SDP offer/answer and RTP packetization for Opus and H.264 Co-Authored-By: Claude Opus 5.5 --- .../MusubiKit/Sources/Intercom/RTC/RTP.swift | 135 +++++++++++++++ .../MusubiKit/Sources/Intercom/RTC/SDP.swift | 154 ++++++++++++++++++ .../Tests/IntercomTests/SDPRTPTests.swift | 140 ++++++++++++++++ 3 files changed, 429 insertions(+) create mode 100644 Packages/MusubiKit/Sources/Intercom/RTC/RTP.swift create mode 100644 Packages/MusubiKit/Sources/Intercom/RTC/SDP.swift create mode 100644 Packages/MusubiKit/Tests/IntercomTests/SDPRTPTests.swift diff --git a/Packages/MusubiKit/Sources/Intercom/RTC/RTP.swift b/Packages/MusubiKit/Sources/Intercom/RTC/RTP.swift new file mode 100644 index 0000000..c53b247 --- /dev/null +++ b/Packages/MusubiKit/Sources/Intercom/RTC/RTP.swift @@ -0,0 +1,135 @@ +import Foundation + +/// Writes RTP packets for one outgoing stream (RFC 3550). +struct RTPStream { + let ssrc: UInt32 + let payloadType: UInt8 + private(set) var sequence: UInt16 = .random(in: 0...UInt16.max) + private(set) var packetCount: UInt32 = 0 + private(set) var octetCount: UInt32 = 0 + + init(ssrc: UInt32, payloadType: UInt8) { + self.ssrc = ssrc + self.payloadType = payloadType + } + + mutating func packet(timestamp: UInt32, marker: Bool, payload: [UInt8]) -> [UInt8] { + var w = ByteWriter(capacity: 12 + payload.count) + w.u8(0x80) + w.u8((marker ? 0x80 : 0) | (payloadType & 0x7F)) + w.u16(sequence) + w.u32(timestamp) + w.u32(ssrc) + w.append(payload) + sequence &+= 1 + packetCount &+= 1 + octetCount &+= UInt32(payload.count) + return w.bytes + } +} + +/// H.264 packetization (RFC 6184, packetization-mode 1): single NAL units, STAP-A and FU-A. +enum H264Packetizer { + /// Largest RTP payload; keeps IP packets well below 1500 bytes after SRTP and IPv6 headers. + static let maxPayload = 1100 + + /// Splits one access unit (NAL units without start codes or length prefixes) into RTP payloads. + /// Small leading parameter sets (SPS/PPS) are aggregated into one STAP-A. + static func payloads(for nalUnits: [[UInt8]]) -> [[UInt8]] { + var out: [[UInt8]] = [] + var aggregate: [[UInt8]] = [] + + func flushAggregate() { + if aggregate.count == 1 { + out.append(aggregate[0]) + } else if aggregate.count > 1 { + var w = ByteWriter() + w.u8((aggregate.map { $0[0] & 0x60 }.max() ?? 0) | 24) // STAP-A, highest NRI + for nal in aggregate { w.u16(UInt16(nal.count)); w.append(nal) } + out.append(w.bytes) + } + aggregate = [] + } + + for nal in nalUnits where !nal.isEmpty { + let type = nal[0] & 0x1F + if type == 7 || type == 8 { + let size = aggregate.reduce(1) { $0 + 2 + $1.count } + 2 + nal.count + if size > maxPayload { flushAggregate() } + aggregate.append(nal) + continue + } + flushAggregate() + if nal.count <= maxPayload { + out.append(nal) + continue + } + let indicator = (nal[0] & 0xE0) | 28 // FU-A + var offset = 1 + while offset < nal.count { + let end = min(offset + maxPayload - 2, nal.count) + var header = type + if offset == 1 { header |= 0x80 } + if end == nal.count { header |= 0x40 } + out.append([indicator, header] + nal[offset.. [[UInt8]] { + var out: [[UInt8]] = [] + var i = 0 + while i + 4 <= avcc.count { + let length = Int(avcc[i]) << 24 | Int(avcc[i + 1]) << 16 | Int(avcc[i + 2]) << 8 | Int(avcc[i + 3]) + i += 4 + guard length > 0, i + length <= avcc.count else { break } + out.append(Array(avcc[i.. Bool { + var i = 0 + while i + 4 <= packet.count { + let fmt = packet[i] & 0x1F + let type = packet[i + 1] + let length = ((Int(packet[i + 2]) << 8 | Int(packet[i + 3])) + 1) * 4 + if (type == 206 && fmt == 1) || (type == 206 && fmt == 4) { return true } + guard length > 0 else { return false } + i += length + } + return false + } + + /// RFC 7983: RTP/RTCP start with version 2 (first byte 128…191); RTCP payload types are 192…223. + static func isRTCP(_ packet: [UInt8]) -> Bool { + packet.count >= 8 && packet[0] >= 128 && packet[0] <= 191 && packet[1] >= 192 && packet[1] <= 223 + } + + /// Sender report so the browser can map RTP timestamps to wall clock (RFC 3550 6.4.1). + static func senderReport(ssrc: UInt32, rtpTimestamp: UInt32, packets: UInt32, octets: UInt32, now: Date = Date()) -> [UInt8] { + let ntp = now.timeIntervalSince1970 + 2_208_988_800 + let seconds = UInt32(ntp) + let fraction = UInt32((ntp - Double(seconds)) * 4_294_967_296) + var w = ByteWriter(capacity: 28) + w.u8(0x80) + w.u8(200) + w.u16(6) + w.u32(ssrc) + w.u32(seconds) + w.u32(fraction) + w.u32(rtpTimestamp) + w.u32(packets) + w.u32(octets) + return w.bytes + } +} diff --git a/Packages/MusubiKit/Sources/Intercom/RTC/SDP.swift b/Packages/MusubiKit/Sources/Intercom/RTC/SDP.swift new file mode 100644 index 0000000..bd2204f --- /dev/null +++ b/Packages/MusubiKit/Sources/Intercom/RTC/SDP.swift @@ -0,0 +1,154 @@ +import Foundation + +/// What Musubi needs from a browser's SDP offer (Unified Plan, BUNDLE, rtcp-mux). +struct SDPOffer: Equatable { + struct Media: Equatable { + var kind: String + var mid: String + var port: Int + var payloadTypes: [Int] + var rtpmap: [Int: String] = [:] + var fmtp: [Int: String] = [:] + } + + var iceUfrag: String + var icePwd: String + /// `sha-256` fingerprint, uppercase hex with colons. + var fingerprint: String + var media: [Media] + + enum ParseError: Error, Equatable { case missing(String) } + + init(_ sdp: String) throws { + var sessionUfrag: String?, sessionPwd: String?, sessionFingerprint: String? + var mediaUfrag: String?, mediaPwd: String?, mediaFingerprint: String? + var media: [Media] = [] + + for raw in sdp.split(whereSeparator: \.isNewline) { + let line = raw.trimmingCharacters(in: .whitespaces) + if line.hasPrefix("m=") { + let parts = line.dropFirst(2).split(separator: " ") + guard parts.count >= 3 else { continue } + media.append(Media(kind: String(parts[0]), mid: "\(media.count)", port: Int(parts[1]) ?? 0, + payloadTypes: parts.dropFirst(3).compactMap { Int($0) })) + continue + } + guard line.hasPrefix("a=") else { continue } + let attribute = line.dropFirst(2) + let (name, value) = attribute.firstIndex(of: ":").map { + (String(attribute[..<$0]), String(attribute[attribute.index(after: $0)...])) + } ?? (String(attribute), "") + let inMedia = !media.isEmpty + + switch name { + case "ice-ufrag": if inMedia { mediaUfrag = mediaUfrag ?? value } else { sessionUfrag = value } + case "ice-pwd": if inMedia { mediaPwd = mediaPwd ?? value } else { sessionPwd = value } + case "fingerprint": + let parts = value.split(separator: " ") + guard parts.count == 2, parts[0].lowercased() == "sha-256" else { continue } + if inMedia { mediaFingerprint = mediaFingerprint ?? parts[1].uppercased() } else { sessionFingerprint = parts[1].uppercased() } + case "mid" where inMedia: + media[media.count - 1].mid = value + case "rtpmap" where inMedia, "fmtp" where inMedia: + guard let space = value.firstIndex(of: " "), let pt = Int(value[.. Int? { + media.payloadTypes.first { media.rtpmap[$0]?.lowercased().hasPrefix("opus/48000") == true } + } + + /// Payload type for H.264 with packetization-mode=1 in (constrained) baseline profile. + func h264PayloadType(in media: Media) -> Int? { + let candidates = media.payloadTypes.filter { pt in + guard media.rtpmap[pt]?.uppercased().hasPrefix("H264/90000") == true, let fmtp = media.fmtp[pt]?.lowercased() else { return false } + return fmtp.contains("packetization-mode=1") + } + func profile(_ pt: Int) -> String { + media.fmtp[pt]?.lowercased().split(separator: ";") + .first { $0.trimmingCharacters(in: .whitespaces).hasPrefix("profile-level-id=") } + .map { String($0.split(separator: "=").last ?? "") } ?? "" + } + return candidates.first { profile($0).hasPrefix("42e0") } ?? candidates.first { profile($0).hasPrefix("4200") } + } +} + +/// Builds the ICE-lite, DTLS-passive, send-only answer. +struct SDPAnswer { + struct Track { + var mid: String + var kind: String + /// nil rejects the section (port 0). + var payloadType: Int? + var ssrc: UInt32 + } + + var iceUfrag: String + var icePwd: String + var fingerprint: String + var address: String + var port: UInt16 + var tracks: [Track] + + var text: String { + let accepted = tracks.filter { $0.payloadType != nil } + var lines = [ + "v=0", + "o=- \(UInt32.random(in: 1...UInt32.max)) 2 IN IP4 \(address)", + "s=Musubi", + "t=0 0", + "a=ice-lite", + "a=group:BUNDLE \(accepted.map(\.mid).joined(separator: " "))", + "a=msid-semantic: WMS musubi", + ] + for track in tracks { + guard let pt = track.payloadType else { + lines += ["m=\(track.kind) 0 UDP/TLS/RTP/SAVPF 0", "c=IN IP4 0.0.0.0", "a=mid:\(track.mid)", "a=inactive"] + continue + } + lines += [ + "m=\(track.kind) \(port) UDP/TLS/RTP/SAVPF \(pt)", + "c=IN IP4 \(address)", + "a=rtcp:\(port) IN IP4 \(address)", + "a=ice-ufrag:\(iceUfrag)", + "a=ice-pwd:\(icePwd)", + "a=fingerprint:sha-256 \(fingerprint)", + "a=setup:passive", + "a=mid:\(track.mid)", + "a=sendonly", + "a=rtcp-mux", + "a=msid:musubi \(track.kind)", + ] + if track.kind == "audio" { + lines += ["a=rtpmap:\(pt) opus/48000/2", "a=fmtp:\(pt) minptime=10;useinbandfec=1"] + } else { + lines += [ + "a=rtpmap:\(pt) H264/90000", + "a=rtcp-fb:\(pt) nack pli", + "a=rtcp-fb:\(pt) ccm fir", + "a=fmtp:\(pt) level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + ] + } + lines += [ + "a=ssrc:\(track.ssrc) cname:musubi", + "a=ssrc:\(track.ssrc) msid:musubi \(track.kind)", + "a=candidate:1 1 udp 2130706431 \(address) \(port) typ host", + "a=end-of-candidates", + ] + } + return lines.joined(separator: "\r\n") + "\r\n" + } +} diff --git a/Packages/MusubiKit/Tests/IntercomTests/SDPRTPTests.swift b/Packages/MusubiKit/Tests/IntercomTests/SDPRTPTests.swift new file mode 100644 index 0000000..fa00d78 --- /dev/null +++ b/Packages/MusubiKit/Tests/IntercomTests/SDPRTPTests.swift @@ -0,0 +1,140 @@ +import Foundation +import Testing +@testable import Intercom + +@Suite struct SDPTests { + // Shortened recvonly offer as Chrome/Safari produce it for two transceivers. + let offer = """ + v=0\r + o=- 4611731400430051336 2 IN IP4 127.0.0.1\r + s=-\r + t=0 0\r + a=group:BUNDLE 0 1\r + a=extmap-allow-mixed\r + a=msid-semantic: WMS\r + m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8\r + c=IN IP4 0.0.0.0\r + a=rtcp:9 IN IP4 0.0.0.0\r + a=ice-ufrag:Ab3x\r + a=ice-pwd:k9Qmw0Rr1qY2yT7cXv8sJp4u\r + a=ice-options:trickle\r + a=fingerprint:sha-256 4c:1b:00:8f:6d:3f:9b:39:72:20:4e:69:ef:4a:16:1c:c2:39:d9:e4:3f:1e:f1:5b:3b:91:85:73:a0:cd:7a:4e\r + a=setup:actpass\r + a=mid:0\r + a=recvonly\r + a=rtcp-mux\r + a=rtpmap:111 opus/48000/2\r + a=fmtp:111 minptime=10;useinbandfec=1\r + a=rtpmap:63 red/48000/2\r + a=rtpmap:9 G722/8000\r + m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 127 125\r + c=IN IP4 0.0.0.0\r + a=ice-ufrag:Ab3x\r + a=ice-pwd:k9Qmw0Rr1qY2yT7cXv8sJp4u\r + a=fingerprint:sha-256 4c:1b:00:8f:6d:3f:9b:39:72:20:4e:69:ef:4a:16:1c:c2:39:d9:e4:3f:1e:f1:5b:3b:91:85:73:a0:cd:7a:4e\r + a=setup:actpass\r + a=mid:1\r + a=recvonly\r + a=rtcp-mux\r + a=rtpmap:96 VP8/90000\r + a=rtpmap:97 rtx/90000\r + a=fmtp:97 apt=96\r + a=rtpmap:102 H264/90000\r + a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r + a=rtpmap:103 H264/90000\r + a=fmtp:103 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r + a=rtpmap:127 H264/90000\r + a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r + a=rtpmap:125 H264/90000\r + a=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f\r + + """ + + @Test func parsesOffer() throws { + let parsed = try SDPOffer(offer) + #expect(parsed.iceUfrag == "Ab3x") + #expect(parsed.icePwd == "k9Qmw0Rr1qY2yT7cXv8sJp4u") + #expect(parsed.fingerprint.hasPrefix("4C:1B:00:8F")) + #expect(parsed.media.map(\.kind) == ["audio", "video"]) + #expect(parsed.media.map(\.mid) == ["0", "1"]) + #expect(parsed.opusPayloadType(in: parsed.media[0]) == 111) + // Constrained baseline with packetization-mode=1 wins. + #expect(parsed.h264PayloadType(in: parsed.media[1]) == 127) + } + + @Test func missingCredentialsFail() { + #expect(throws: SDPOffer.ParseError.missing("ice-ufrag")) { try SDPOffer("v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n") } + } + + @Test func answerIsIceLitePassiveSendOnly() throws { + let answer = SDPAnswer(iceUfrag: "musu", icePwd: "0123456789abcdefghijklmn", fingerprint: "AA:BB", address: "192.168.1.20", + port: 50000, tracks: [.init(mid: "0", kind: "audio", payloadType: 111, ssrc: 11), + .init(mid: "1", kind: "video", payloadType: 127, ssrc: 22)]).text + #expect(answer.contains("a=ice-lite\r\n")) + #expect(answer.contains("a=group:BUNDLE 0 1\r\n")) + #expect(answer.contains("m=audio 50000 UDP/TLS/RTP/SAVPF 111\r\n")) + #expect(answer.contains("m=video 50000 UDP/TLS/RTP/SAVPF 127\r\n")) + #expect(answer.contains("a=setup:passive\r\n")) + #expect(answer.contains("a=candidate:1 1 udp 2130706431 192.168.1.20 50000 typ host\r\n")) + #expect(!answer.contains("a=recvonly")) + } + + @Test func rejectedTrackGetsPortZero() { + let answer = SDPAnswer(iceUfrag: "u", icePwd: "p", fingerprint: "F", address: "10.0.0.1", port: 1, + tracks: [.init(mid: "0", kind: "audio", payloadType: 111, ssrc: 1), + .init(mid: "1", kind: "video", payloadType: nil, ssrc: 2)]).text + #expect(answer.contains("a=group:BUNDLE 0\r\n")) + #expect(answer.contains("m=video 0 UDP/TLS/RTP/SAVPF 0\r\n")) + } +} + +@Suite struct RTPTests { + @Test func headerFields() { + var stream = RTPStream(ssrc: 0x0102_0304, payloadType: 111) + let first = stream.packet(timestamp: 960, marker: true, payload: [9, 9]) + let second = stream.packet(timestamp: 1920, marker: false, payload: []) + #expect(first[0] == 0x80) + #expect(first[1] == 0x80 | 111) + #expect(Array(first[4..<12]) == [0, 0, 3, 0xC0, 1, 2, 3, 4]) + #expect(Array(first[12...]) == [9, 9]) + let seq1 = UInt16(first[2]) << 8 | UInt16(first[3]) + let seq2 = UInt16(second[2]) << 8 | UInt16(second[3]) + #expect(seq2 == seq1 &+ 1) + } + + @Test func aggregatesParameterSetsAndKeepsSmallNALs() { + let sps: [UInt8] = [0x67, 1, 2, 3] + let pps: [UInt8] = [0x68, 4] + let idr: [UInt8] = [0x65] + Array(repeating: 7, count: 100) + let payloads = H264Packetizer.payloads(for: [sps, pps, idr]) + #expect(payloads.count == 2) + #expect(payloads[0] == [0x78, 0, 4] + sps + [0, 2] + pps) // STAP-A with NRI 3 + #expect(payloads[1] == idr) + } + + @Test func fragmentsLargeNALAndReassembles() { + let nal: [UInt8] = [0x65] + (0..<5000).map { UInt8($0 & 0xFF) } + let payloads = H264Packetizer.payloads(for: [nal]) + #expect(payloads.count > 1) + #expect(payloads.allSatisfy { $0.count <= H264Packetizer.maxPayload }) + #expect(payloads.first?[1] == 0x85) // start bit + type 5 + #expect(payloads.last?[1] == 0x45) // end bit + type 5 + #expect(payloads.allSatisfy { $0[0] == 0x7C }) // FU-A, NRI 3 + let rebuilt = [UInt8(0x65)] + payloads.flatMap { $0.dropFirst(2) } + #expect(rebuilt == nal) + } + + @Test func splitsAVCC() { + let avcc: [UInt8] = [0, 0, 0, 2, 0x67, 1, 0, 0, 0, 3, 0x65, 2, 3] + #expect(H264Packetizer.nalUnits(avcc: avcc) == [[0x67, 1], [0x65, 2, 3]]) + } + + @Test func detectsKeyFrameRequests() { + let receiverReport: [UInt8] = [0x80, 201, 0, 1, 0, 0, 0, 1] + let pli: [UInt8] = [0x81, 206, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2] + #expect(RTCPFeedback.requestsKeyFrame(receiverReport + pli)) + #expect(!RTCPFeedback.requestsKeyFrame(receiverReport)) + #expect(RTCPFeedback.isRTCP(pli)) + #expect(!RTCPFeedback.isRTCP([0x80, 111, 0, 1, 0, 0, 0, 0])) + } +}