feat(intercom): SDP offer/answer and RTP packetization for Opus and H.264

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
main
Samuel Zielke 1 day ago
parent 3e245b1ae3
commit 7d885ecd57

@ -0,0 +1,135 @@
import Foundation
/// Writes RTP packets for one outgoing stream (RFC 3550).
struct RTPStream {
let ssrc: UInt32
let payloadType: UInt8
private(set) var sequence: UInt16 = .random(in: 0...UInt16.max)
private(set) var packetCount: UInt32 = 0
private(set) var octetCount: UInt32 = 0
init(ssrc: UInt32, payloadType: UInt8) {
self.ssrc = ssrc
self.payloadType = payloadType
}
mutating func packet(timestamp: UInt32, marker: Bool, payload: [UInt8]) -> [UInt8] {
var w = ByteWriter(capacity: 12 + payload.count)
w.u8(0x80)
w.u8((marker ? 0x80 : 0) | (payloadType & 0x7F))
w.u16(sequence)
w.u32(timestamp)
w.u32(ssrc)
w.append(payload)
sequence &+= 1
packetCount &+= 1
octetCount &+= UInt32(payload.count)
return w.bytes
}
}
/// H.264 packetization (RFC 6184, packetization-mode 1): single NAL units, STAP-A and FU-A.
enum H264Packetizer {
/// Largest RTP payload; keeps IP packets well below 1500 bytes after SRTP and IPv6 headers.
static let maxPayload = 1100
/// Splits one access unit (NAL units without start codes or length prefixes) into RTP payloads.
/// Small leading parameter sets (SPS/PPS) are aggregated into one STAP-A.
static func payloads(for nalUnits: [[UInt8]]) -> [[UInt8]] {
var out: [[UInt8]] = []
var aggregate: [[UInt8]] = []
func flushAggregate() {
if aggregate.count == 1 {
out.append(aggregate[0])
} else if aggregate.count > 1 {
var w = ByteWriter()
w.u8((aggregate.map { $0[0] & 0x60 }.max() ?? 0) | 24) // STAP-A, highest NRI
for nal in aggregate { w.u16(UInt16(nal.count)); w.append(nal) }
out.append(w.bytes)
}
aggregate = []
}
for nal in nalUnits where !nal.isEmpty {
let type = nal[0] & 0x1F
if type == 7 || type == 8 {
let size = aggregate.reduce(1) { $0 + 2 + $1.count } + 2 + nal.count
if size > maxPayload { flushAggregate() }
aggregate.append(nal)
continue
}
flushAggregate()
if nal.count <= maxPayload {
out.append(nal)
continue
}
let indicator = (nal[0] & 0xE0) | 28 // FU-A
var offset = 1
while offset < nal.count {
let end = min(offset + maxPayload - 2, nal.count)
var header = type
if offset == 1 { header |= 0x80 }
if end == nal.count { header |= 0x40 }
out.append([indicator, header] + nal[offset..<end])
offset = end
}
}
flushAggregate()
return out
}
/// Splits an AVCC buffer (4-byte big-endian length prefixes, as from VideoToolbox) into NAL units.
static func nalUnits(avcc: [UInt8]) -> [[UInt8]] {
var out: [[UInt8]] = []
var i = 0
while i + 4 <= avcc.count {
let length = Int(avcc[i]) << 24 | Int(avcc[i + 1]) << 16 | Int(avcc[i + 2]) << 8 | Int(avcc[i + 3])
i += 4
guard length > 0, i + length <= avcc.count else { break }
out.append(Array(avcc[i..<i + length]))
i += length
}
return out
}
}
/// RTCP feedback the browser sends that the sender must act on.
enum RTCPFeedback {
/// True if a (compound) RTCP packet asks for a key frame (PLI or FIR).
static func requestsKeyFrame(_ packet: [UInt8]) -> Bool {
var i = 0
while i + 4 <= packet.count {
let fmt = packet[i] & 0x1F
let type = packet[i + 1]
let length = ((Int(packet[i + 2]) << 8 | Int(packet[i + 3])) + 1) * 4
if (type == 206 && fmt == 1) || (type == 206 && fmt == 4) { return true }
guard length > 0 else { return false }
i += length
}
return false
}
/// RFC 7983: RTP/RTCP start with version 2 (first byte 128…191); RTCP payload types are 192…223.
static func isRTCP(_ packet: [UInt8]) -> Bool {
packet.count >= 8 && packet[0] >= 128 && packet[0] <= 191 && packet[1] >= 192 && packet[1] <= 223
}
/// Sender report so the browser can map RTP timestamps to wall clock (RFC 3550 6.4.1).
static func senderReport(ssrc: UInt32, rtpTimestamp: UInt32, packets: UInt32, octets: UInt32, now: Date = Date()) -> [UInt8] {
let ntp = now.timeIntervalSince1970 + 2_208_988_800
let seconds = UInt32(ntp)
let fraction = UInt32((ntp - Double(seconds)) * 4_294_967_296)
var w = ByteWriter(capacity: 28)
w.u8(0x80)
w.u8(200)
w.u16(6)
w.u32(ssrc)
w.u32(seconds)
w.u32(fraction)
w.u32(rtpTimestamp)
w.u32(packets)
w.u32(octets)
return w.bytes
}
}

@ -0,0 +1,154 @@
import Foundation
/// What Musubi needs from a browser's SDP offer (Unified Plan, BUNDLE, rtcp-mux).
struct SDPOffer: Equatable {
struct Media: Equatable {
var kind: String
var mid: String
var port: Int
var payloadTypes: [Int]
var rtpmap: [Int: String] = [:]
var fmtp: [Int: String] = [:]
}
var iceUfrag: String
var icePwd: String
/// `sha-256` fingerprint, uppercase hex with colons.
var fingerprint: String
var media: [Media]
enum ParseError: Error, Equatable { case missing(String) }
init(_ sdp: String) throws {
var sessionUfrag: String?, sessionPwd: String?, sessionFingerprint: String?
var mediaUfrag: String?, mediaPwd: String?, mediaFingerprint: String?
var media: [Media] = []
for raw in sdp.split(whereSeparator: \.isNewline) {
let line = raw.trimmingCharacters(in: .whitespaces)
if line.hasPrefix("m=") {
let parts = line.dropFirst(2).split(separator: " ")
guard parts.count >= 3 else { continue }
media.append(Media(kind: String(parts[0]), mid: "\(media.count)", port: Int(parts[1]) ?? 0,
payloadTypes: parts.dropFirst(3).compactMap { Int($0) }))
continue
}
guard line.hasPrefix("a=") else { continue }
let attribute = line.dropFirst(2)
let (name, value) = attribute.firstIndex(of: ":").map {
(String(attribute[..<$0]), String(attribute[attribute.index(after: $0)...]))
} ?? (String(attribute), "")
let inMedia = !media.isEmpty
switch name {
case "ice-ufrag": if inMedia { mediaUfrag = mediaUfrag ?? value } else { sessionUfrag = value }
case "ice-pwd": if inMedia { mediaPwd = mediaPwd ?? value } else { sessionPwd = value }
case "fingerprint":
let parts = value.split(separator: " ")
guard parts.count == 2, parts[0].lowercased() == "sha-256" else { continue }
if inMedia { mediaFingerprint = mediaFingerprint ?? parts[1].uppercased() } else { sessionFingerprint = parts[1].uppercased() }
case "mid" where inMedia:
media[media.count - 1].mid = value
case "rtpmap" where inMedia, "fmtp" where inMedia:
guard let space = value.firstIndex(of: " "), let pt = Int(value[..<space]) else { continue }
let rest = String(value[value.index(after: space)...])
if name == "rtpmap" { media[media.count - 1].rtpmap[pt] = rest } else { media[media.count - 1].fmtp[pt] = rest }
default: break
}
}
guard let ufrag = sessionUfrag ?? mediaUfrag else { throw ParseError.missing("ice-ufrag") }
guard let pwd = sessionPwd ?? mediaPwd else { throw ParseError.missing("ice-pwd") }
guard let fingerprint = sessionFingerprint ?? mediaFingerprint else { throw ParseError.missing("fingerprint") }
iceUfrag = ufrag
icePwd = pwd
self.fingerprint = fingerprint
self.media = media
}
/// Payload type for Opus at 48 kHz.
func opusPayloadType(in media: Media) -> Int? {
media.payloadTypes.first { media.rtpmap[$0]?.lowercased().hasPrefix("opus/48000") == true }
}
/// Payload type for H.264 with packetization-mode=1 in (constrained) baseline profile.
func h264PayloadType(in media: Media) -> Int? {
let candidates = media.payloadTypes.filter { pt in
guard media.rtpmap[pt]?.uppercased().hasPrefix("H264/90000") == true, let fmtp = media.fmtp[pt]?.lowercased() else { return false }
return fmtp.contains("packetization-mode=1")
}
func profile(_ pt: Int) -> String {
media.fmtp[pt]?.lowercased().split(separator: ";")
.first { $0.trimmingCharacters(in: .whitespaces).hasPrefix("profile-level-id=") }
.map { String($0.split(separator: "=").last ?? "") } ?? ""
}
return candidates.first { profile($0).hasPrefix("42e0") } ?? candidates.first { profile($0).hasPrefix("4200") }
}
}
/// Builds the ICE-lite, DTLS-passive, send-only answer.
struct SDPAnswer {
struct Track {
var mid: String
var kind: String
/// nil rejects the section (port 0).
var payloadType: Int?
var ssrc: UInt32
}
var iceUfrag: String
var icePwd: String
var fingerprint: String
var address: String
var port: UInt16
var tracks: [Track]
var text: String {
let accepted = tracks.filter { $0.payloadType != nil }
var lines = [
"v=0",
"o=- \(UInt32.random(in: 1...UInt32.max)) 2 IN IP4 \(address)",
"s=Musubi",
"t=0 0",
"a=ice-lite",
"a=group:BUNDLE \(accepted.map(\.mid).joined(separator: " "))",
"a=msid-semantic: WMS musubi",
]
for track in tracks {
guard let pt = track.payloadType else {
lines += ["m=\(track.kind) 0 UDP/TLS/RTP/SAVPF 0", "c=IN IP4 0.0.0.0", "a=mid:\(track.mid)", "a=inactive"]
continue
}
lines += [
"m=\(track.kind) \(port) UDP/TLS/RTP/SAVPF \(pt)",
"c=IN IP4 \(address)",
"a=rtcp:\(port) IN IP4 \(address)",
"a=ice-ufrag:\(iceUfrag)",
"a=ice-pwd:\(icePwd)",
"a=fingerprint:sha-256 \(fingerprint)",
"a=setup:passive",
"a=mid:\(track.mid)",
"a=sendonly",
"a=rtcp-mux",
"a=msid:musubi \(track.kind)",
]
if track.kind == "audio" {
lines += ["a=rtpmap:\(pt) opus/48000/2", "a=fmtp:\(pt) minptime=10;useinbandfec=1"]
} else {
lines += [
"a=rtpmap:\(pt) H264/90000",
"a=rtcp-fb:\(pt) nack pli",
"a=rtcp-fb:\(pt) ccm fir",
"a=fmtp:\(pt) level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
]
}
lines += [
"a=ssrc:\(track.ssrc) cname:musubi",
"a=ssrc:\(track.ssrc) msid:musubi \(track.kind)",
"a=candidate:1 1 udp 2130706431 \(address) \(port) typ host",
"a=end-of-candidates",
]
}
return lines.joined(separator: "\r\n") + "\r\n"
}
}

@ -0,0 +1,140 @@
import Foundation
import Testing
@testable import Intercom
@Suite struct SDPTests {
// Shortened recvonly offer as Chrome/Safari produce it for two transceivers.
let offer = """
v=0\r
o=- 4611731400430051336 2 IN IP4 127.0.0.1\r
s=-\r
t=0 0\r
a=group:BUNDLE 0 1\r
a=extmap-allow-mixed\r
a=msid-semantic: WMS\r
m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8\r
c=IN IP4 0.0.0.0\r
a=rtcp:9 IN IP4 0.0.0.0\r
a=ice-ufrag:Ab3x\r
a=ice-pwd:k9Qmw0Rr1qY2yT7cXv8sJp4u\r
a=ice-options:trickle\r
a=fingerprint:sha-256 4c:1b:00:8f:6d:3f:9b:39:72:20:4e:69:ef:4a:16:1c:c2:39:d9:e4:3f:1e:f1:5b:3b:91:85:73:a0:cd:7a:4e\r
a=setup:actpass\r
a=mid:0\r
a=recvonly\r
a=rtcp-mux\r
a=rtpmap:111 opus/48000/2\r
a=fmtp:111 minptime=10;useinbandfec=1\r
a=rtpmap:63 red/48000/2\r
a=rtpmap:9 G722/8000\r
m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 127 125\r
c=IN IP4 0.0.0.0\r
a=ice-ufrag:Ab3x\r
a=ice-pwd:k9Qmw0Rr1qY2yT7cXv8sJp4u\r
a=fingerprint:sha-256 4c:1b:00:8f:6d:3f:9b:39:72:20:4e:69:ef:4a:16:1c:c2:39:d9:e4:3f:1e:f1:5b:3b:91:85:73:a0:cd:7a:4e\r
a=setup:actpass\r
a=mid:1\r
a=recvonly\r
a=rtcp-mux\r
a=rtpmap:96 VP8/90000\r
a=rtpmap:97 rtx/90000\r
a=fmtp:97 apt=96\r
a=rtpmap:102 H264/90000\r
a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r
a=rtpmap:103 H264/90000\r
a=fmtp:103 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r
a=rtpmap:127 H264/90000\r
a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r
a=rtpmap:125 H264/90000\r
a=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f\r
"""
@Test func parsesOffer() throws {
let parsed = try SDPOffer(offer)
#expect(parsed.iceUfrag == "Ab3x")
#expect(parsed.icePwd == "k9Qmw0Rr1qY2yT7cXv8sJp4u")
#expect(parsed.fingerprint.hasPrefix("4C:1B:00:8F"))
#expect(parsed.media.map(\.kind) == ["audio", "video"])
#expect(parsed.media.map(\.mid) == ["0", "1"])
#expect(parsed.opusPayloadType(in: parsed.media[0]) == 111)
// Constrained baseline with packetization-mode=1 wins.
#expect(parsed.h264PayloadType(in: parsed.media[1]) == 127)
}
@Test func missingCredentialsFail() {
#expect(throws: SDPOffer.ParseError.missing("ice-ufrag")) { try SDPOffer("v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n") }
}
@Test func answerIsIceLitePassiveSendOnly() throws {
let answer = SDPAnswer(iceUfrag: "musu", icePwd: "0123456789abcdefghijklmn", fingerprint: "AA:BB", address: "192.168.1.20",
port: 50000, tracks: [.init(mid: "0", kind: "audio", payloadType: 111, ssrc: 11),
.init(mid: "1", kind: "video", payloadType: 127, ssrc: 22)]).text
#expect(answer.contains("a=ice-lite\r\n"))
#expect(answer.contains("a=group:BUNDLE 0 1\r\n"))
#expect(answer.contains("m=audio 50000 UDP/TLS/RTP/SAVPF 111\r\n"))
#expect(answer.contains("m=video 50000 UDP/TLS/RTP/SAVPF 127\r\n"))
#expect(answer.contains("a=setup:passive\r\n"))
#expect(answer.contains("a=candidate:1 1 udp 2130706431 192.168.1.20 50000 typ host\r\n"))
#expect(!answer.contains("a=recvonly"))
}
@Test func rejectedTrackGetsPortZero() {
let answer = SDPAnswer(iceUfrag: "u", icePwd: "p", fingerprint: "F", address: "10.0.0.1", port: 1,
tracks: [.init(mid: "0", kind: "audio", payloadType: 111, ssrc: 1),
.init(mid: "1", kind: "video", payloadType: nil, ssrc: 2)]).text
#expect(answer.contains("a=group:BUNDLE 0\r\n"))
#expect(answer.contains("m=video 0 UDP/TLS/RTP/SAVPF 0\r\n"))
}
}
@Suite struct RTPTests {
@Test func headerFields() {
var stream = RTPStream(ssrc: 0x0102_0304, payloadType: 111)
let first = stream.packet(timestamp: 960, marker: true, payload: [9, 9])
let second = stream.packet(timestamp: 1920, marker: false, payload: [])
#expect(first[0] == 0x80)
#expect(first[1] == 0x80 | 111)
#expect(Array(first[4..<12]) == [0, 0, 3, 0xC0, 1, 2, 3, 4])
#expect(Array(first[12...]) == [9, 9])
let seq1 = UInt16(first[2]) << 8 | UInt16(first[3])
let seq2 = UInt16(second[2]) << 8 | UInt16(second[3])
#expect(seq2 == seq1 &+ 1)
}
@Test func aggregatesParameterSetsAndKeepsSmallNALs() {
let sps: [UInt8] = [0x67, 1, 2, 3]
let pps: [UInt8] = [0x68, 4]
let idr: [UInt8] = [0x65] + Array(repeating: 7, count: 100)
let payloads = H264Packetizer.payloads(for: [sps, pps, idr])
#expect(payloads.count == 2)
#expect(payloads[0] == [0x78, 0, 4] + sps + [0, 2] + pps) // STAP-A with NRI 3
#expect(payloads[1] == idr)
}
@Test func fragmentsLargeNALAndReassembles() {
let nal: [UInt8] = [0x65] + (0..<5000).map { UInt8($0 & 0xFF) }
let payloads = H264Packetizer.payloads(for: [nal])
#expect(payloads.count > 1)
#expect(payloads.allSatisfy { $0.count <= H264Packetizer.maxPayload })
#expect(payloads.first?[1] == 0x85) // start bit + type 5
#expect(payloads.last?[1] == 0x45) // end bit + type 5
#expect(payloads.allSatisfy { $0[0] == 0x7C }) // FU-A, NRI 3
let rebuilt = [UInt8(0x65)] + payloads.flatMap { $0.dropFirst(2) }
#expect(rebuilt == nal)
}
@Test func splitsAVCC() {
let avcc: [UInt8] = [0, 0, 0, 2, 0x67, 1, 0, 0, 0, 3, 0x65, 2, 3]
#expect(H264Packetizer.nalUnits(avcc: avcc) == [[0x67, 1], [0x65, 2, 3]])
}
@Test func detectsKeyFrameRequests() {
let receiverReport: [UInt8] = [0x80, 201, 0, 1, 0, 0, 0, 1]
let pli: [UInt8] = [0x81, 206, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2]
#expect(RTCPFeedback.requestsKeyFrame(receiverReport + pli))
#expect(!RTCPFeedback.requestsKeyFrame(receiverReport))
#expect(RTCPFeedback.isRTCP(pli))
#expect(!RTCPFeedback.isRTCP([0x80, 111, 0, 1, 0, 0, 0, 0]))
}
}
Loading…
Cancel
Save

Powered by TurnKey Linux.