Preview comes from the NDI proxy stream (lowest bandwidth); talkback taps the headset mic and is gated by the talk key. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>main
parent
1dc6b58d8a
commit
c61c543a65
@ -0,0 +1,155 @@
|
|||||||
|
import AudioMatrix
|
||||||
|
import Foundation
|
||||||
|
import Intercom
|
||||||
|
import MusubiCore
|
||||||
|
import NDIKit
|
||||||
|
import Observation
|
||||||
|
|
||||||
|
/// App-side glue for the intercom (control only): server on the clinic interface,
|
||||||
|
/// PGM preview from an NDI proxy stream, talkback from the headset mic gated by the talk key.
|
||||||
|
@MainActor @Observable
|
||||||
|
final class IntercomController {
|
||||||
|
let server = IntercomServer()
|
||||||
|
|
||||||
|
/// NDI source shown on the phones (usually PGM).
|
||||||
|
var previewSourceName: String {
|
||||||
|
didSet {
|
||||||
|
UserDefaults.standard.set(previewSourceName, forKey: "intercomPreviewSource")
|
||||||
|
startPreview()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Input device whose signal the phones hear while the talk key is down.
|
||||||
|
var talkbackDeviceUID: String {
|
||||||
|
didSet {
|
||||||
|
UserDefaults.standard.set(talkbackDeviceUID, forKey: "talkbackDeviceUID")
|
||||||
|
startTalkback()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private let settings: AppSettings
|
||||||
|
private let audio: AudioController
|
||||||
|
private var preview: NDIReceiver?
|
||||||
|
private var tap: (feed: AudioFeed, deviceUID: String)?
|
||||||
|
private var pump: TalkbackPump?
|
||||||
|
|
||||||
|
init(settings: AppSettings, audio: AudioController) {
|
||||||
|
self.settings = settings
|
||||||
|
self.audio = audio
|
||||||
|
previewSourceName = UserDefaults.standard.string(forKey: "intercomPreviewSource") ?? ""
|
||||||
|
talkbackDeviceUID = UserDefaults.standard.string(forKey: "talkbackDeviceUID") ?? ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var isEnabled: Bool { settings.role == .control && !settings.clinicInterfaceName.isEmpty }
|
||||||
|
|
||||||
|
/// Technical and clinic network on the same interface is fine at home, never on site.
|
||||||
|
var sharesTechnicalInterface: Bool { settings.clinicInterfaceName == settings.interfaceName }
|
||||||
|
|
||||||
|
/// (Re)starts or stops everything according to role and clinic interface; call after settings changes.
|
||||||
|
func restart() async {
|
||||||
|
stopAll()
|
||||||
|
guard isEnabled else { return }
|
||||||
|
await server.start(interfaceName: settings.clinicInterfaceName)
|
||||||
|
startPreview()
|
||||||
|
startTalkback()
|
||||||
|
}
|
||||||
|
|
||||||
|
func setTalkActive(_ active: Bool) {
|
||||||
|
pump?.gateOpen = active
|
||||||
|
}
|
||||||
|
|
||||||
|
private func stopAll() {
|
||||||
|
server.stop()
|
||||||
|
preview?.stop()
|
||||||
|
preview = nil
|
||||||
|
stopTalkback()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func startPreview() {
|
||||||
|
preview?.stop()
|
||||||
|
preview = nil
|
||||||
|
guard isEnabled, !previewSourceName.isEmpty else { return }
|
||||||
|
let input = server.input
|
||||||
|
preview = NDIReceiver(source: Source(ndiName: previewSourceName), receiverName: "Musubi Intercom",
|
||||||
|
bandwidth: .lowest, onFrame: { input.pushVideo($0) })
|
||||||
|
Log.intercom.info("Preview from \(self.previewSourceName, privacy: .public)")
|
||||||
|
}
|
||||||
|
|
||||||
|
private func startTalkback() {
|
||||||
|
stopTalkback()
|
||||||
|
guard isEnabled else { return }
|
||||||
|
let uid = talkbackDeviceUID.isEmpty ? defaultTalkbackDevice() : talkbackDeviceUID
|
||||||
|
guard let uid, audio.matrix.inputDevices.contains(where: { $0.uid == uid }) else { return }
|
||||||
|
let feed = audio.matrix.addInputTap(deviceUID: uid)
|
||||||
|
tap = (feed, uid)
|
||||||
|
let pump = TalkbackPump(feed: feed, input: server.input)
|
||||||
|
pump.gateOpen = audio.talkKey.isActive
|
||||||
|
pump.start()
|
||||||
|
self.pump = pump
|
||||||
|
}
|
||||||
|
|
||||||
|
private func stopTalkback() {
|
||||||
|
pump?.stop()
|
||||||
|
pump = nil
|
||||||
|
if let tap { audio.matrix.removeInputTap(tap.feed, deviceUID: tap.deviceUID) }
|
||||||
|
tap = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The mic already routed in the sound matrix, else the system's default input.
|
||||||
|
func defaultTalkbackDevice() -> String? {
|
||||||
|
for route in audio.matrix.routes {
|
||||||
|
if case .input(let uid) = route.source { return uid }
|
||||||
|
}
|
||||||
|
return AudioDevices.defaultInput()?.uid ?? audio.matrix.inputDevices.first?.uid
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Name of the device "Automatisch" currently resolves to.
|
||||||
|
var automaticTalkbackName: String? {
|
||||||
|
defaultTalkbackDevice().flatMap { uid in audio.matrix.inputDevices.first { $0.uid == uid }?.name }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Moves mic audio from the tap to the Opus encoder every 10 ms on its own queue.
|
||||||
|
/// Sends silence while the talk key is up, so the phones' audio clock keeps running.
|
||||||
|
private final class TalkbackPump: @unchecked Sendable {
|
||||||
|
private let feed: AudioFeed
|
||||||
|
private let input: IntercomInput
|
||||||
|
private let queue = DispatchQueue(label: "studio.zielke.musubi.intercom.talkback", qos: .userInitiated)
|
||||||
|
private var timer: DispatchSourceTimer?
|
||||||
|
private let lock = NSLock()
|
||||||
|
private var open = false
|
||||||
|
private var stereo = [Float](repeating: 0, count: 4096 * 2)
|
||||||
|
private var mono = [Float](repeating: 0, count: 4096)
|
||||||
|
|
||||||
|
init(feed: AudioFeed, input: IntercomInput) {
|
||||||
|
self.feed = feed
|
||||||
|
self.input = input
|
||||||
|
}
|
||||||
|
|
||||||
|
var gateOpen: Bool {
|
||||||
|
get { lock.withLock { open } }
|
||||||
|
set { lock.withLock { open = newValue } }
|
||||||
|
}
|
||||||
|
|
||||||
|
func start() {
|
||||||
|
let timer = DispatchSource.makeTimerSource(queue: queue)
|
||||||
|
timer.schedule(deadline: .now(), repeating: .milliseconds(10), leeway: .milliseconds(2))
|
||||||
|
timer.setEventHandler { [weak self] in self?.drain() }
|
||||||
|
timer.resume()
|
||||||
|
self.timer = timer
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
timer?.cancel()
|
||||||
|
timer = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
private func drain() {
|
||||||
|
let rate = feed.currentSampleRate
|
||||||
|
let frames = stereo.withUnsafeMutableBufferPointer { feed.read(into: $0.baseAddress!, frames: 4096) }
|
||||||
|
guard frames > 0, rate > 0 else { return }
|
||||||
|
let open = gateOpen
|
||||||
|
for i in 0..<frames { mono[i] = open ? stereo[i * 2] : 0 }
|
||||||
|
mono.withUnsafeBufferPointer { input.pushTalkback(UnsafeBufferPointer(rebasing: $0[0..<frames]), sampleRate: rate) }
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,163 @@
|
|||||||
|
import CoreImage.CIFilterBuiltins
|
||||||
|
import DesignSystem
|
||||||
|
import Intercom
|
||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
/// Intercom on the control: QR code for the phones, who is connected, what they see and hear.
|
||||||
|
struct IntercomView: View {
|
||||||
|
let intercom: IntercomController
|
||||||
|
let sourceList: SourceListModel
|
||||||
|
let audio: AudioController
|
||||||
|
@Environment(\.openSettings) private var openSettings
|
||||||
|
|
||||||
|
private var server: IntercomServer { intercom.server }
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
ScrollView {
|
||||||
|
VStack(alignment: .leading, spacing: Theme.Space.card) {
|
||||||
|
if !intercom.isEnabled {
|
||||||
|
VStack(alignment: .leading, spacing: Theme.Space.unit) {
|
||||||
|
Text("Intercom ist aus. Wähl in den Einstellungen das Klinik-Netz, dann verbinden sich Handys per QR-Code.")
|
||||||
|
.font(Theme.Font.body)
|
||||||
|
.foregroundStyle(Theme.Color.nezumi)
|
||||||
|
Button("Einstellungen öffnen") { openSettings() }
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if let error = server.error {
|
||||||
|
Text(error).font(Theme.Font.body).foregroundStyle(Theme.Color.kohaku)
|
||||||
|
}
|
||||||
|
if intercom.sharesTechnicalInterface {
|
||||||
|
Text("Technik- und Klinik-Netz laufen über dieselbe Schnittstelle. Zum Testen okay, im Einsatz müssen sie getrennt sein.")
|
||||||
|
.font(Theme.Font.caption)
|
||||||
|
.foregroundStyle(Theme.Color.kohaku)
|
||||||
|
}
|
||||||
|
HStack(alignment: .top, spacing: Theme.Space.card) {
|
||||||
|
joinCard
|
||||||
|
VStack(alignment: .leading, spacing: Theme.Space.card) {
|
||||||
|
sourcesCard
|
||||||
|
participantsCard
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.padding(Theme.Space.card)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
}
|
||||||
|
.background(Theme.Color.sumi)
|
||||||
|
}
|
||||||
|
|
||||||
|
private var joinCard: some View {
|
||||||
|
Card {
|
||||||
|
Text("Handy verbinden").font(Theme.Font.title).foregroundStyle(Theme.Color.washi)
|
||||||
|
if let url = server.url {
|
||||||
|
QRCode(text: url.absoluteString)
|
||||||
|
.frame(width: 200, height: 200)
|
||||||
|
Text(verbatim: url.absoluteString)
|
||||||
|
.font(Theme.Font.number)
|
||||||
|
.lineLimit(1)
|
||||||
|
.minimumScaleFactor(0.6)
|
||||||
|
.foregroundStyle(Theme.Color.washi)
|
||||||
|
.textSelection(.enabled)
|
||||||
|
Text("Handy ins Klinik-Netz, QR-Code scannen, Name und Rolle eintragen.")
|
||||||
|
.font(Theme.Font.caption)
|
||||||
|
.foregroundStyle(Theme.Color.nezumi)
|
||||||
|
.fixedSize(horizontal: false, vertical: true)
|
||||||
|
} else {
|
||||||
|
Text("Startet …").font(Theme.Font.body).foregroundStyle(Theme.Color.nezumi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.frame(width: 240)
|
||||||
|
}
|
||||||
|
|
||||||
|
private var sourcesCard: some View {
|
||||||
|
Card {
|
||||||
|
Picker("Vorschau", selection: Binding(get: { intercom.previewSourceName }, set: { intercom.previewSourceName = $0 })) {
|
||||||
|
Text("Keine").tag("")
|
||||||
|
ForEach(sourceList.sources, id: \.ndiName) { source in
|
||||||
|
Text(verbatim: source.ndiName).tag(source.ndiName)
|
||||||
|
}
|
||||||
|
if !intercom.previewSourceName.isEmpty, !sourceList.sources.contains(where: { $0.ndiName == intercom.previewSourceName }) {
|
||||||
|
Text(verbatim: "\(intercom.previewSourceName) · nicht gefunden").tag(intercom.previewSourceName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Picker("Talkback", selection: Binding(get: { intercom.talkbackDeviceUID }, set: { intercom.talkbackDeviceUID = $0 })) {
|
||||||
|
if let name = intercom.automaticTalkbackName {
|
||||||
|
Text("Automatisch · \(name)").tag("")
|
||||||
|
} else {
|
||||||
|
Text("Automatisch").tag("")
|
||||||
|
}
|
||||||
|
ForEach(audio.matrix.inputDevices) { device in
|
||||||
|
Text(verbatim: device.name).tag(device.uid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Text("Handys hören das Mikro, solange die Sprechtaste (\(audio.talkKey.displayName)) gedrückt ist.")
|
||||||
|
.font(Theme.Font.caption)
|
||||||
|
.foregroundStyle(Theme.Color.nezumi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private var participantsCard: some View {
|
||||||
|
Card {
|
||||||
|
HStack {
|
||||||
|
Text("Verbunden").font(Theme.Font.title).foregroundStyle(Theme.Color.washi)
|
||||||
|
Spacer()
|
||||||
|
Text(verbatim: "\(server.participants.filter(\.connected).count)")
|
||||||
|
.font(Theme.Font.number)
|
||||||
|
.foregroundStyle(Theme.Color.nezumi)
|
||||||
|
}
|
||||||
|
if server.participants.isEmpty {
|
||||||
|
Text("Noch niemand da. Scann den QR-Code mit einem Handy.")
|
||||||
|
.font(Theme.Font.body)
|
||||||
|
.foregroundStyle(Theme.Color.nezumi)
|
||||||
|
}
|
||||||
|
ForEach(server.participants) { participant in
|
||||||
|
HStack(spacing: Theme.Space.unit) {
|
||||||
|
Circle()
|
||||||
|
.fill(participant.connected ? Theme.Color.moegi : Theme.Color.kohaku)
|
||||||
|
.frame(width: 8, height: 8)
|
||||||
|
Text(verbatim: participant.name).font(Theme.Font.body).foregroundStyle(Theme.Color.washi)
|
||||||
|
Text(verbatim: participant.role).font(Theme.Font.caption).foregroundStyle(Theme.Color.nezumi)
|
||||||
|
Spacer()
|
||||||
|
if !participant.connected {
|
||||||
|
Text("verbindet …").font(Theme.Font.caption).foregroundStyle(Theme.Color.nezumi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct Card<Content: View>: View {
|
||||||
|
@ViewBuilder let content: Content
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
VStack(alignment: .leading, spacing: Theme.Space.unit) { content }
|
||||||
|
.padding(Theme.Space.card)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
.background(Theme.Color.kasumi, in: RoundedRectangle(cornerRadius: Theme.Radius.card))
|
||||||
|
.overlay(RoundedRectangle(cornerRadius: Theme.Radius.card).stroke(Theme.Color.usuzumi, lineWidth: Theme.Stroke.line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Black on white with a quiet zone: what phone cameras read best, regardless of the dark theme.
|
||||||
|
private struct QRCode: View {
|
||||||
|
let text: String
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
if let image = Self.image(for: text) {
|
||||||
|
Image(decorative: image, scale: 1)
|
||||||
|
.interpolation(.none)
|
||||||
|
.resizable()
|
||||||
|
.padding(Theme.Space.unit)
|
||||||
|
.background(.white, in: RoundedRectangle(cornerRadius: Theme.Radius.button))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static func image(for text: String) -> CGImage? {
|
||||||
|
let filter = CIFilter.qrCodeGenerator()
|
||||||
|
filter.message = Data(text.utf8)
|
||||||
|
filter.correctionLevel = "M"
|
||||||
|
guard let output = filter.outputImage else { return nil }
|
||||||
|
return CIContext().createCGImage(output, from: output.extent)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue