Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions native/ios-app/Poki/AppShortcuts.xcstrings
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,22 @@
}
}
}
},
"Toggle microphone mute in ${applicationName}" : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "切换${applicationName}的麦克风静音"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "切換${applicationName}的麥克風靜音"
}
}
}
}
},
"version" : "1.0"
Expand Down
1 change: 1 addition & 0 deletions native/ios-app/Poki/Chat/ChatView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,7 @@ struct ComposerInputView: View {
Group {
if let talk = voice.talkState {
TalkBar(state: talk, since: talkStarted, muted: voice.muted, toggleMute: { voice.muted.toggle() },
microphoneMuted: voice.microphone?.isMuted, toggleMicrophoneMute: { voice.toggleMicrophoneMute() },
interrupt: voice.stopTalking, end: voice.stop,
retry: { voice.dismissProblem(); openVoice() }, dismiss: voice.dismissProblem)
} else {
Expand Down
9 changes: 9 additions & 0 deletions native/ios-app/Poki/Chat/TalkViews.swift
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ struct TalkBar: View {
var since: Date?
var muted: Bool
var toggleMute: () -> Void
var microphoneMuted: Bool?
var toggleMicrophoneMute: () -> Void
var interrupt: () -> Void
var end: () -> Void
var retry: () -> Void
Expand Down Expand Up @@ -119,6 +121,9 @@ struct TalkBar: View {
if let since, !state.isFailed {
Text(since, style: .timer).font(.footnote).monospacedDigit().foregroundStyle(Color(.secondaryLabel))
}
if microphoneMuted == true {
Text("Microphone muted").font(.footnote).foregroundStyle(Color(.secondaryLabel))
}
}
.accessibilityElement(children: .combine)
Spacer(minLength: 0)
Expand All @@ -140,6 +145,10 @@ struct TalkBar: View {
round("hand.raised.fill", label: "Interrupt", identifier: "composer.talk.interrupt", action: interrupt)
.transition(.scale(scale: 0.6).combined(with: .opacity))
}
if let microphoneMuted {
round(microphoneMuted ? "mic.slash.fill" : "mic.fill", label: microphoneMuted ? "Unmute Microphone" : "Mute Microphone",
identifier: "composer.talk.microphone", action: toggleMicrophoneMute)
}
round(muted ? "speaker.slash.fill" : "speaker.wave.2.fill", label: muted ? "Unmute Poki’s Voice" : "Mute Poki’s Voice",
identifier: "composer.talk.mute", action: toggleMute)
// The send button's blue, in light and dark (the owner, 2026-11-13: not white on black).
Expand Down
1 change: 1 addition & 0 deletions native/ios-app/Poki/Debug/TalkRehearsal.swift
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ final class TalkRehearsal: VoiceConnection, @unchecked Sendable {
throw VoiceError.noBridge
}
func hasRemoteSpeech() async -> Bool { false }
func setMicrophoneMuted(_ muted: Bool) {}
func setRemoteVolume(_ value: Double) {}
func diagnose(_ write: @escaping @MainActor (String) -> Void) {}
func close() { script?.cancel() }
Expand Down
3 changes: 3 additions & 0 deletions native/ios-app/Poki/Entry/SendTaskIntent.swift
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ struct PokiShortcuts: AppShortcutsProvider {
AppShortcut(intent: TalkHereIntent(),
phrases: ["\(.applicationName) Anytime", "Talk to \(.applicationName) without leaving this app"],
shortTitle: LocalizedStringResource("Poki Anytime", table: "Entry"), systemImageName: "waveform.badge.mic")
AppShortcut(intent: ToggleMicrophoneMuteIntent(),
phrases: ["Toggle microphone mute in \(.applicationName)"],
shortTitle: LocalizedStringResource("Toggle Mic Mute", table: "Entry"), systemImageName: "mic.slash")
AppShortcut(intent: SendTaskToPokiIntent(),
phrases: ["Send a task to \(.applicationName)", "Give \(.applicationName) a task"],
shortTitle: LocalizedStringResource("Send Task", table: "Entry"), systemImageName: "paperplane")
Expand Down
24 changes: 24 additions & 0 deletions native/ios-app/Poki/Entry/ToggleMicrophoneMuteIntent.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import AppIntents

struct ToggleMicrophoneMuteIntent: AppIntent {
static let title = LocalizedStringResource("Toggle Microphone Mute", table: "Entry")
static let description = IntentDescription(LocalizedStringResource("Mutes or unmutes your microphone in the current Poki call.", table: "Entry"))
static let openAppWhenRun = false
@available(iOS 26.0, *)
static var supportedModes: IntentModes { .background }
static let authenticationPolicy: IntentAuthenticationPolicy = .requiresLocalDeviceAuthentication

@MainActor
func perform() async throws -> some IntentResult & ProvidesDialog {
let words: LocalizedStringResource
switch AskPoki.app?.voice.toggleMicrophoneMute() {
case .some(true):
words = LocalizedStringResource("Microphone muted.", table: "Entry")
case .some(false):
words = LocalizedStringResource("Microphone unmuted.", table: "Entry")
case nil:
words = LocalizedStringResource("There is no active call.", table: "Entry")
}
return .result(dialog: IntentDialog(words))
}
}
12 changes: 12 additions & 0 deletions native/ios-app/Poki/Voice/VoiceLink.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ protocol VoiceConnection: AnyObject, Sendable {
@MainActor func hasSentMicrophoneAudio() async -> Bool
func hasRemoteSpeech() async -> Bool
@discardableResult func send(_ event: [String: Any]) -> Bool
@MainActor func setMicrophoneMuted(_ muted: Bool)
func setRemoteVolume(_ value: Double)
func diagnose(_ write: @escaping @MainActor (String) -> Void)
@MainActor func close()
Expand Down Expand Up @@ -54,6 +55,8 @@ final class VoiceLink: NSObject, VoiceConnection, RTCPeerConnectionDelegate, RTC
private var peer: RTCPeerConnection?
private var channel: RTCDataChannel?
@MainActor private var holdsAudioSession = false
@MainActor private var microphoneTrack: RTCAudioTrack?
@MainActor private var microphoneMuted = false
/// WebRTC calls the delegate methods on its own threads and the app calls in from the main actor,
/// so these three are only touched under `lock`.
private let lock = NSLock()
Expand Down Expand Up @@ -120,6 +123,8 @@ final class VoiceLink: NSObject, VoiceConnection, RTCPeerConnectionDelegate, RTC
try session.setActive(true)
holdsAudioSession = true
let mic = factory.audioTrack(with: factory.audioSource(with: constraints), trackId: "poki-mic")
mic.isEnabled = !microphoneMuted
microphoneTrack = mic
peer.add(mic, streamIds: ["poki-voice"])
} else {
let receiveOnly = RTCRtpTransceiverInit()
Expand Down Expand Up @@ -228,6 +233,12 @@ final class VoiceLink: NSObject, VoiceConnection, RTCPeerConnectionDelegate, RTC
return channel.sendData(RTCDataBuffer(data: data, isBinary: false))
}

@MainActor
func setMicrophoneMuted(_ muted: Bool) {
microphoneMuted = muted
microphoneTrack?.isEnabled = !muted
}

/// 0 mutes Poki's voice (silent test runs); 1 is normal.
func setRemoteVolume(_ value: Double) {
lock.withLock {
Expand All @@ -244,6 +255,7 @@ final class VoiceLink: NSObject, VoiceConnection, RTCPeerConnectionDelegate, RTC
peer?.close()
channel = nil
peer = nil
microphoneTrack = nil
// An interruption can make the session inactive before we give up our activation.
guard holdsAudioSession else { return }
holdsAudioSession = false
Expand Down
33 changes: 33 additions & 0 deletions native/ios-app/Poki/Voice/VoiceModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,11 @@ struct VoiceClock {
@MainActor
@Observable
final class VoiceModel {
struct CallMicrophone {
let callID = UUID()
var isMuted = false
}

private(set) var bridge = VoiceBridge.saved
private(set) var phase = VoicePhase.off
private(set) var lines: [VoiceLine] = [] { didSet { saveTranscript() } }
Expand All @@ -306,6 +311,29 @@ final class VoiceModel {
#endif
/// Model audio off: for silent test runs. Speech is still transcribed into `lines`.
var muted = false { didSet { link?.setRemoteVolume(muted || liveSpeechStopped || liveCloseTimeout != nil ? 0 : 1) } }
private(set) var microphone: CallMicrophone?

@discardableResult
func setMicrophoneMuted(_ muted: Bool, callID: UUID) -> Bool {
guard let microphone, microphone.callID == callID else {
log.add("voice", "Ignored microphone mute for call \(callID): the call is no longer active")
return false
}
guard microphone.isMuted != muted else { return true }
self.microphone?.isMuted = muted
link?.setMicrophoneMuted(muted)
log.add("voice", "Microphone \(muted ? "muted" : "unmuted") for call \(callID), session \(session)")
return true
}

@discardableResult
func toggleMicrophoneMute() -> Bool? {
guard let microphone else { return nil }
let muted = !microphone.isMuted
setMicrophoneMuted(muted, callID: microphone.callID)
return muted
}

/// The last measured turn: from the end of speech to the first audio of the answer that carries a tool result.
private(set) var lastTurnMS: Int?
/// Timestamps of the current turn, from the end of the person's speech.
Expand Down Expand Up @@ -678,6 +706,7 @@ final class VoiceModel {
session += 1
PokiEntry.shared.advanceEpoch()
let number = session
self.microphone = microphone ? CallMicrophone() : nil
awaitingFirstLaunchUtterance = request
request?.record(.started, reason: "newTalk", session: number)
let bridge = self.bridge
Expand All @@ -694,6 +723,7 @@ final class VoiceModel {
return try await bridge.token(preferLive: microphone)
}
func connect(_ link: VoiceConnection, _ token: VoiceBridge.Token, direct: Bool) async throws {
link.setMicrophoneMuted(self.microphone?.isMuted ?? false)
usesLive = token.live == true
localTools = Set(token.local_tools)
if usesLive {
Expand Down Expand Up @@ -749,6 +779,7 @@ final class VoiceModel {
// Ended while connecting: stop() already put everything back.
guard number == session, liveCloseTimeout == nil else { return }
transcript = nil
self.microphone = nil
link?.close()
link = nil
holdingMicrophone = false
Expand Down Expand Up @@ -805,6 +836,7 @@ final class VoiceModel {
func stop() {
guard liveCloseTimeout == nil else { return }
PokiEntry.shared.advanceEpoch()
microphone = nil
if let dictation {
dictation.cancel()
self.dictation = nil
Expand Down Expand Up @@ -850,6 +882,7 @@ final class VoiceModel {
/// Drops the connection and the state of its turns; calls already running keep running. The lines of
/// the conversation are kept for one opened soon after, before the chat takes them.
private func endSession() {
microphone = nil
keepOpen()
inputReadyWatch?.cancel()
inputReadyWatch = nil
Expand Down
24 changes: 24 additions & 0 deletions native/ios-app/PokiTests/VoiceModelTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ final class FakeConnection: VoiceConnection, @unchecked Sendable {
}
func send(_ event: [String: Any]) -> Bool { sent.append(event); return true }
func setRemoteVolume(_ value: Double) { remoteVolume = value }
func setMicrophoneMuted(_ muted: Bool) {}
func hasRemoteSpeech() async -> Bool { await MainActor.run { remoteSpeech } }
func diagnose(_ write: @escaping @MainActor (String) -> Void) {}
func close() { closed = true }
Expand Down Expand Up @@ -155,6 +156,29 @@ private actor VoiceRunnerTransport: HTTPTransport {

@MainActor
final class VoiceModelTests: XCTestCase {
func testMicrophoneMuteIsScopedToTheActiveCall() async throws {
let h = Harness()
defer { h.voice.stop() }
await h.talk()
let callID = try XCTUnwrap(h.voice.microphone?.callID)

XCTAssertTrue(h.voice.setMicrophoneMuted(true, callID: callID))
XCTAssertTrue(h.voice.setMicrophoneMuted(true, callID: callID))
XCTAssertEqual(h.voice.microphone?.callID, callID)
XCTAssertEqual(h.voice.microphone?.isMuted, true)
XCTAssertEqual(h.link.remoteVolume, 1)

h.voice.stop()
XCTAssertNil(h.voice.microphone)
XCTAssertFalse(h.voice.setMicrophoneMuted(true, callID: callID))
XCTAssertTrue(AppLog.shared.lines.last?.contains("Ignored microphone mute for call \(callID): the call is no longer active") == true)

await h.talk()
XCTAssertNotEqual(h.voice.microphone?.callID, callID)
XCTAssertFalse(h.voice.setMicrophoneMuted(true, callID: callID))
XCTAssertEqual(h.voice.microphone?.isMuted, false)
}

func testAConnectionWhoseSessionNeverStartsEndsAfterSixtySeconds() async {
let h = Harness()
await h.connect()
Expand Down
96 changes: 96 additions & 0 deletions native/ios-app/Shared/Entry.xcstrings
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,102 @@
}
}
}
},
"Toggle Microphone Mute" : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "切换麦克风静音"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "切換麥克風靜音"
}
}
}
},
"Toggle Mic Mute" : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "切换麦克风静音"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "切換麥克風靜音"
}
}
}
},
"Mutes or unmutes your microphone in the current Poki call." : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "在当前 Poki 通话中将麦克风静音或取消静音。"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "在目前的 Poki 通話中將麥克風靜音或取消靜音。"
}
}
}
},
"Microphone muted." : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "麦克风已静音。"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "麥克風已靜音。"
}
}
}
},
"Microphone unmuted." : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "麦克风已取消静音。"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "麥克風已取消靜音。"
}
}
}
},
"There is no active call." : {
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
"value" : "没有进行中的通话。"
}
},
"zh-Hant" : {
"stringUnit" : {
"state" : "translated",
"value" : "沒有進行中的通話。"
}
}
}
}
},
"version" : "1.0"
Expand Down
Loading
Loading