feat(app): add crash visibility, memory relief, and clipboard reply work
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,9 @@
|
||||
// Wire models for the host-private OSG account protocol.
|
||||
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public struct AccountTokenSession: Codable, Equatable, Sendable {
|
||||
public let accountId: UUID
|
||||
@@ -285,23 +288,23 @@ extension AccountAPIError: LocalizedError {
|
||||
.rateLimited(let message):
|
||||
return message
|
||||
case .refreshTokenReuse:
|
||||
return "The session was revoked because refresh-token reuse was detected."
|
||||
return SharedL10n.string("error.account.refreshTokenReuse")
|
||||
case .server(_, _, let message):
|
||||
return message
|
||||
case .transport:
|
||||
return "The account service could not be reached."
|
||||
return SharedL10n.string("error.account.transport")
|
||||
case .invalidResponse:
|
||||
return "The account service returned an invalid response."
|
||||
return SharedL10n.string("error.account.invalidResponse")
|
||||
case .decoding:
|
||||
return "The account service response could not be decoded."
|
||||
return SharedL10n.string("error.account.decoding")
|
||||
case .secureStorage:
|
||||
return "The private account session could not be stored securely."
|
||||
return SharedL10n.string("error.account.secureStorage")
|
||||
case .sessionUnavailable:
|
||||
return "No account session is available."
|
||||
return SharedL10n.string("error.account.sessionUnavailable")
|
||||
case .appleAuthorization:
|
||||
return "Sign in with Apple did not return valid credentials."
|
||||
return SharedL10n.string("error.account.appleAuthorization")
|
||||
case .integrityUnavailable:
|
||||
return "Device integrity verification is unavailable."
|
||||
return SharedL10n.string("error.account.integrityUnavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,15 +450,16 @@ struct PromptCloudASRClient: CloudASRTranscribing {
|
||||
let urlString = "\(trimmedBase)/chat/completions"
|
||||
guard let url = URL(string: urlString) else { throw CloudASRError.invalidURL }
|
||||
|
||||
var userContent: [[String: Any]] = []
|
||||
let prompt = dictionary.asrPromptBias()
|
||||
if !prompt.isEmpty {
|
||||
userContent.append(["type": "text", "text": prompt])
|
||||
}
|
||||
userContent.append([
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI]
|
||||
])
|
||||
// MiMo-V2.5-ASR is an end-to-end ASR model: the `content` array must
|
||||
// hold the single `input_audio` part and nothing else. Adding a text
|
||||
// prompt (personal-dictionary bias) makes the API reject the request
|
||||
// with HTTP 400 "param incorrect", so we never send prompt bias here.
|
||||
let userContent: [[String: Any]] = [
|
||||
[
|
||||
"type": "input_audio",
|
||||
"input_audio": ["data": dataURI]
|
||||
]
|
||||
]
|
||||
|
||||
let body: [String: Any] = [
|
||||
"model": model,
|
||||
|
||||
@@ -9,6 +9,13 @@ import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum CloudASRConnectionCheck {
|
||||
/// Overall wall-clock budget for a connection check. The probe requests set
|
||||
/// `URLRequest.timeoutInterval` (~90 s), but that is an *idle* timeout that
|
||||
/// resets on every byte and never fires when a provider accepts the socket
|
||||
/// then stalls or trickles a keep-alive — leaving the settings spinner
|
||||
/// turning forever. This hard deadline guarantees the check returns.
|
||||
public static let probeTimeout: TimeInterval = 30
|
||||
|
||||
/// Verifies the active cloud ASR client can connect + authenticate.
|
||||
///
|
||||
/// Each backend decides how to probe (see `CloudASRTranscribing`):
|
||||
@@ -17,6 +24,12 @@ public enum CloudASRConnectionCheck {
|
||||
/// handshake (and auth) only — silence clips make streaming backends fail.
|
||||
public static func validate(store: any ConfigurationStore) async throws {
|
||||
let client = CloudASRClientFactory.make(store: store)
|
||||
try await client.probeConnection()
|
||||
do {
|
||||
try await HardTimeout.run(seconds: probeTimeout) {
|
||||
try await client.probeConnection()
|
||||
}
|
||||
} catch is HardTimeoutError {
|
||||
throw CloudASRError.timedOut
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,8 @@ enum CloudASRLogMetadata {
|
||||
return "category=audioTooLong"
|
||||
case .providerUnsupported:
|
||||
return "category=providerUnsupported"
|
||||
case .timedOut:
|
||||
return "category=timedOut"
|
||||
}
|
||||
}
|
||||
if let urlError = error as? URLError {
|
||||
|
||||
@@ -439,15 +439,15 @@ public final class CustomLanguageModelManager: @unchecked Sendable {
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missingManifest:
|
||||
return "Missing bundled custom language model manifest."
|
||||
return SharedL10n.string("error.clm.missingManifest")
|
||||
case .missingTrainingAsset:
|
||||
return "Missing bundled custom language model training asset."
|
||||
return SharedL10n.string("error.clm.missingTrainingAsset")
|
||||
case .missingAppGroupContainer:
|
||||
return "App Group container unavailable for custom language model preparation."
|
||||
return SharedL10n.string("error.clm.missingAppGroupContainer")
|
||||
case .missingPreparedArtifacts:
|
||||
return "Custom language model preparation did not produce LM/Vocab artifacts."
|
||||
return SharedL10n.string("error.clm.missingPreparedArtifacts")
|
||||
case .retryBudgetExhausted:
|
||||
return "Custom language model preparation retry budget exhausted."
|
||||
return SharedL10n.string("error.clm.retryBudgetExhausted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
// FlowAdaptiveDownsampler.swift
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Shared route-adaptive resampler for every realtime `installTap` consumer
|
||||
// (`FlowContinuousCapture`, `LiveDictationController`, and the Mac recorder's
|
||||
// iOS sibling paths). Extracted so the crash-safe tap pattern lives in exactly
|
||||
// one place: a second copy of this logic is how the preview surface ended up
|
||||
// still installing taps with a stale explicit format.
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import os
|
||||
|
||||
/// Why a tap buffer never reached the recogniser.
|
||||
///
|
||||
/// Recorded as a plain integer on the realtime audio thread and rendered on the
|
||||
/// main actor — calling `Logger` inside the tap would allocate and risk
|
||||
/// priority inversion. Each of these was previously a bare `return`, which is
|
||||
/// what made "waveform moves but the transcript is empty" invisible: levels and
|
||||
/// the audio-proof timestamp are taken from the *raw* buffer, before
|
||||
/// conversion, so they keep looking healthy while ASR receives nothing.
|
||||
public enum FlowDownsampleFailure: Int, Sendable {
|
||||
case none = 0
|
||||
case invalidSourceFormat
|
||||
case converterCreateFailed
|
||||
case scratchOverflow
|
||||
case converterError
|
||||
case emptyOutput
|
||||
|
||||
public var label: String {
|
||||
switch self {
|
||||
case .none: return "none"
|
||||
case .invalidSourceFormat: return "invalidSourceFormat"
|
||||
case .converterCreateFailed: return "converterCreateFailed"
|
||||
case .scratchOverflow: return "scratchOverflow"
|
||||
case .converterError: return "converterError"
|
||||
case .emptyOutput: return "emptyOutput"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of one realtime conversion attempt. Carries the reason (and the
|
||||
/// formats involved) so the drop can be explained after the fact.
|
||||
enum FlowDownsampleOutcome {
|
||||
case converted(AVAudioPCMBuffer)
|
||||
case failed(
|
||||
failure: FlowDownsampleFailure,
|
||||
sourceRate: Double,
|
||||
inputFrames: Int,
|
||||
wantedFrames: Int
|
||||
)
|
||||
}
|
||||
|
||||
/// Route-adaptive downsampling converter, safe to call from the realtime tap.
|
||||
///
|
||||
/// `AVAudioEngine.installTap(format:)` traps with an **uncatchable** NSException
|
||||
/// when the format passed to it does not match the input node's *live* format.
|
||||
/// After an audio-route change — which the on-device `SpeechAnalyzer` triggers
|
||||
/// during warmup by reconfiguring the shared `AVAudioSession`, and which any
|
||||
/// Bluetooth/headset swap triggers at will — the value returned by
|
||||
/// `inputNode.outputFormat(forBus:)` can lag behind the real hardware rate
|
||||
/// (e.g. it reports 48 kHz while the node has already switched to 24 kHz).
|
||||
/// Installing a tap with that stale explicit format crashes the whole app
|
||||
/// (`Failed to create tap due to format mismatch`).
|
||||
///
|
||||
/// Every caller must therefore install the tap with `format: nil` (which always
|
||||
/// uses the node's live format) and rebuild the sample-rate converter *here*
|
||||
/// whenever the incoming buffer's format actually changes, so downsampling to
|
||||
/// the ASR target rate is always valid regardless of route churn.
|
||||
final class FlowAdaptiveDownsampler: @unchecked Sendable {
|
||||
// `AVAudioConverter` / `AVAudioFormat` / `AVAudioPCMBuffer` are not
|
||||
// `Sendable`, so the state is guarded manually via the unchecked lock
|
||||
// APIs. The scratch output buffer is REUSED across tap callbacks —
|
||||
// allocating on the realtime audio thread risks priority inversion, and
|
||||
// taps on one bus are serialized, so a single scratch is safe as long as
|
||||
// callers copy its contents out before returning (AudioBufferSnapshot
|
||||
// does exactly that).
|
||||
private struct State {
|
||||
var converter: AVAudioConverter
|
||||
var source: AVAudioFormat
|
||||
var scratch: AVAudioPCMBuffer
|
||||
}
|
||||
|
||||
private let lock = OSAllocatedUnfairLock<State?>(uncheckedState: nil)
|
||||
let targetFormat: AVAudioFormat
|
||||
|
||||
/// Starting headroom for the reusable output buffer. Taps normally deliver
|
||||
/// ≤4096 input frames; output frames = input × (16k / hardwareRate), which
|
||||
/// exceeds input only for sub-16 kHz hardware (rare telephony routes), so
|
||||
/// 2× the requested tap size covers the common case without allocating.
|
||||
private static let initialScratchCapacity: AVAudioFrameCount = 8_192
|
||||
|
||||
/// Ceiling for one-time growth. `bufferSize:` is a hint — macOS in
|
||||
/// particular hands pro audio interfaces larger slices than requested — and
|
||||
/// silently dropping those frames would be a worse bug than the allocation.
|
||||
/// Anything past this is a nonsense ratio, not a real route.
|
||||
private static let maxScratchCapacity: AVAudioFrameCount = 32_768
|
||||
|
||||
init(targetFormat: AVAudioFormat) {
|
||||
self.targetFormat = targetFormat
|
||||
}
|
||||
|
||||
/// Downsamples `buffer` into the reusable scratch buffer and returns it,
|
||||
/// rebuilding the converter lazily when the hardware route (and thus the
|
||||
/// source format) changes. The returned buffer is only valid until the
|
||||
/// next call — copy its samples out synchronously.
|
||||
func convertReusingScratch(_ buffer: AVAudioPCMBuffer) -> FlowDownsampleOutcome {
|
||||
let sourceFormat = buffer.format
|
||||
let sourceRate = sourceFormat.sampleRate
|
||||
let inputFrames = Int(buffer.frameLength)
|
||||
guard sourceRate > 0 else {
|
||||
return .failed(
|
||||
failure: .invalidSourceFormat,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
return lock.withLockUnchecked { state -> FlowDownsampleOutcome in
|
||||
if state == nil || state!.source != sourceFormat {
|
||||
guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat),
|
||||
let scratch = AVAudioPCMBuffer(
|
||||
pcmFormat: targetFormat,
|
||||
frameCapacity: Self.initialScratchCapacity
|
||||
) else {
|
||||
state = nil
|
||||
return .failed(
|
||||
failure: .converterCreateFailed,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
state = State(converter: converter, source: sourceFormat, scratch: scratch)
|
||||
}
|
||||
guard var current = state else {
|
||||
return .failed(
|
||||
failure: .converterCreateFailed,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
|
||||
// Stay in `Double` until the range is proven: `AVAudioFrameCount(_:)`
|
||||
// traps on NaN / infinite / out-of-range input, and a degenerate
|
||||
// route (sub-1 Hz `sourceRate`) would otherwise crash the realtime
|
||||
// render thread instead of dropping one frame.
|
||||
let wantedExact = Double(buffer.frameLength) * targetFormat.sampleRate / sourceRate
|
||||
let ceiling = Double(Self.maxScratchCapacity)
|
||||
guard wantedExact.isFinite, wantedExact >= 1, wantedExact <= ceiling else {
|
||||
return .failed(
|
||||
failure: .scratchOverflow,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: wantedExact.isFinite ? Int(min(wantedExact, ceiling)) : 0
|
||||
)
|
||||
}
|
||||
let wanted = AVAudioFrameCount(wantedExact)
|
||||
|
||||
if wanted > current.scratch.frameCapacity {
|
||||
// Grow once, with headroom, rather than dropping audio for the
|
||||
// rest of the session. This costs an allocation on the realtime
|
||||
// thread, which is exactly what the reusable scratch exists to
|
||||
// avoid — but it happens at most a couple of times per route,
|
||||
// the same as the converter rebuild above, and losing the audio
|
||||
// is the worse trade.
|
||||
let grownCapacity = min(
|
||||
max(wanted &* 2, Self.initialScratchCapacity),
|
||||
Self.maxScratchCapacity
|
||||
)
|
||||
guard let grown = AVAudioPCMBuffer(
|
||||
pcmFormat: targetFormat,
|
||||
frameCapacity: grownCapacity
|
||||
) else {
|
||||
return .failed(
|
||||
failure: .scratchOverflow,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
current.scratch = grown
|
||||
state = current
|
||||
}
|
||||
current.scratch.frameLength = 0
|
||||
|
||||
// ONE-SHOT input: the converter keeps pulling until the output
|
||||
// buffer's frameCapacity is full, and the scratch is deliberately
|
||||
// oversized — feeding the same tap buffer on every pull would
|
||||
// duplicate the audio ~6× (stuttering ASR input). After the
|
||||
// single feed we report "ran dry", so the expected status is
|
||||
// `.inputRanDry` (output not full), not `.haveData`.
|
||||
let provided = OSAllocatedUnfairLock(initialState: false)
|
||||
var error: NSError?
|
||||
let status = current.converter.convert(to: current.scratch, error: &error) { _, outStatus in
|
||||
if provided.withLock({ $0 }) {
|
||||
outStatus.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
provided.withLock { $0 = true }
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status != .error, error == nil else {
|
||||
return .failed(
|
||||
failure: .converterError,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
guard current.scratch.frameLength > 0 else {
|
||||
return .failed(
|
||||
failure: .emptyOutput,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
return .converted(current.scratch)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,33 +169,9 @@ private final class FlowAudioProofStore: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a tap buffer never reached the recogniser.
|
||||
///
|
||||
/// Recorded as a plain integer on the realtime audio thread and rendered on the
|
||||
/// main actor — calling `Logger` inside the tap would allocate and risk
|
||||
/// priority inversion. Each of these was previously a bare `return`, which is
|
||||
/// what made "waveform moves but the transcript is empty" invisible: levels and
|
||||
/// the audio-proof timestamp are taken from the *raw* buffer, before
|
||||
/// conversion, so they keep looking healthy while ASR receives nothing.
|
||||
public enum FlowDownsampleFailure: Int, Sendable {
|
||||
case none = 0
|
||||
case invalidSourceFormat
|
||||
case converterCreateFailed
|
||||
case scratchOverflow
|
||||
case converterError
|
||||
case emptyOutput
|
||||
// `FlowDownsampleFailure` lives in `FlowAdaptiveDownsampler.swift` next to
|
||||
// the resampler that produces it.
|
||||
|
||||
public var label: String {
|
||||
switch self {
|
||||
case .none: return "none"
|
||||
case .invalidSourceFormat: return "invalidSourceFormat"
|
||||
case .converterCreateFailed: return "converterCreateFailed"
|
||||
case .scratchOverflow: return "scratchOverflow"
|
||||
case .converterError: return "converterError"
|
||||
case .emptyOutput: return "emptyOutput"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tap accounting for one utterance (`beginUtterance()` resets it).
|
||||
public struct FlowCaptureFrameReport: Sendable, Equatable {
|
||||
@@ -275,152 +251,10 @@ private final class FlowCaptureFrameStats: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of one realtime conversion attempt. Carries the reason (and the
|
||||
/// formats involved) so the drop can be explained after the fact.
|
||||
private enum FlowDownsampleOutcome {
|
||||
case converted(AVAudioPCMBuffer)
|
||||
case failed(
|
||||
failure: FlowDownsampleFailure,
|
||||
sourceRate: Double,
|
||||
inputFrames: Int,
|
||||
wantedFrames: Int
|
||||
)
|
||||
}
|
||||
// `FlowDownsampleOutcome` and the route-adaptive resampler now live in
|
||||
// `FlowAdaptiveDownsampler.swift` so every realtime tap consumer shares one
|
||||
// crash-safe implementation.
|
||||
|
||||
/// Route-adaptive downsampling converter, safe to call from the realtime tap.
|
||||
///
|
||||
/// `AVAudioEngine.installTap(format:)` traps with an **uncatchable** NSException
|
||||
/// when the format passed to it does not match the input node's *live* format.
|
||||
/// After an audio-route change — which the on-device `SpeechAnalyzer` triggers
|
||||
/// during warmup by reconfiguring the shared `AVAudioSession` — the value
|
||||
/// returned by `inputNode.outputFormat(forBus:)` can lag behind the real
|
||||
/// hardware rate (e.g. it reports 48 kHz while the node has already switched to
|
||||
/// 24 kHz). Installing a tap with that stale explicit format crashes the whole
|
||||
/// app (`Failed to create tap due to format mismatch`).
|
||||
///
|
||||
/// We therefore install the tap with `format: nil` (which always uses the
|
||||
/// node's live format) and rebuild the sample-rate converter *here* whenever the
|
||||
/// incoming buffer's format actually changes, so downsampling to the ASR target
|
||||
/// rate is always valid regardless of route churn.
|
||||
private final class AdaptiveDownsampler: @unchecked Sendable {
|
||||
// `AVAudioConverter` / `AVAudioFormat` / `AVAudioPCMBuffer` are not
|
||||
// `Sendable`, so the state is guarded manually via the unchecked lock
|
||||
// APIs. The scratch output buffer is REUSED across tap callbacks —
|
||||
// allocating on the realtime audio thread risks priority inversion, and
|
||||
// taps on one bus are serialized, so a single scratch is safe as long as
|
||||
// callers copy its contents out before returning (AudioBufferSnapshot
|
||||
// does exactly that).
|
||||
private struct State {
|
||||
var converter: AVAudioConverter
|
||||
var source: AVAudioFormat
|
||||
var scratch: AVAudioPCMBuffer
|
||||
}
|
||||
|
||||
private let lock = OSAllocatedUnfairLock<State?>(uncheckedState: nil)
|
||||
let targetFormat: AVAudioFormat
|
||||
|
||||
/// Frame headroom for the reusable output buffer. Taps deliver ≤4096
|
||||
/// input frames; output frames = input × (16k / hardwareRate), which
|
||||
/// exceeds input only for sub-16 kHz hardware (rare telephony routes),
|
||||
/// so 2× the tap size covers every realistic ratio.
|
||||
private static let scratchCapacity: AVAudioFrameCount = 8_192
|
||||
|
||||
init(targetFormat: AVAudioFormat) {
|
||||
self.targetFormat = targetFormat
|
||||
}
|
||||
|
||||
/// Downsamples `buffer` into the reusable scratch buffer and returns it,
|
||||
/// rebuilding the converter lazily when the hardware route (and thus the
|
||||
/// source format) changes. The returned buffer is only valid until the
|
||||
/// next call — copy its samples out synchronously.
|
||||
func convertReusingScratch(_ buffer: AVAudioPCMBuffer) -> FlowDownsampleOutcome {
|
||||
let sourceFormat = buffer.format
|
||||
let sourceRate = sourceFormat.sampleRate
|
||||
let inputFrames = Int(buffer.frameLength)
|
||||
guard sourceRate > 0 else {
|
||||
return .failed(
|
||||
failure: .invalidSourceFormat,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
return lock.withLockUnchecked { state -> FlowDownsampleOutcome in
|
||||
if state == nil || state!.source != sourceFormat {
|
||||
guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat),
|
||||
let scratch = AVAudioPCMBuffer(
|
||||
pcmFormat: targetFormat,
|
||||
frameCapacity: Self.scratchCapacity
|
||||
) else {
|
||||
state = nil
|
||||
return .failed(
|
||||
failure: .converterCreateFailed,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
state = State(converter: converter, source: sourceFormat, scratch: scratch)
|
||||
}
|
||||
guard let current = state else {
|
||||
return .failed(
|
||||
failure: .converterCreateFailed,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: 0
|
||||
)
|
||||
}
|
||||
|
||||
let wanted = AVAudioFrameCount(
|
||||
Double(buffer.frameLength) * targetFormat.sampleRate / sourceRate
|
||||
)
|
||||
guard wanted > 0, wanted <= current.scratch.frameCapacity else {
|
||||
return .failed(
|
||||
failure: .scratchOverflow,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
current.scratch.frameLength = 0
|
||||
|
||||
// ONE-SHOT input: the converter keeps pulling until the output
|
||||
// buffer's frameCapacity is full, and the scratch is deliberately
|
||||
// oversized — feeding the same tap buffer on every pull would
|
||||
// duplicate the audio ~6× (stuttering ASR input). After the
|
||||
// single feed we report "ran dry", so the expected status is
|
||||
// `.inputRanDry` (output not full), not `.haveData`.
|
||||
let provided = OSAllocatedUnfairLock(initialState: false)
|
||||
var error: NSError?
|
||||
let status = current.converter.convert(to: current.scratch, error: &error) { _, outStatus in
|
||||
if provided.withLock({ $0 }) {
|
||||
outStatus.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
provided.withLock { $0 = true }
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status != .error, error == nil else {
|
||||
return .failed(
|
||||
failure: .converterError,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
guard current.scratch.frameLength > 0 else {
|
||||
return .failed(
|
||||
failure: .emptyOutput,
|
||||
sourceRate: sourceRate,
|
||||
inputFrames: inputFrames,
|
||||
wantedFrames: Int(wanted)
|
||||
)
|
||||
}
|
||||
return .converted(current.scratch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the session-long capture graph on the main actor. The realtime tap
|
||||
/// must not touch UserDefaults, log, or invoke actor callbacks; it exchanges
|
||||
@@ -478,7 +312,7 @@ public final class FlowContinuousCapture {
|
||||
)
|
||||
private let frameStats = FlowCaptureFrameStats()
|
||||
|
||||
private var downsampler: AdaptiveDownsampler?
|
||||
private var downsampler: FlowAdaptiveDownsampler?
|
||||
private var targetFormat: AVAudioFormat?
|
||||
private var hwFormat: AVAudioFormat?
|
||||
private var activeRouteSnapshot: FlowAudioSessionSnapshot?
|
||||
@@ -676,7 +510,7 @@ public final class FlowContinuousCapture {
|
||||
|
||||
// Route-adaptive converter: it rebuilds itself from the live buffer
|
||||
// format inside the tap, so it never assumes a fixed hardware rate.
|
||||
let downsampler = AdaptiveDownsampler(targetFormat: resolvedTargetFormat)
|
||||
let downsampler = FlowAdaptiveDownsampler(targetFormat: resolvedTargetFormat)
|
||||
self.downsampler = downsampler
|
||||
targetFormat = resolvedTargetFormat
|
||||
hwFormat = hardwareFormat
|
||||
@@ -1166,7 +1000,7 @@ public final class FlowContinuousCapture {
|
||||
// MARK: - Audio tap (nonisolated — runs on realtime thread)
|
||||
|
||||
private nonisolated static func makeAudioTapBlock(
|
||||
downsampler: AdaptiveDownsampler,
|
||||
downsampler: FlowAdaptiveDownsampler,
|
||||
gate: OSAllocatedUnfairLock<UtteranceGatePhase>,
|
||||
levelStore: FlowLevelStore,
|
||||
audioProofStore: FlowAudioProofStore,
|
||||
|
||||
@@ -98,9 +98,11 @@ public final class LiveDictationController: ObservableObject {
|
||||
private var chunkedPipeline: ChunkedUtterancePipeline?
|
||||
private let captureGate = OSAllocatedUnfairLock(initialState: LiveCaptureGatePhase.idle)
|
||||
private let drainTracker = FlowCaptureDrainTracker()
|
||||
private var audioConverter: AVAudioConverter?
|
||||
private var targetFormat: AVAudioFormat?
|
||||
private var hwFormat: AVAudioFormat?
|
||||
/// Route-adaptive resampler shared with the Flow capture path. Rebuilt
|
||||
/// lazily whenever the live input format changes, so a headset/Bluetooth
|
||||
/// swap mid-session downsamples correctly instead of feeding the converter
|
||||
/// a stale rate.
|
||||
private var downsampler: FlowAdaptiveDownsampler?
|
||||
private var didConfigureAudioSession = false
|
||||
private var didInstallTap = false
|
||||
|
||||
@@ -273,16 +275,20 @@ public final class LiveDictationController: ObservableObject {
|
||||
let hwFormat = inputNode.inputFormat(forBus: 0)
|
||||
|
||||
// Pre-flight check: a placeholder / unconfigured input bus
|
||||
// reports `sampleRate == 0` (or `channelCount == 0`).
|
||||
// `installTap` on such a bus traps with "Failed to create
|
||||
// tap due to format mismatch" (an NSException, not a Swift
|
||||
// `Error`, so we can't `try`/`catch` it). The safest fix
|
||||
// is to refuse the tap up front and surface a clear
|
||||
// `.error` phase instead of crashing the app. We've seen
|
||||
// this on the iOS Simulator when the host's microphone
|
||||
// permission isn't granted to CoreSimulator, and on
|
||||
// devices where the audio session is in an unexpected
|
||||
// state from a previous foreground/background transition.
|
||||
// reports `sampleRate == 0` (or `channelCount == 0`), and no
|
||||
// amount of adaptive resampling downstream makes a dead bus
|
||||
// usable — `installTap` on one raises "Failed to create tap
|
||||
// due to format mismatch", an NSException rather than a Swift
|
||||
// `Error`, so we can't `try`/`catch` it. Refuse up front and
|
||||
// surface a clear `.error` phase instead of crashing. We've
|
||||
// seen this on the iOS Simulator when the host's microphone
|
||||
// permission isn't granted to CoreSimulator, and on devices
|
||||
// where the audio session is in an unexpected state from a
|
||||
// previous foreground/background transition.
|
||||
//
|
||||
// The *other* half of that crash — a format that was valid
|
||||
// here but stale by the time the tap is installed — is handled
|
||||
// by passing `format: nil` below rather than by this check.
|
||||
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else {
|
||||
debug("invalid hardware format sr=\(hwFormat.sampleRate) ch=\(hwFormat.channelCount)")
|
||||
phase = .error(
|
||||
@@ -305,15 +311,18 @@ public final class LiveDictationController: ObservableObject {
|
||||
phase = .error(NSLocalizedString("preview.error.formatCreate", comment: ""))
|
||||
return
|
||||
}
|
||||
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else {
|
||||
// The converter itself is built lazily by the downsampler from the
|
||||
// format of the *first buffer the tap actually delivers*, so a route
|
||||
// change between here and the first callback cannot desync it. Probe
|
||||
// once up front purely to fail fast with a clear message.
|
||||
guard AVAudioConverter(from: hwFormat, to: targetFormat) != nil else {
|
||||
debug("converter creation failed")
|
||||
phase = .error(NSLocalizedString("preview.error.converterCreate", comment: ""))
|
||||
return
|
||||
}
|
||||
|
||||
audioConverter = converter
|
||||
self.targetFormat = targetFormat
|
||||
self.hwFormat = hwFormat
|
||||
let downsampler = FlowAdaptiveDownsampler(targetFormat: targetFormat)
|
||||
self.downsampler = downsampler
|
||||
drainTracker.reset()
|
||||
captureGate.withLock { $0 = .recording }
|
||||
|
||||
@@ -351,13 +360,17 @@ public final class LiveDictationController: ObservableObject {
|
||||
}
|
||||
}
|
||||
let tap = Self.makeAudioTapBlock(
|
||||
converter: converter,
|
||||
targetFormat: targetFormat,
|
||||
hwFormat: hwFormat,
|
||||
downsampler: downsampler,
|
||||
onMeter: onMeter,
|
||||
onSnapshot: onSnapshot
|
||||
)
|
||||
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
|
||||
// `format: nil` — NEVER pass an explicit format here. `installTap`
|
||||
// raises an uncatchable NSException ("Failed to create tap due to
|
||||
// format mismatch") when the format we captured above has already been
|
||||
// superseded by a route change, and that kills the whole app. `nil`
|
||||
// always resolves to the node's live format; `FlowAdaptiveDownsampler`
|
||||
// absorbs whatever rate actually arrives.
|
||||
inputNode.installTap(onBus: 0, bufferSize: 4096, format: nil, block: tap)
|
||||
didInstallTap = true
|
||||
|
||||
audioEngine.prepare()
|
||||
@@ -489,9 +502,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
// main actor via `Task { @MainActor in … }`, which is itself
|
||||
// safe to call from a non-isolated context.
|
||||
private nonisolated static func makeAudioTapBlock(
|
||||
converter: AVAudioConverter,
|
||||
targetFormat: AVAudioFormat,
|
||||
hwFormat: AVAudioFormat,
|
||||
downsampler: FlowAdaptiveDownsampler,
|
||||
onMeter: @Sendable @escaping (Double) -> Void,
|
||||
onSnapshot: @Sendable @escaping (AudioBufferSnapshot) -> Void
|
||||
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
|
||||
@@ -514,21 +525,15 @@ public final class LiveDictationController: ObservableObject {
|
||||
onMeter(meter)
|
||||
|
||||
// 2) Downsample to the 16 kHz mono Float32 format expected by
|
||||
// HostSupport ASR and Apple's `considering:` hint.
|
||||
let outFrames = AVAudioFrameCount(
|
||||
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
|
||||
)
|
||||
guard outFrames > 0,
|
||||
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
|
||||
else { return }
|
||||
|
||||
var error: NSError?
|
||||
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
guard status == .haveData, error == nil, outBuffer.frameLength > 0 else { return }
|
||||
// HostSupport ASR and Apple's `considering:` hint. The rate comes
|
||||
// from the buffer we were just handed, not from a format captured
|
||||
// at install time, so a mid-session route change degrades to a few
|
||||
// dropped frames instead of garbled audio.
|
||||
guard case .converted(let outBuffer) =
|
||||
downsampler.convertReusingScratch(buffer) else { return }
|
||||
|
||||
// `outBuffer` is the downsampler's reusable scratch — valid only
|
||||
// until the next callback, so copy the samples out right here.
|
||||
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
|
||||
guard !snapshot.samples.isEmpty else { return }
|
||||
onSnapshot(snapshot)
|
||||
@@ -565,9 +570,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
teardownCaptureEngine()
|
||||
captureGate.withLock { $0 = .idle }
|
||||
drainTracker.reset()
|
||||
audioConverter = nil
|
||||
targetFormat = nil
|
||||
hwFormat = nil
|
||||
downsampler = nil
|
||||
|
||||
try? AVAudioSession.sharedInstance().setActive(
|
||||
false,
|
||||
|
||||
Reference in New Issue
Block a user