feat: iCloud settings sync and cold-start return redesign

- Add iCloud key-value settings sync (engine/language/polish/Flow prefs);
  API keys stay on-device. New "Sync settings via iCloud" toggle.
- Redesign cold-start handoff: bottom-bar left-to-right swipe guidance,
  auto-dismiss on app switch, tap-anywhere to close, retained return link.
- Harden keyboard->app handoff with host-disconnected hint.
- Include prior Unreleased ASR fixes (route-change crash, fallback warning,
  multi-utterance recognition, local ASR diagnostics).

Release 0.5.0 (build 18).
This commit is contained in:
Rocky
2026-07-07 17:56:01 +08:00
parent bf844caa7f
commit 128aab1b02
42 changed files with 1818 additions and 337 deletions
+123 -34
View File
@@ -198,28 +198,30 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
func warmup(locale: Locale) async {
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
Self.debug("warmup locale unsupported requested=\(locale.identifier(.bcp47))")
return
}
let localeID = resolvedLocale.identifier(.bcp47)
let cachedLocaleID = lock.withLock { chunkPreparedLocaleID }
if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) {
Self.debug("warmup cache hit locale=\(localeID)")
return
}
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
locale: resolvedLocale
)
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
locale: resolvedLocale,
lmConfiguration: lmConfiguration
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"warmup start locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
do {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber],
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
Self.debug("warmup format unsupported locale=\(localeID)")
return
}
lock.withLock {
@@ -236,13 +238,25 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
guard !samples.isEmpty else { return .success("") }
if Task.isCancelled { return .cancelled }
let startedAt = Date()
let rms = Self.rms(of: samples)
Self.debug(
"chunk start samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " +
"locale=\(locale.identifier(.bcp47))"
)
do {
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
Self.debug(
"chunk success textLen=\(trimmed.count) elapsed=\(Self.elapsed(startedAt))s " +
"empty=\(trimmed.isEmpty)"
)
return trimmed.isEmpty ? .success("") : .success(trimmed)
} catch is CancellationError {
Self.debug("chunk cancelled elapsed=\(Self.elapsed(startedAt))s")
return .cancelled
} catch {
Self.debug("chunk failed elapsed=\(Self.elapsed(startedAt))s error=\(error.localizedDescription)")
return .failure(error.localizedDescription)
}
}
@@ -257,12 +271,11 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
throw ASRChunkError.localeUnsupported
}
let localeID = resolvedLocale.identifier(.bcp47)
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
locale: resolvedLocale
)
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
locale: resolvedLocale,
lmConfiguration: lmConfiguration
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"chunk setup locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
let analyzerFormat: AVAudioFormat
@@ -271,10 +284,14 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
cachedPrep.0 == localeID,
let cached = cachedPrep.1 {
analyzerFormat = cached
Self.debug(
"chunk using cached analyzer format sr=\(Int(cached.sampleRate)) " +
"channels=\(cached.channelCount) common=\(cached.commonFormat.rawValue)"
)
} else {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber],
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
throw ASRChunkError.formatUnsupported
@@ -284,6 +301,10 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
chunkPreparedLocaleID = localeID
chunkAnalyzerFormat = format
}
Self.debug(
"chunk prepared analyzer format sr=\(Int(format.sampleRate)) " +
"channels=\(format.channelCount) common=\(format.commonFormat.rawValue)"
)
}
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
@@ -291,12 +312,12 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
throw ASRChunkError.formatUnsupported
}
let analyzer = SpeechAnalyzer(modules: [transcriber])
let analyzer = SpeechAnalyzer(modules: [setup.transcriber])
try await analyzer.prepareToAnalyze(in: analyzerFormat)
let resultsTask = Task<String, Error> {
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in transcriber.results {
for try await result in setup.transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
_ = accumulator.ingest(range: result.range, text: text)
@@ -369,15 +390,15 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
}
// Each pipelined chunk is 30 s; long dictation preset keeps a
// single chunk coherent (Flow utterances run up to 3 min).
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
locale: resolvedLocale
)
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
locale: resolvedLocale,
lmConfiguration: lmConfiguration
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
Self.debug(
"stream setup locale=\(resolvedLocale.identifier(.bcp47)) " +
"customLMEnabled=\(setup.customLanguageModelEnabled) " +
"customLMAttached=\(setup.usesCustomLanguageModel) " +
"clmState=\(Self.describeCLMState(setup.clmState))"
)
do {
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
} catch {
Self.debug("asset prepare failed: \(error.localizedDescription)")
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
@@ -385,11 +406,11 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
return
}
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
let newAnalyzer = SpeechAnalyzer(modules: [setup.transcriber])
self.lock.withLock { self.analyzer = newAnalyzer }
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber],
compatibleWith: [setup.transcriber],
considering: Self.captureFormat
) else {
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
@@ -405,7 +426,7 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
// while `analyzeSequence` drains the input stream.
let resultsTask = Task<String, Error> {
var accumulator = ProgressiveDictationTranscriptAccumulator()
for try await result in transcriber.results {
for try await result in setup.transcriber.results {
if Task.isCancelled { break }
let text = String(result.text.characters)
guard let full = accumulator.ingest(range: result.range, text: text) else {
@@ -457,23 +478,91 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
}
}
private struct DiagnosticTranscriber {
let transcriber: DictationTranscriber
let customLanguageModelEnabled: Bool
let usesCustomLanguageModel: Bool
let clmState: CustomLanguageModelManager.PrepareState
}
private static func makeDiagnosticTranscriber(locale: Locale) -> DiagnosticTranscriber {
let defaults = AppGroup.defaultsIfAvailable
let clmKey = AppGroupConfiguration.Keys.localASRCustomLanguageModelEnabled
let clmEnabled = defaults?.object(forKey: clmKey) == nil
? true
: (defaults?.bool(forKey: clmKey) ?? true)
let clmState = CustomLanguageModelManager.shared.currentState()
let lmConfiguration = clmEnabled
? CustomLanguageModelManager.shared.configurationForTranscription(locale: locale)
: nil
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
locale: locale,
lmConfiguration: lmConfiguration
)
return DiagnosticTranscriber(
transcriber: transcriber,
customLanguageModelEnabled: clmEnabled,
usesCustomLanguageModel: lmConfiguration != nil,
clmState: clmState
)
}
private static func describeCLMState(_ state: CustomLanguageModelManager.PrepareState) -> String {
switch state {
case .idle:
return "idle"
case .preparing:
return "preparing"
case .ready:
return "ready"
case .failed(let message):
return "failed(\(message))"
}
}
private static func rms(of samples: [Float]) -> Float {
guard !samples.isEmpty else { return 0 }
var sum: Float = 0
for sample in samples {
sum += sample * sample
}
return sqrtf(sum / Float(samples.count))
}
private static func elapsed(_ start: Date) -> String {
String(format: "%.2f", Date().timeIntervalSince(start))
}
private static func debug(_ message: String) {
#if DEBUG
print("🎙️[ASRService] \(message)")
#endif
OSGLog.asr.info("\(message, privacy: .public)")
}
private static func prepareAssetsIfNeeded(
for transcriber: DictationTranscriber,
locale: Locale
) async throws {
let localeID = locale.identifier(.bcp47)
let startedAt = Date()
do {
_ = try await AssetInventory.reserve(locale: locale)
Self.debug("asset reserve ok locale=\(localeID)")
} catch {
// Reservation may already exist or slots are full; continue.
// Reservation may already exist or slots are full; continue, but
// log it so local-ASR setup failures are not hidden behind a later
// "no speech" timeout.
Self.debug("asset reserve non-fatal locale=\(localeID) error=\(error.localizedDescription)")
}
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
try await request.downloadAndInstall()
do {
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
Self.debug("asset install required locale=\(localeID)")
try await request.downloadAndInstall()
Self.debug("asset install done locale=\(localeID) elapsed=\(elapsed(startedAt))s")
} else {
Self.debug("asset already installed locale=\(localeID) elapsed=\(elapsed(startedAt))s")
}
} catch {
Self.debug("asset prepare failed locale=\(localeID) elapsed=\(elapsed(startedAt))s error=\(error.localizedDescription)")
throw error
}
}
@@ -57,6 +57,7 @@ public struct AppGroupStore: @unchecked Sendable {
public var polishModeForPipeline: PolishingService.PolishMode { configuration.polishModeForPipeline }
public var polishProviderIdOverride: String? { configuration.polishProviderIdOverride }
public var isCloudAPIKeyMissingForVoiceInput: Bool { configuration.isCloudAPIKeyMissingForVoiceInput }
public var localASRCustomLanguageModelEnabled: Bool { configuration.localASRCustomLanguageModelEnabled }
/// Whether the keyboard top-bar translation chip should render.
public var isTranslationChipVisible: Bool { true }
@@ -112,6 +113,10 @@ public struct AppGroupStore: @unchecked Sendable {
mutateConfiguration { $0.polishIntensity = intensity }
}
public func setLocalASRCustomLanguageModelEnabled(_ enabled: Bool) {
mutateConfiguration { $0.localASRCustomLanguageModelEnabled = enabled }
}
public var hasCompletedOnboarding: Bool {
get { configuration.hasCompletedOnboarding }
set { setHasCompletedOnboarding(newValue) }
@@ -129,6 +134,8 @@ public struct AppGroupStore: @unchecked Sendable {
config.onboardingPage = 0
}
}
// Mirror to the reboot-durable Keychain marker (keyboard-side completion).
Keychain.setOnboardingCompleted(completed)
}
public func setOnboardingPage(_ page: Int) {
@@ -167,6 +174,22 @@ public struct AppGroupStore: @unchecked Sendable {
mutateConfiguration { $0.personalDictionaryICloudSyncEnabled = enabled }
}
public var settingsICloudSyncEnabled: Bool {
get { configuration.settingsICloudSyncEnabled }
set { setSettingsICloudSyncEnabled(newValue) }
}
public func setSettingsICloudSyncEnabled(_ enabled: Bool) {
mutateConfiguration { $0.settingsICloudSyncEnabled = enabled }
}
/// Timestamp of the last settings blob applied from iCloud KVS.
public var settingsCloudUpdatedAt: Date? {
let raw = defaults.double(forKey: AppGroupConfiguration.Keys.settingsCloudUpdatedAt)
guard raw > 0 else { return nil }
return Date(timeIntervalSince1970: raw)
}
// MARK: - Client
public func makeClient() -> LLMClient {
@@ -123,6 +123,48 @@ private final class FlowLevelStore: @unchecked Sendable {
}
}
/// Route-adaptive downsampling converter, safe to call from the realtime tap.
///
/// `AVAudioEngine.installTap(format:)` traps with an **uncatchable** NSException
/// when the format passed to it does not match the input node's *live* format.
/// After an audio-route change which the on-device `SpeechAnalyzer` triggers
/// during warmup by reconfiguring the shared `AVAudioSession` the value
/// returned by `inputNode.outputFormat(forBus:)` can lag behind the real
/// hardware rate (e.g. it reports 48 kHz while the node has already switched to
/// 24 kHz). Installing a tap with that stale explicit format crashes the whole
/// app (`Failed to create tap due to format mismatch`).
///
/// We therefore install the tap with `format: nil` (which always uses the
/// node's live format) and rebuild the sample-rate converter *here* whenever the
/// incoming buffer's format actually changes, so downsampling to the ASR target
/// rate is always valid regardless of route churn.
private final class AdaptiveDownsampler: @unchecked Sendable {
// `AVAudioConverter` / `AVAudioFormat` are not `Sendable`, so the state and
// the returned converter are guarded manually via the unchecked lock APIs.
private let lock = OSAllocatedUnfairLock<(converter: AVAudioConverter, source: AVAudioFormat)?>(uncheckedState: nil)
let targetFormat: AVAudioFormat
init(targetFormat: AVAudioFormat) {
self.targetFormat = targetFormat
}
/// Returns a converter valid for `sourceFormat`, rebuilding it lazily when
/// the hardware route (and thus the buffer format) changes.
func converter(for sourceFormat: AVAudioFormat) -> AVAudioConverter? {
lock.withLockUnchecked { state in
if let state, state.source == sourceFormat {
return state.converter
}
guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
state = nil
return nil
}
state = (converter, sourceFormat)
return converter
}
}
}
@MainActor
public final class FlowContinuousCapture {
@@ -169,16 +211,18 @@ public final class FlowContinuousCapture {
private let drainTracker = FlowCaptureDrainTracker()
private let tailSampleCounter = OSAllocatedUnfairLock(initialState: 0)
private var audioConverter: AVAudioConverter?
private var downsampler: AdaptiveDownsampler?
private var targetFormat: AVAudioFormat?
private var hwFormat: AVAudioFormat?
private var drainPolicy = FlowCaptureTailDrainPolicy.flowDefault
private var didInstallTap = false
private var isRunning = false
private var isRebuilding = false
private var routeObserver: NSObjectProtocol?
private var interruptionObserver: NSObjectProtocol?
private var mediaResetObserver: NSObjectProtocol?
private let log = Logger(subsystem: "com.osgkeyboard.shared", category: "FlowCapture")
public init() {}
@@ -227,11 +271,11 @@ public final class FlowContinuousCapture {
) else {
throw StartError.formatCreateFailed
}
guard let converter = AVAudioConverter(from: hardwareFormat, to: resolvedTargetFormat) else {
throw StartError.converterCreateFailed
}
audioConverter = converter
// Route-adaptive converter: it rebuilds itself from the live buffer
// format inside the tap, so it never assumes a fixed hardware rate.
let downsampler = AdaptiveDownsampler(targetFormat: resolvedTargetFormat)
self.downsampler = downsampler
targetFormat = resolvedTargetFormat
hwFormat = hardwareFormat
@@ -249,9 +293,7 @@ public final class FlowContinuousCapture {
let tailCounter = tailSampleCounter
let policy = drainPolicy
let tap = Self.makeAudioTapBlock(
converter: converter,
targetFormat: resolvedTargetFormat,
hwFormat: hardwareFormat,
downsampler: downsampler,
gate: gateLock,
levelStore: levels,
prerollStore: preroll,
@@ -260,7 +302,10 @@ public final class FlowContinuousCapture {
tailSampleCounter: tailCounter,
drainPolicy: policy
)
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hardwareFormat, block: tap)
// `format: nil` binds the tap to the input node's *live* format. Passing
// an explicit (possibly stale) format here is what crashed the app on a
// route change (48 kHz client vs 24 kHz hardware); nil can never mismatch.
inputNode.installTap(onBus: 0, bufferSize: 4096, format: nil, block: tap)
didInstallTap = true
audioEngine.prepare()
@@ -287,7 +332,7 @@ public final class FlowContinuousCapture {
audioEngine.stop()
}
isRunning = false
audioConverter = nil
downsampler = nil
targetFormat = nil
hwFormat = nil
try? AVAudioSession.sharedInstance().setActive(
@@ -339,14 +384,35 @@ public final class FlowContinuousCapture {
}
}
}
// Apple QA1749: when the system media server resets, the engine,
// converter and audio session all become orphaned and must be
// rebuilt from scratch otherwise capture silently produces no
// audio (another cause of "waveform moves but ASR is empty").
if mediaResetObserver == nil {
mediaResetObserver = center.addObserver(
forName: AVAudioSession.mediaServicesWereResetNotification,
object: nil,
queue: .main
) { [weak self] _ in
MainActor.assumeIsolated { self?.handleMediaServicesReset() }
}
}
}
private func removeSessionObservers() {
let center = NotificationCenter.default
if let routeObserver { center.removeObserver(routeObserver) }
if let interruptionObserver { center.removeObserver(interruptionObserver) }
if let mediaResetObserver { center.removeObserver(mediaResetObserver) }
routeObserver = nil
interruptionObserver = nil
mediaResetObserver = nil
}
private func handleMediaServicesReset() {
guard isRunning else { return }
log.info("Media services were reset — rebuilding engine and converter")
rebuildEngine()
}
private func handleRouteChange(reasonRaw: UInt?) {
@@ -388,7 +454,9 @@ public final class FlowContinuousCapture {
/// Stop and rebuild the engine against the current route, keeping
/// `isRunning` intact so the session survives the swap transparently.
private func rebuildEngine() {
guard isRunning else { return }
guard isRunning, !isRebuilding else { return }
isRebuilding = true
defer { isRebuilding = false }
if audioEngine.isRunning {
audioEngine.stop()
}
@@ -436,9 +504,15 @@ public final class FlowContinuousCapture {
try? await Task.sleep(nanoseconds: FlowCaptureConstants.drainPollIntervalNs)
}
let flushSamples = flushConverterTailToStream()
tailSampleCounter.withLock { $0 += flushSamples }
// NOTE: We intentionally do NOT signal `.endOfStream` to the shared
// downsampling converter here. `AVAudioConverter` is stateful: once its
// input block returns `.endOfStream`, the converter is permanently
// finished and every subsequent `.haveData` conversion (from the live
// tap) returns no data which silently starved every utterance after
// the first (Apple docs + AVAudioConverter reuse guidance). Trailing
// speech is already preserved by the live `.draining` forwarding loop
// above; the converter's sub-millisecond internal filter tail is not
// worth poisoning a session-long converter for.
streamRelay.finish()
gate.withLock { $0 = .idle }
@@ -466,58 +540,10 @@ public final class FlowContinuousCapture {
levelStore.snapshot()
}
// MARK: - Converter flush
@discardableResult
private func flushConverterTailToStream() -> Int {
guard let converter = audioConverter,
let targetFormat,
let hwFormat else {
return 0
}
var flushedSamples = 0
let capacity = AVAudioFrameCount(max(512, hwFormat.sampleRate / 20))
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
return 0
}
var endOfStreamSignaled = false
while true {
outBuffer.frameLength = 0
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
if endOfStreamSignaled {
outStatus.pointee = .noDataNow
return nil
}
endOfStreamSignaled = true
outStatus.pointee = .endOfStream
return nil
}
if status == .error || error != nil {
break
}
guard status == .haveData, outBuffer.frameLength > 0 else {
break
}
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { break }
streamRelay.yield(snapshot)
flushedSamples += snapshot.samples.count
}
return flushedSamples
}
// MARK: - Audio tap (nonisolated runs on realtime thread)
private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat,
downsampler: AdaptiveDownsampler,
gate: OSAllocatedUnfairLock<UtteranceGatePhase>,
levelStore: FlowLevelStore,
prerollStore: FlowPrerollStore,
@@ -529,8 +555,15 @@ public final class FlowContinuousCapture {
return { buffer, _ in
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
// Derive the converter from the *live* buffer format so a mid-session
// route change (e.g. 48 kHz 24 kHz) is handled transparently.
let sourceFormat = buffer.format
let targetFormat = downsampler.targetFormat
guard sourceFormat.sampleRate > 0,
let converter = downsampler.converter(for: sourceFormat) else { return }
let outFrames = AVAudioFrameCount(
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
Double(buffer.frameLength) * targetFormat.sampleRate / sourceFormat.sampleRate
)
guard outFrames > 0,
let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outFrames)
@@ -117,9 +117,9 @@ public enum FlowSessionBridge {
// MARK: - Session validity (keyboard)
/// True when the session contract is still valid (not expired).
/// Does not require a fresh heartbeat the host may be suspended in
/// background while the continuous audio session is frozen.
/// True when the App Group session contract is still valid (not expired).
/// Does **not** mean the host process is alive use `isHostReachable()` for
/// recording gates and "session ready" UI.
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
let store = resolvedDefaults(defaults)
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
@@ -128,19 +128,47 @@ public enum FlowSessionBridge {
return expires > Date().timeIntervalSince1970
}
/// Seconds since the host last wrote `flowHeartbeat`; nil when never written.
public static func heartbeatStaleness(defaults: UserDefaults? = nil) -> TimeInterval? {
let store = resolvedDefaults(defaults)
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
guard heartbeat > 0 else { return nil }
return Date().timeIntervalSince1970 - heartbeat
}
/// True when the host app recently wrote a heartbeat (foreground or
/// actively processing). Used for auto-start heuristics, not gating record.
/// actively processing). Gating record / "session ready" UI must use this,
/// not `isSessionActive()` alone.
public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
let store = resolvedDefaults(defaults)
guard isSessionActive(defaults: store) else { return false }
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
guard heartbeat > 0 else { return false }
let staleness = Date().timeIntervalSince1970 - heartbeat
guard let staleness = heartbeatStaleness(defaults: store) else { return false }
return staleness <= FlowSessionKeys.heartbeatStaleInterval
}
/// True when the session contract flag is still set but the host heartbeat
/// proves the process is gone (reboot, force-quit, long suspend).
public static func isHostStale(
staleAfter: TimeInterval = FlowSessionKeys.heartbeatZombieInterval,
defaults: UserDefaults? = nil
) -> Bool {
let store = resolvedDefaults(defaults)
guard isSessionActive(defaults: store) else { return false }
guard let staleness = heartbeatStaleness(defaults: store) else { return true }
return staleness > staleAfter
}
/// Clears orphaned App Group Flow state when the host is provably dead.
@discardableResult
public static func clearIfHostStale(
staleAfter: TimeInterval = FlowSessionKeys.heartbeatZombieInterval,
defaults: UserDefaults? = nil
) -> Bool {
guard isHostStale(staleAfter: staleAfter, defaults: defaults) else { return false }
clearFlowState(defaults: defaults)
return true
}
public static func sessionExpiresAt(defaults: UserDefaults? = nil) -> TimeInterval? {
let store = resolvedDefaults(defaults)
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
@@ -26,9 +26,16 @@ public enum FlowSessionKeys {
/// Wall-clock timestamp of the last utterance completion or session start.
public static let lastActivityAt = "flow.lastActivityAt"
/// Heartbeat older than this while the host is foreground likely killed.
/// Heartbeat older than this host is not actively reachable for recording.
public static let heartbeatStaleInterval: TimeInterval = 3
/// Session flag still set but heartbeat older than this host process is
/// dead (force-quit, reboot). Keyboard / host should clear persisted state.
public static let heartbeatZombieInterval: TimeInterval = 60
/// After mic stop, fail fast when the host heartbeat is gone longer than this.
public static let keyboardHostDisconnectFailFast: TimeInterval = 15
/// Legacy fixed session length prefer `FlowSessionPolicy.sessionDuration()`.
public static let defaultSessionDuration: TimeInterval = 480
@@ -0,0 +1,70 @@
// AppCloudSync.swift
// OSGKeyboard · Shared
//
// Single entry point for iCloud KVS sync in the main app: preferences
// toggles, settings payload, and personal dictionary.
import Foundation
@MainActor
public final class AppCloudSync {
public static let shared = AppCloudSync()
private let kvs: UbiquitousKeyValueStoreing
private let makeStore: () -> AppGroupStore
private let settingsSync: SettingsCloudSync
private let dictionarySync: PersonalDictionaryCloudSync
private var externalChangeObserver: NSObjectProtocol?
public init(
kvs: UbiquitousKeyValueStoreing = NSUbiquitousKeyValueStore.default,
makeStore: @escaping () -> AppGroupStore = { AppGroupStore() },
settingsSync: SettingsCloudSync? = nil,
dictionarySync: PersonalDictionaryCloudSync? = nil
) {
self.kvs = kvs
self.makeStore = makeStore
self.settingsSync = settingsSync ?? SettingsCloudSync(kvs: kvs, makeStore: makeStore)
self.dictionarySync = dictionarySync ?? PersonalDictionaryCloudSync(kvs: kvs, makeStore: makeStore)
}
public func startObservingExternalChanges() {
guard externalChangeObserver == nil else { return }
externalChangeObserver = NotificationCenter.default.addObserver(
forName: NSUbiquitousKeyValueStore.didChangeExternallyNotification,
object: nil,
queue: .main
) { [weak self] _ in
guard let self else { return }
Task { @MainActor in
await self.pullAllIfEnabled()
}
}
}
public func stopObservingExternalChanges() {
if let externalChangeObserver {
NotificationCenter.default.removeObserver(externalChangeObserver)
self.externalChangeObserver = nil
}
}
/// Launch / foreground: refresh KVS toggles, then pull payloads.
public func pullAllIfEnabled() async {
let store = makeStore()
ICloudSyncPreferences.migrateLegacyTogglesIfNeeded(kvs: kvs, store: store)
let toggles = ICloudSyncPreferences.load(from: kvs, store: store)
ICloudSyncPreferences.cacheToAppGroup(
settingsEnabled: toggles.settings,
dictionaryEnabled: toggles.dictionary,
store: store
)
await settingsSync.pullAndMergeIfEnabled()
await dictionarySync.pullAndMergeIfEnabled()
}
public var settingsSyncService: SettingsCloudSync { settingsSync }
public var dictionarySyncService: PersonalDictionaryCloudSync { dictionarySync }
}
@@ -0,0 +1,53 @@
// ICloudSyncPreferences.swift
// OSGKeyboard · Shared
//
// iCloud KVS is the source of truth for cross-device sync toggles
// (scheme A). App Group UserDefaults keeps a local cache so the
// keyboard extension and offline UI can read the last-known state.
import Foundation
public enum ICloudSyncPreferences {
public static let settingsEnabledKey = "iCloudSync.settingsEnabled"
public static let dictionaryEnabledKey = "personalDictionary.syncEnabled"
/// Read sync toggles from KVS, falling back to the App Group cache
/// when a key has not been uploaded yet.
public static func load(from kvs: UbiquitousKeyValueStoreing, store: AppGroupStore) -> (settings: Bool, dictionary: Bool) {
let settings = kvs.object(forKey: settingsEnabledKey) as? Bool
?? store.settingsICloudSyncEnabled
let dictionary = kvs.object(forKey: dictionaryEnabledKey) as? Bool
?? store.personalDictionaryICloudSyncEnabled
return (settings, dictionary)
}
/// Mirror KVS toggles into the App Group cache.
public static func cacheToAppGroup(
settingsEnabled: Bool,
dictionaryEnabled: Bool,
store: AppGroupStore
) {
store.setSettingsICloudSyncEnabled(settingsEnabled)
store.setPersonalDictionaryICloudSyncEnabled(dictionaryEnabled)
}
public static func pushSettingsEnabled(_ enabled: Bool, kvs: UbiquitousKeyValueStoreing) {
kvs.set(enabled, forKey: settingsEnabledKey)
_ = kvs.synchronize()
}
public static func pushDictionaryEnabled(_ enabled: Bool, kvs: UbiquitousKeyValueStoreing) {
kvs.set(enabled, forKey: dictionaryEnabledKey)
_ = kvs.synchronize()
}
/// One-time migration: upload locally cached toggles when KVS has no value yet.
public static func migrateLegacyTogglesIfNeeded(kvs: UbiquitousKeyValueStoreing, store: AppGroupStore) {
if kvs.object(forKey: dictionaryEnabledKey) == nil {
pushDictionaryEnabled(store.personalDictionaryICloudSyncEnabled, kvs: kvs)
}
if kvs.object(forKey: settingsEnabledKey) == nil {
pushSettingsEnabled(store.settingsICloudSyncEnabled, kvs: kvs)
}
}
}
@@ -0,0 +1,137 @@
// SettingsCloudSync.swift
// OSGKeyboard · Shared
//
// Mirrors user-facing app settings through iCloud KVS. API keys stay
// in Keychain and are never uploaded.
import Foundation
public extension Notification.Name {
/// Posted after remote settings are applied to the App Group cache.
static let settingsDidSyncFromCloud = Notification.Name(
"com.osgkeyboard.settings.didSyncFromCloud"
)
}
public enum SettingsCloudSyncError: Error, Equatable, Sendable {
case encodeFailed
case decodeFailed
}
@MainActor
public final class SettingsCloudSync {
public static let shared = SettingsCloudSync()
public static let kvsKey = "appSettings.v1"
private let kvs: UbiquitousKeyValueStoreing
private let makeStore: () -> AppGroupStore
public init(
kvs: UbiquitousKeyValueStoreing = NSUbiquitousKeyValueStore.default,
makeStore: @escaping () -> AppGroupStore = { AppGroupStore() }
) {
self.kvs = kvs
self.makeStore = makeStore
}
public func pullAndMergeIfEnabled() async {
let store = makeStore()
guard store.settingsICloudSyncEnabled else { return }
await pullAndMerge(store: store)
}
public func pushLocalIfEnabled() async throws {
let store = makeStore()
guard store.settingsICloudSyncEnabled else { return }
let local = SyncedAppSettings.from(configuration: store.configurationSnapshot())
try push(local)
}
public func enableSync() async throws {
let store = makeStore()
ICloudSyncPreferences.pushSettingsEnabled(true, kvs: kvs)
ICloudSyncPreferences.cacheToAppGroup(
settingsEnabled: true,
dictionaryEnabled: store.personalDictionaryICloudSyncEnabled,
store: store
)
let local = SyncedAppSettings.from(configuration: store.configurationSnapshot())
let remote = loadRemote() ?? local
let merged = SyncedAppSettings.merge(local: local, remote: remote)
apply(merged, to: store, postNotification: false)
try push(merged)
NotificationCenter.default.post(name: .settingsDidSyncFromCloud, object: nil)
}
public func disableSync() {
let store = makeStore()
ICloudSyncPreferences.pushSettingsEnabled(false, kvs: kvs)
store.setSettingsICloudSyncEnabled(false)
}
public func pullAndMerge(store: AppGroupStore) async {
guard store.settingsICloudSyncEnabled else { return }
guard let remote = loadRemote() else { return }
let local = SyncedAppSettings.from(
configuration: store.configurationSnapshot(),
updatedAt: store.settingsCloudUpdatedAt ?? .distantPast
)
let merged = SyncedAppSettings.merge(local: local, remote: remote)
guard merged != local else { return }
apply(merged, to: store, postNotification: true)
}
public func push(_ settings: SyncedAppSettings) throws {
let encoder = JSONEncoder()
encoder.dateEncodingStrategy = .iso8601
guard let data = try? encoder.encode(settings) else {
throw SettingsCloudSyncError.encodeFailed
}
kvs.set(data, forKey: Self.kvsKey)
_ = kvs.synchronize()
}
public func loadRemote() -> SyncedAppSettings? {
guard let data = kvs.data(forKey: Self.kvsKey) else { return nil }
return try? decode(data)
}
public func decode(_ data: Data) throws -> SyncedAppSettings {
let decoder = JSONDecoder()
decoder.dateDecodingStrategy = .iso8601
guard let settings = try? decoder.decode(SyncedAppSettings.self, from: data) else {
throw SettingsCloudSyncError.decodeFailed
}
return settings
}
private func apply(
_ settings: SyncedAppSettings,
to store: AppGroupStore,
postNotification: Bool
) {
var config = store.configurationSnapshot()
settings.applying(to: &config)
store.saveConfiguration(config, settingsCloudUpdatedAt: settings.updatedAt)
if postNotification {
AppGroupConfigDarwin.postConfigChanged()
NotificationCenter.default.post(name: .settingsDidSyncFromCloud, object: nil)
}
}
}
private extension AppGroupStore {
func configurationSnapshot() -> AppGroupConfiguration {
AppGroupConfiguration.load(fromAvailable: defaults)
}
func saveConfiguration(_ configuration: AppGroupConfiguration, settingsCloudUpdatedAt: Date) {
let config = configuration
config.save(to: defaults)
defaults.set(settingsCloudUpdatedAt.timeIntervalSince1970, forKey: AppGroupConfiguration.Keys.settingsCloudUpdatedAt)
}
}
@@ -80,7 +80,8 @@ public final class KeyboardState: ObservableObject {
@Published public var onDeviceSupported: Bool = false
/// Seconds remaining in the current utterance (Flow tap-to-talk).
@Published public var utteranceRemainingSeconds: Int = Int(FlowSessionKeys.maxUtteranceDuration)
/// Whether the host app's Flow voice session is currently valid.
/// Whether the host app's Flow voice session is live and reachable (fresh
/// heartbeat). Do not use the App Group session flag alone for UI gating.
@Published public var flowSessionActive: Bool = false
/// When true, the mic is intentionally disabled (e.g. cloud engine
/// selected but the provider-specific API key is missing).
+72
View File
@@ -180,4 +180,76 @@ public enum Keychain: @unchecked Sendable {
throw KeychainError.unexpectedStatus(status)
}
}
// MARK: - Onboarding completion (reboot-durable flag)
// App Group UserDefaults can transiently read empty right after a device
// reboot (data protection / `cfprefsd` not warmed), which made the app
// falsely re-show onboarding. This Keychain marker uses the same
// `AfterFirstUnlockThisDeviceOnly` class reliably readable once the app
// can run, device-local, never synced so it stays a trustworthy fallback
// that survives the App Group read race.
private static let onboardingService = "com.osgkeyboard.onboarding"
private static let onboardingAccount = "hasCompletedOnboarding"
/// Durable "user finished onboarding" marker. `false` when unset or unreadable.
public static func hasCompletedOnboarding() -> Bool {
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrService as String: onboardingService,
kSecAttrAccount as String: onboardingAccount,
kSecReturnData as String: true,
kSecMatchLimit as String: kSecMatchLimitOne,
]
var result: CFTypeRef?
let status = SecItemCopyMatching(query as CFDictionary, &result)
guard status == errSecSuccess,
let data = result as? Data,
let str = String(data: data, encoding: .utf8) else {
OSGLog.config.info("[onboarding] Keychain read: status=\(status, privacy: .public) → false")
return false
}
let completed = str == "1"
OSGLog.config.info(
"[onboarding] Keychain read: status=ok value=\(str, privacy: .public)\(completed, privacy: .public)"
)
return completed
}
/// Mirror the onboarding-completed flag. Best-effort and idempotent a
/// no-op when the stored value already matches, so it can be called from
/// frequently-saved config paths without Keychain churn.
public static func setOnboardingCompleted(_ completed: Bool) {
guard hasCompletedOnboarding() != completed else {
OSGLog.config.info("[onboarding] Keychain write skipped (already \(completed, privacy: .public))")
return
}
let baseQuery: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrService as String: onboardingService,
kSecAttrAccount as String: onboardingAccount,
]
guard completed else {
let delStatus = SecItemDelete(baseQuery as CFDictionary)
OSGLog.config.info("[onboarding] Keychain delete: status=\(delStatus, privacy: .public)")
return
}
let data = Data("1".utf8)
let updateStatus = SecItemUpdate(
baseQuery as CFDictionary,
[kSecValueData as String: data] as CFDictionary
)
if updateStatus == errSecItemNotFound {
var addQuery = baseQuery
addQuery[kSecValueData as String] = data
addQuery[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
let addStatus = SecItemAdd(addQuery as CFDictionary, nil)
OSGLog.config.info("[onboarding] Keychain add: status=\(addStatus, privacy: .public)")
} else {
OSGLog.config.info("[onboarding] Keychain update: status=\(updateStatus, privacy: .public)")
}
}
}
@@ -565,7 +565,12 @@ public final class LiveDictationController: ObservableObject {
try? await Task.sleep(nanoseconds: 20_000_000)
}
_ = flushConverterTailToStream()
// Trailing speech is preserved by the live `.draining` forwarding
// loop above. We deliberately do NOT signal `.endOfStream` to the
// converter to squeeze its internal filter tail: that both races the
// still-running audio-thread tap on the same non-thread-safe converter
// and (in reused-converter paths) permanently locks it. The dropped
// tail is sub-millisecond and inaudible.
streamRelay.finish()
teardownCaptureEngine()
captureGate.withLock { $0 = .idle }
@@ -580,50 +585,6 @@ public final class LiveDictationController: ObservableObject {
)
}
@discardableResult
private func flushConverterTailToStream() -> Int {
guard let converter = audioConverter,
let targetFormat,
let hwFormat else {
return 0
}
var flushedSamples = 0
let capacity = AVAudioFrameCount(max(512, hwFormat.sampleRate / 20))
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
return 0
}
var endOfStreamSignaled = false
while true {
outBuffer.frameLength = 0
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
if endOfStreamSignaled {
outStatus.pointee = .noDataNow
return nil
}
endOfStreamSignaled = true
outStatus.pointee = .endOfStream
return nil
}
if status == .error || error != nil {
break
}
guard status == .haveData, outBuffer.frameLength > 0 else {
break
}
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { break }
streamRelay.yield(snapshot)
flushedSamples += snapshot.samples.count
}
return flushedSamples
}
private func teardownCaptureEngine() {
if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0)
@@ -80,7 +80,12 @@ public final class PersonalDictionaryCloudSync {
/// Enable sync: merge local + remote, persist locally, then upload.
public func enableSync() async throws {
let store = makeStore()
store.setPersonalDictionaryICloudSyncEnabled(true)
ICloudSyncPreferences.pushDictionaryEnabled(true, kvs: kvs)
ICloudSyncPreferences.cacheToAppGroup(
settingsEnabled: store.settingsICloudSyncEnabled,
dictionaryEnabled: true,
store: store
)
let local = store.personalDictionary
let remote = loadRemote() ?? .empty
@@ -90,7 +95,9 @@ public final class PersonalDictionaryCloudSync {
}
public func disableSync() {
makeStore().setPersonalDictionaryICloudSyncEnabled(false)
let store = makeStore()
ICloudSyncPreferences.pushDictionaryEnabled(false, kvs: kvs)
store.setPersonalDictionaryICloudSyncEnabled(false)
}
// MARK: - Core operations
@@ -8,6 +8,8 @@ import Foundation
public protocol UbiquitousKeyValueStoreing: AnyObject {
func data(forKey key: String) -> Data?
func set(_ value: Data?, forKey key: String)
func object(forKey key: String) -> Any?
func set(_ value: Any?, forKey key: String)
@discardableResult
func synchronize() -> Bool
}
@@ -43,6 +43,19 @@ public enum TranscriptPostProcessor: Sendable {
text.trimmingCharacters(in: .whitespacesAndNewlines)
}
/// Conservative cleanup for raw ASR fallback delivery. This is used when
/// polish/translation cannot run, so it must not rewrite meaning or invent
/// punctuation; it only removes formatting artifacts that ASR/chunking can
/// introduce.
public static func cleanRawASRFallback(_ text: String) -> String {
var result = text.trimmingCharacters(in: .whitespacesAndNewlines)
result = repairMidSentenceLineBreaks(result)
result = collapseHorizontalWhitespace(result)
result = removeCJKBoundarySpaces(result)
result = normalizePunctuationSpacing(result)
return normalizeWhitespaceAndPunctuation(result)
}
// MARK: - Post-LLM pipeline
/// Apply deterministic cleanup and quality gate to LLM output.
@@ -220,6 +233,53 @@ public enum TranscriptPostProcessor: Sendable {
return result.trimmingCharacters(in: .whitespacesAndNewlines)
}
private static func collapseHorizontalWhitespace(_ text: String) -> String {
text.replacingOccurrences(
of: #"[^\S\r\n]+"#,
with: " ",
options: .regularExpression
)
}
private static func removeCJKBoundarySpaces(_ text: String) -> String {
var output = ""
let characters = Array(text)
for index in characters.indices {
let current = characters[index]
if current.isWhitespace,
let previous = previousNonWhitespace(in: characters, before: index),
let next = nextNonWhitespace(in: characters, after: index),
shouldDropSpaceBetween(previous: previous, next: next) {
continue
}
output.append(current)
}
return output
}
private static func normalizePunctuationSpacing(_ text: String) -> String {
var output = ""
let characters = Array(text)
for index in characters.indices {
let current = characters[index]
if current.isWhitespace,
let next = nextNonWhitespace(in: characters, after: index),
isClosingPunctuation(next) {
continue
}
if isOpeningPunctuation(current),
let next = nextNonWhitespace(in: characters, after: index),
next.isWhitespace {
output.append(current)
continue
}
output.append(current)
}
return output
.replacingOccurrences(of: #"([(「“])\s+"#, with: "$1", options: .regularExpression)
.replacingOccurrences(of: #"\s+([,。!?;:、,.!?;:])"#, with: "$1", options: .regularExpression)
}
// MARK: - Prefix / quote cleanup
public static func stripExplanatoryPrefix(from text: String) -> String {
@@ -271,6 +331,41 @@ public enum TranscriptPostProcessor: Sendable {
return (pAscii && nAscii) ? " " : ""
}
private static func previousNonWhitespace(in characters: [Character], before index: Int) -> Character? {
guard index > characters.startIndex else { return nil }
for i in stride(from: index - 1, through: characters.startIndex, by: -1) {
if !characters[i].isWhitespace { return characters[i] }
}
return nil
}
private static func nextNonWhitespace(in characters: [Character], after index: Int) -> Character? {
let nextIndex = index + 1
guard nextIndex < characters.endIndex else { return nil }
for i in nextIndex..<characters.endIndex {
if !characters[i].isWhitespace { return characters[i] }
}
return nil
}
private static func shouldDropSpaceBetween(previous: Character, next: Character) -> Bool {
(isCJKCharacter(previous) && isCJKCharacter(next)) || isClosingPunctuation(next)
}
private static func isCJKCharacter(_ character: Character) -> Bool {
character.unicodeScalars.contains(where: isCJKScalar)
}
private static func isClosingPunctuation(_ character: Character) -> Bool {
let closing: Set<Character> = ["", "", "", "", "", "", "", ",", ".", "!", "?", ";", ":"]
return closing.contains(character)
}
private static func isOpeningPunctuation(_ character: Character) -> Bool {
let opening: Set<Character> = ["", "", ""]
return opening.contains(character)
}
private static func extractEmojis(from text: String) -> [String] {
text.unicodeScalars.filter(isEmojiScalar).map { String($0) }
}