perf(asr): speed up local Flow dictation and land CLM/keyboard refactor
Reduce perceived latency from key release to final text: - Adaptive chunking: 2.5s first chunk + 5s follow-ups so short utterances start on-device recognition while still recording. - Session-level ASR warmup and audio-format cache reuse to remove per-utterance cold-start of SpeechAnalyzer. - Mirror live pipelined partials to the keyboard transcript line via a new flow.transcriptionPartial App Group key + Darwin ping. Also commits the accumulated custom language model, Flow session, keyboard extension restructure, and Xiaomi MiMo provider work in progress on this branch.
This commit is contained in:
Rocky
@@ -47,6 +47,9 @@ public protocol ASRService: Sendable {
|
||||
/// Clears cancellation / cached session state before a new utterance.
|
||||
func resetForNewUtterance()
|
||||
|
||||
/// Pre-load locale assets and analyzer format for lower first-chunk latency.
|
||||
func warmup(locale: Locale) async
|
||||
|
||||
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
|
||||
}
|
||||
@@ -60,6 +63,8 @@ public enum ASRChunkResult: Sendable, Equatable {
|
||||
extension ASRService {
|
||||
public func resetForNewUtterance() {}
|
||||
|
||||
public func warmup(locale: Locale) async {}
|
||||
|
||||
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
@@ -116,27 +121,7 @@ public enum ASREvent: Sendable, Equatable {
|
||||
// MARK: - Factory
|
||||
|
||||
public enum ASRServiceFactory {
|
||||
/// Returns the on-device ASR backend. As of v0.2.0 the only
|
||||
/// supported `LocalASRBackend` is iOS 26 `SpeechAnalyzer` +
|
||||
/// `DictationTranscriber` (always on-device, no asset download),
|
||||
/// so the factory collapses to a single concrete type. We keep the
|
||||
/// `localBackend` parameter on the signature so the next non-iOS
|
||||
/// backend can slot in without touching every call site.
|
||||
///
|
||||
/// The cloud engine also routes through `SpeechAnalyzerASR`: the
|
||||
/// user expectation is that ASR is the local half of the pipeline
|
||||
/// regardless of where the LLM polish happens.
|
||||
public static func make(
|
||||
engineMode: String,
|
||||
localBackend: LocalASRBackend = .speechAnalyzer
|
||||
) -> ASRService {
|
||||
SpeechAnalyzerASR()
|
||||
}
|
||||
|
||||
/// Back-compat overload for callers that only ever want the
|
||||
/// SpeechAnalyzer path. The previous single-backend build used
|
||||
/// this signature; new code should pass the engine mode explicitly
|
||||
/// so any future non-iOS backend is honoured.
|
||||
/// Returns the on-device `SpeechAnalyzer` + `DictationTranscriber` backend.
|
||||
public static func make() -> ASRService {
|
||||
SpeechAnalyzerASR()
|
||||
}
|
||||
@@ -197,12 +182,52 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
private var chunkAnalyzerFormat: AVAudioFormat?
|
||||
|
||||
func resetForNewUtterance() {
|
||||
// Keep chunk format / asset cache warm across utterances in one Flow session.
|
||||
}
|
||||
|
||||
func invalidateChunkPreparationCache() {
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = nil
|
||||
chunkAnalyzerFormat = nil
|
||||
}
|
||||
}
|
||||
|
||||
func warmup(locale: Locale) async {
|
||||
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
||||
return
|
||||
}
|
||||
let localeID = resolvedLocale.identifier(.bcp47)
|
||||
let cachedLocaleID = lock.withLock { chunkPreparedLocaleID }
|
||||
if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) {
|
||||
return
|
||||
}
|
||||
|
||||
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
|
||||
locale: resolvedLocale
|
||||
)
|
||||
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
|
||||
locale: resolvedLocale,
|
||||
lmConfiguration: lmConfiguration
|
||||
)
|
||||
|
||||
do {
|
||||
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
||||
compatibleWith: [transcriber],
|
||||
considering: Self.captureFormat
|
||||
) else {
|
||||
return
|
||||
}
|
||||
lock.withLock {
|
||||
chunkPreparedLocaleID = localeID
|
||||
chunkAnalyzerFormat = format
|
||||
}
|
||||
Self.debug("warmup ready locale=\(localeID)")
|
||||
} catch {
|
||||
Self.debug("warmup failed: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
guard !samples.isEmpty else { return .success("") }
|
||||
if Task.isCancelled { return .cancelled }
|
||||
@@ -228,9 +253,12 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
throw ASRChunkError.localeUnsupported
|
||||
}
|
||||
let localeID = resolvedLocale.identifier(.bcp47)
|
||||
let transcriber = DictationTranscriber(
|
||||
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
|
||||
locale: resolvedLocale
|
||||
)
|
||||
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
|
||||
locale: resolvedLocale,
|
||||
preset: .progressiveLongDictation
|
||||
lmConfiguration: lmConfiguration
|
||||
)
|
||||
|
||||
let analyzerFormat: AVAudioFormat
|
||||
@@ -337,9 +365,12 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
}
|
||||
// Each pipelined chunk is ≤ 30 s; long dictation preset keeps a
|
||||
// single chunk coherent (Flow utterances run up to 3 min).
|
||||
let transcriber = DictationTranscriber(
|
||||
let lmConfiguration = CustomLanguageModelManager.shared.configurationForTranscription(
|
||||
locale: resolvedLocale
|
||||
)
|
||||
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
|
||||
locale: resolvedLocale,
|
||||
preset: .progressiveLongDictation
|
||||
lmConfiguration: lmConfiguration
|
||||
)
|
||||
do {
|
||||
try await Self.prepareAssetsIfNeeded(for: transcriber, locale: resolvedLocale)
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
// AppGroupStore.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Convenience wrapper around App Group UserDefaults for non-Published reads.
|
||||
// Used by the keyboard extension (no SwiftUI) to read config without
|
||||
// instantiating an ObservableObject.
|
||||
// Thin read/write facade over `AppGroupConfiguration` for the keyboard
|
||||
// extension (no SwiftUI) and other non-ObservableObject call sites.
|
||||
//
|
||||
// `apiKey` is NOT read from UserDefaults — see `Keychain.swift`. We
|
||||
// share access between the host app and the keyboard extension via a
|
||||
// shared keychain-access-group declared in both targets' entitlements.
|
||||
// `apiKey` is NOT stored in UserDefaults — see `Keychain.swift`.
|
||||
|
||||
import Foundation
|
||||
|
||||
@@ -19,340 +16,150 @@ public struct AppGroupStore: @unchecked Sendable {
|
||||
self.defaults = defaults
|
||||
return
|
||||
}
|
||||
// Never hard-crash on implicit construction sites (e.g. default
|
||||
// service initializers). If App Group is unavailable, use .standard
|
||||
// so callers can still surface a user-facing setup error.
|
||||
self.defaults = AppGroup.isAvailable ? AppGroup.defaults : .standard
|
||||
guard let available = AppGroup.defaultsIfAvailable else {
|
||||
#if DEBUG
|
||||
fatalError("App Group unavailable — inject UserDefaults in tests or fix entitlements.")
|
||||
#else
|
||||
// Callers must check `AppGroup.isAvailable` before constructing.
|
||||
fatalError("App Group unavailable.")
|
||||
#endif
|
||||
}
|
||||
self.defaults = available
|
||||
}
|
||||
|
||||
// MARK: - Keys
|
||||
private var configuration: AppGroupConfiguration {
|
||||
AppGroupConfiguration.load(fromAvailable: defaults)
|
||||
}
|
||||
|
||||
private enum Key {
|
||||
static let providerId = "config.providerId"
|
||||
static let baseURL = "config.baseURL"
|
||||
static let model = "config.model"
|
||||
static let modeId = "config.modeId"
|
||||
static let localeId = "config.localeId"
|
||||
static let engineMode = "config.engineMode"
|
||||
static let localASRBackend = "config.localASRBackend"
|
||||
static let uiLanguage = "config.uiLanguage"
|
||||
// v0.2.0: opt-in cloud polish step after local-mode ASR.
|
||||
static let localModeCloudPolishEnabled = "config.localModeCloudPolishEnabled"
|
||||
// v0.2.1 follow-up: `config.translationEnabled` was *removed* as a
|
||||
// persisted key — translation is derived from the target locale
|
||||
// id. New code should only write/read `translationTargetLocaleId`;
|
||||
// the `translationEnabled` Bool accessor below is kept as a
|
||||
// computed shim for source compatibility.
|
||||
static let translationTargetLocaleId = "config.translationTargetLocaleId"
|
||||
static let handednessPreference = "config.handednessPreference"
|
||||
// Drag pads beside the mic move the caret like arrow keys.
|
||||
static let cursorDragNavigationEnabled = "config.cursorDragNavigationEnabled"
|
||||
// v0.3.0: polish intensity (off / light / medium / heavy).
|
||||
static let polishIntensity = "config.polishIntensity"
|
||||
// v0.3.0: last app context detected by the keyboard extension.
|
||||
// Reused across calls within a 30-minute window so the LLM
|
||||
// prompt remains consistent during a single typing session.
|
||||
static let detectedAppContext = "config.detectedAppContext"
|
||||
static let detectedAppContextAt = "config.detectedAppContextAt"
|
||||
// v0.3.0: personal dictionary — JSON-encoded `PersonalDictionary`.
|
||||
static let personalDictionary = "config.personalDictionary.v1"
|
||||
private func mutateConfiguration(_ transform: (inout AppGroupConfiguration) -> Void) {
|
||||
var config = AppGroupConfiguration.load(fromAvailable: defaults)
|
||||
transform(&config)
|
||||
config.save(to: defaults)
|
||||
}
|
||||
|
||||
// MARK: - Reads
|
||||
|
||||
public var providerId: String {
|
||||
defaults.string(forKey: Key.providerId) ?? "openai"
|
||||
}
|
||||
|
||||
public var baseURL: String {
|
||||
defaults.string(forKey: Key.baseURL) ?? LLMProvider.provider(id: providerId).defaultBaseURL
|
||||
}
|
||||
|
||||
/// API key lives in the Keychain (cross-process, encrypted at rest).
|
||||
/// Returns "" when nothing is stored so the LLMClient can surface a
|
||||
/// `noAPIKey` error rather than firing off an obviously-bad request.
|
||||
public var apiKey: String {
|
||||
Keychain.apiKey(for: providerId) ?? ""
|
||||
}
|
||||
|
||||
public var model: String {
|
||||
defaults.string(forKey: Key.model) ?? LLMProvider.provider(id: providerId).defaultModel
|
||||
}
|
||||
|
||||
public var modeId: String {
|
||||
defaults.string(forKey: Key.modeId) ?? "polish"
|
||||
}
|
||||
|
||||
public var localeId: String {
|
||||
defaults.string(forKey: Key.localeId) ?? "auto"
|
||||
}
|
||||
|
||||
/// "local" → on-device ASR only (raw transcript delivery).
|
||||
/// "cloud" → ASR + LLM polish (default behaviour).
|
||||
public var engineMode: String {
|
||||
defaults.string(forKey: Key.engineMode) ?? "cloud"
|
||||
}
|
||||
|
||||
/// Which on-device ASR engine backs the "local" engine mode. Falls
|
||||
/// back to the iOS SpeechAnalyzer path so legacy installs (which
|
||||
/// never wrote this key) keep working.
|
||||
public var localASRBackend: LocalASRBackend {
|
||||
let raw = defaults.string(forKey: Key.localASRBackend) ?? LocalASRBackend.speechAnalyzer.rawValue
|
||||
return LocalASRBackend(rawValue: raw) ?? .speechAnalyzer
|
||||
}
|
||||
|
||||
/// v0.2.0: whether the local engine should route its transcript
|
||||
/// through the configured cloud LLM (DeepSeek by default) before
|
||||
/// insertion. Defaults to `false`; the keyboard extension reads
|
||||
/// this so Flow sessions honour the toggle.
|
||||
public var localModeCloudPolishEnabled: Bool {
|
||||
guard defaults.object(forKey: Key.localModeCloudPolishEnabled) != nil else {
|
||||
return false
|
||||
}
|
||||
return defaults.bool(forKey: Key.localModeCloudPolishEnabled)
|
||||
}
|
||||
|
||||
/// Host-app UI language override (`auto` / `en` / `zh-Hans`).
|
||||
public var uiLanguage: AppUILanguage {
|
||||
AppUILanguage.fromStored(defaults.string(forKey: Key.uiLanguage))
|
||||
}
|
||||
|
||||
/// v0.2.1 follow-up: derived — translation is on iff a target locale
|
||||
/// has been selected. The `translationTargetLocaleId` getter below
|
||||
/// is the source of truth; this property exists for backwards
|
||||
/// compatibility with call sites that read `store.translationEnabled`.
|
||||
public var translationEnabled: Bool {
|
||||
translationTargetLocaleId != TranslationLanguageCatalog.offLocaleId
|
||||
}
|
||||
|
||||
/// v0.2.1: target locale id the translate-and-polish prompt should
|
||||
/// produce (e.g. `"en"`, `"ja"`). Defaults to `offLocaleId` ("off")
|
||||
/// when nothing is stored, matching the picker / chip UX where the
|
||||
/// user has to actively pick a language to turn translation on.
|
||||
public var translationTargetLocaleId: String {
|
||||
defaults.string(forKey: Key.translationTargetLocaleId)
|
||||
?? TranslationLanguageCatalog.offLocaleId
|
||||
}
|
||||
|
||||
/// Bottom-row key order on the keyboard extension.
|
||||
public var handednessPreference: HandednessPreference {
|
||||
HandednessPreference.fromStored(defaults.string(forKey: Key.handednessPreference))
|
||||
}
|
||||
|
||||
/// Press-and-drag pads beside the mic for four-way caret movement.
|
||||
/// Defaults to `true` for new installs.
|
||||
public var cursorDragNavigationEnabled: Bool {
|
||||
guard defaults.object(forKey: Key.cursorDragNavigationEnabled) != nil else {
|
||||
return true
|
||||
}
|
||||
return defaults.bool(forKey: Key.cursorDragNavigationEnabled)
|
||||
}
|
||||
|
||||
// MARK: - Writes
|
||||
|
||||
public func setModeId(_ id: String) {
|
||||
defaults.set(id, forKey: Key.modeId)
|
||||
}
|
||||
|
||||
public func setLocaleId(_ id: String) {
|
||||
defaults.set(id, forKey: Key.localeId)
|
||||
}
|
||||
|
||||
public func setEngineMode(_ mode: String) {
|
||||
defaults.set(mode, forKey: Key.engineMode)
|
||||
}
|
||||
|
||||
public func setLocalASRBackend(_ backend: LocalASRBackend) {
|
||||
defaults.set(backend.rawValue, forKey: Key.localASRBackend)
|
||||
}
|
||||
|
||||
public func setUILanguage(_ language: AppUILanguage) {
|
||||
defaults.set(language.rawValue, forKey: Key.uiLanguage)
|
||||
}
|
||||
|
||||
/// v0.2.1 follow-up: kept for source compatibility with callers that
|
||||
/// still pass a Bool (e.g. older tests, any leftover bridge code).
|
||||
/// `enabled == true` selects `defaultLocaleId` ("en") as a sensible
|
||||
/// on-ramp target; `enabled == false` resets to `offLocaleId`.
|
||||
/// The keyboard chip / pipeline now write the locale id directly
|
||||
/// via `setTranslationTargetLocaleId`, which is the preferred path.
|
||||
public func setTranslationEnabled(_ enabled: Bool) {
|
||||
defaults.set(
|
||||
enabled ? TranslationLanguageCatalog.defaultLocaleId : TranslationLanguageCatalog.offLocaleId,
|
||||
forKey: Key.translationTargetLocaleId
|
||||
)
|
||||
}
|
||||
|
||||
/// v0.2.1: persist target locale id (e.g. `"en"`, `"ja"`, or
|
||||
/// `TranslationLanguageCatalog.offLocaleId`). The keyboard
|
||||
/// extension reads this on every `load()` and `refreshRuntimeFlags()`
|
||||
/// so the chip reflects the latest value without a host-app
|
||||
/// round-trip.
|
||||
public func setTranslationTargetLocaleId(_ id: String) {
|
||||
defaults.set(id, forKey: Key.translationTargetLocaleId)
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public func setHandednessPreference(_ preference: HandednessPreference) {
|
||||
defaults.set(preference.rawValue, forKey: Key.handednessPreference)
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public func setCursorDragNavigationEnabled(_ enabled: Bool) {
|
||||
defaults.set(enabled, forKey: Key.cursorDragNavigationEnabled)
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
/// Whether ASR output should be sent through the LLM polish step.
|
||||
/// Both engines always run polish after ASR completes (chunked
|
||||
/// pipeline stitches first). Ultra-short structure-free utterances
|
||||
/// may skip the LLM inside `PolishingService`.
|
||||
public var shouldRunCloudLLMStep: Bool { true }
|
||||
|
||||
/// Whether translate-and-polish should run (vs polish-only).
|
||||
public var isTranslationEffective: Bool {
|
||||
translationEnabled
|
||||
}
|
||||
public var providerId: String { configuration.providerId }
|
||||
public var baseURL: String { configuration.baseURL }
|
||||
public var apiKey: String { configuration.apiKey }
|
||||
public var model: String { configuration.model }
|
||||
public var modeId: String { configuration.modeId }
|
||||
public var localeId: String { configuration.localeId }
|
||||
public var engineMode: String { configuration.engineMode }
|
||||
public var uiLanguage: AppUILanguage { configuration.uiLanguage }
|
||||
public var translationEnabled: Bool { configuration.translationEnabled }
|
||||
public var translationTargetLocaleId: String { configuration.translationTargetLocaleId }
|
||||
public var handednessPreference: HandednessPreference { configuration.handednessPreference }
|
||||
public var cursorDragNavigationEnabled: Bool { configuration.cursorDragNavigationEnabled }
|
||||
public var polishIntensity: PolishIntensity { configuration.polishIntensity }
|
||||
public var isTranslationEffective: Bool { configuration.isTranslationEffective }
|
||||
public var isLocalEngine: Bool { configuration.isLocalEngine }
|
||||
public var polishModeForPipeline: PolishingService.PolishMode { configuration.polishModeForPipeline }
|
||||
public var polishProviderIdOverride: String? { configuration.polishProviderIdOverride }
|
||||
public var isCloudAPIKeyMissingForVoiceInput: Bool { configuration.isCloudAPIKeyMissingForVoiceInput }
|
||||
|
||||
/// Whether the keyboard top-bar translation chip should render.
|
||||
public var isTranslationChipVisible: Bool { true }
|
||||
|
||||
/// Cloud engine requires a provider-specific API key before the user
|
||||
/// can start voice input. Local engine uses the built-in DeepSeek path.
|
||||
public var isCloudAPIKeyMissingForVoiceInput: Bool {
|
||||
guard engineMode == "cloud" else { return false }
|
||||
return apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
// MARK: - Writes
|
||||
|
||||
public func setModeId(_ id: String) {
|
||||
mutateConfiguration { $0.modeId = id }
|
||||
}
|
||||
|
||||
/// Polish vs translate-and-polish for the active pipeline.
|
||||
public var polishModeForPipeline: PolishingService.PolishMode {
|
||||
isTranslationEffective
|
||||
? .translate(targetLocaleId: translationTargetLocaleId)
|
||||
: .polish
|
||||
public func setLocaleId(_ id: String) {
|
||||
mutateConfiguration { $0.localeId = id }
|
||||
}
|
||||
|
||||
/// Local engine pins the LLM step to DeepSeek; cloud uses the
|
||||
/// user's configured provider.
|
||||
public var polishProviderIdOverride: String? {
|
||||
engineMode == "local" ? "deepseek" : nil
|
||||
}
|
||||
|
||||
// MARK: - Polish settings (v0.3.0+)
|
||||
|
||||
/// How aggressively the LLM should rewrite the ASR transcript.
|
||||
/// Defaults to `medium` for new installs.
|
||||
public var polishIntensity: PolishIntensity {
|
||||
guard let raw = defaults.string(forKey: Key.polishIntensity) else {
|
||||
return .default
|
||||
public func setEngineMode(_ mode: String) {
|
||||
mutateConfiguration { config in
|
||||
config.engineMode = mode
|
||||
if mode == "cloud", config.providerId == "deepseek" {
|
||||
let openAI = LLMProvider.provider(id: "openai")
|
||||
config.providerId = openAI.id
|
||||
config.baseURL = openAI.defaultBaseURL
|
||||
config.model = openAI.defaultModel
|
||||
}
|
||||
}
|
||||
let resolved = PolishIntensity.resolve(storedRawValue: raw)
|
||||
if raw == PolishIntensity.legacyOffRawValue {
|
||||
defaults.set(resolved.rawValue, forKey: Key.polishIntensity)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
public func setUILanguage(_ language: AppUILanguage) {
|
||||
mutateConfiguration { $0.uiLanguage = language }
|
||||
}
|
||||
|
||||
public func setTranslationEnabled(_ enabled: Bool) {
|
||||
setTranslationTargetLocaleId(
|
||||
enabled ? TranslationLanguageCatalog.defaultLocaleId : TranslationLanguageCatalog.offLocaleId
|
||||
)
|
||||
}
|
||||
|
||||
public func setTranslationTargetLocaleId(_ id: String) {
|
||||
mutateConfiguration { $0.translationTargetLocaleId = id }
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public func setHandednessPreference(_ preference: HandednessPreference) {
|
||||
mutateConfiguration { $0.handednessPreference = preference }
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public func setCursorDragNavigationEnabled(_ enabled: Bool) {
|
||||
mutateConfiguration { $0.cursorDragNavigationEnabled = enabled }
|
||||
AppGroupConfigDarwin.postConfigChanged()
|
||||
}
|
||||
|
||||
public func setPolishIntensity(_ intensity: PolishIntensity) {
|
||||
defaults.set(intensity.rawValue, forKey: Key.polishIntensity)
|
||||
mutateConfiguration { $0.polishIntensity = intensity }
|
||||
}
|
||||
|
||||
// MARK: - Onboarding (v0.3.0+)
|
||||
//
|
||||
// Mirrored from `ProviderConfig` so the keyboard extension's
|
||||
// overlay can read / write the same source of truth without
|
||||
// instantiating the main-app config (which would drag in
|
||||
// SwiftUI / Combine and fight the keyboard's main-thread budget).
|
||||
|
||||
public var hasCompletedOnboarding: Bool {
|
||||
get { defaults.bool(forKey: "config.hasCompletedOnboarding") }
|
||||
set { defaults.set(newValue, forKey: "config.hasCompletedOnboarding") }
|
||||
get { configuration.hasCompletedOnboarding }
|
||||
set { setHasCompletedOnboarding(newValue) }
|
||||
}
|
||||
|
||||
public var onboardingPage: Int {
|
||||
get { defaults.integer(forKey: "config.onboardingPage") }
|
||||
set { defaults.set(newValue, forKey: "config.onboardingPage") }
|
||||
get { configuration.onboardingPage }
|
||||
set { setOnboardingPage(newValue) }
|
||||
}
|
||||
|
||||
public func setHasCompletedOnboarding(_ completed: Bool) {
|
||||
defaults.set(completed, forKey: "config.hasCompletedOnboarding")
|
||||
mutateConfiguration { config in
|
||||
config.hasCompletedOnboarding = completed
|
||||
if completed {
|
||||
config.onboardingPage = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func setOnboardingPage(_ page: Int) {
|
||||
defaults.set(page, forKey: "config.onboardingPage")
|
||||
mutateConfiguration { $0.onboardingPage = page }
|
||||
}
|
||||
|
||||
// MARK: - Detected app context (v0.3.0+)
|
||||
// MARK: - Detected app context
|
||||
|
||||
/// Last app context the keyboard extension detected for this
|
||||
/// user, plus the timestamp it was observed. Callers should
|
||||
/// treat values older than 30 minutes as stale.
|
||||
public var detectedAppContext: (context: AppContext, observedAt: Date)? {
|
||||
guard let raw = defaults.string(forKey: Key.detectedAppContext),
|
||||
let value = AppContext(rawValue: raw)
|
||||
else { return nil }
|
||||
let timestamp = defaults.object(forKey: Key.detectedAppContextAt) as? Date ?? .distantPast
|
||||
return (value, timestamp)
|
||||
configuration.detectedAppContext(from: defaults)
|
||||
}
|
||||
|
||||
public func setDetectedAppContext(_ context: AppContext, at date: Date = Date()) {
|
||||
defaults.set(context.rawValue, forKey: Key.detectedAppContext)
|
||||
defaults.set(date, forKey: Key.detectedAppContextAt)
|
||||
var config = configuration
|
||||
config.setDetectedAppContext(context, at: date, to: defaults)
|
||||
}
|
||||
|
||||
// MARK: - Personal dictionary (v0.3.0+)
|
||||
// MARK: - Personal dictionary
|
||||
|
||||
/// Personal dictionary persisted in the App Group so both the
|
||||
/// main app's Settings UI and the keyboard extension's LLM call
|
||||
/// read the same source of truth. Returns an empty dictionary
|
||||
/// when nothing is stored (and when the stored JSON is corrupt —
|
||||
/// failing closed is safer than crashing the keyboard).
|
||||
public var personalDictionary: PersonalDictionary {
|
||||
get {
|
||||
guard let data = defaults.data(forKey: Key.personalDictionary) else {
|
||||
return .empty
|
||||
}
|
||||
do {
|
||||
var dictionary = try JSONDecoder().decode(PersonalDictionary.self, from: data)
|
||||
if dictionary.entries.contains(where: { $0.source == .history }) {
|
||||
for index in dictionary.entries.indices where dictionary.entries[index].source == .history {
|
||||
dictionary.entries[index].source = .manual
|
||||
}
|
||||
dictionary.version += 1
|
||||
if let migrated = try? JSONEncoder().encode(dictionary) {
|
||||
defaults.set(migrated, forKey: Key.personalDictionary)
|
||||
}
|
||||
}
|
||||
return dictionary
|
||||
} catch {
|
||||
#if DEBUG
|
||||
print("⚠️ [AppGroupStore] personalDictionary decode failed: \(error)")
|
||||
#endif
|
||||
return .empty
|
||||
}
|
||||
}
|
||||
set {
|
||||
setPersonalDictionary(newValue)
|
||||
}
|
||||
get { configuration.personalDictionary }
|
||||
set { setPersonalDictionary(newValue) }
|
||||
}
|
||||
|
||||
public func setPersonalDictionary(_ dictionary: PersonalDictionary) {
|
||||
do {
|
||||
let data = try JSONEncoder().encode(dictionary)
|
||||
defaults.set(data, forKey: Key.personalDictionary)
|
||||
} catch {
|
||||
#if DEBUG
|
||||
print("⚠️ [AppGroupStore] personalDictionary encode failed: \(error)")
|
||||
#endif
|
||||
}
|
||||
mutateConfiguration { $0.personalDictionary = dictionary }
|
||||
}
|
||||
|
||||
// MARK: - Client
|
||||
|
||||
public func makeClient() -> LLMClient {
|
||||
OpenAICompatibleClient(
|
||||
baseURL: baseURL,
|
||||
apiKey: apiKey,
|
||||
model: model
|
||||
)
|
||||
configuration.makeClient()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
// CustomLanguageModelManager.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Prepares the bundled SFCustomLanguageModelData asset on device and shares
|
||||
// the compiled LM + Vocab through the App Group container. Both the host app
|
||||
// and keyboard extension read the same prepared configuration for
|
||||
// DictationTranscriber content hints.
|
||||
|
||||
import Foundation
|
||||
import Speech
|
||||
import os
|
||||
|
||||
public final class CustomLanguageModelManager: @unchecked Sendable {
|
||||
|
||||
public static let shared = CustomLanguageModelManager()
|
||||
|
||||
public enum PrepareState: Equatable, Sendable {
|
||||
case idle
|
||||
case preparing
|
||||
case ready
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct BundledManifest: Decodable {
|
||||
let version: String
|
||||
let bin_bytes: Int
|
||||
let identifier: String
|
||||
}
|
||||
|
||||
private enum Storage {
|
||||
static let subdirectory = "CustomLanguageModel/v1"
|
||||
static let fingerprintKey = "customLM.preparedFingerprint"
|
||||
static let preparedAtKey = "customLM.preparedAt"
|
||||
static let lastFailureAtKey = "customLM.lastFailureAt"
|
||||
static let attemptCountKey = "customLM.attemptCount"
|
||||
static let maxRetryAttempts = 3
|
||||
/// Backoff after failure attempts 1, 2, and 3 (seconds).
|
||||
static let backoffIntervals: [TimeInterval] = [30, 120, 600]
|
||||
}
|
||||
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var cachedConfiguration: SFSpeechLanguageModel.Configuration?
|
||||
private var state: PrepareState = .idle
|
||||
private var prepareTask: Task<Void, Never>?
|
||||
|
||||
private init() {}
|
||||
|
||||
// MARK: - Public API
|
||||
|
||||
/// Returns a prepared configuration for Chinese locales when available.
|
||||
public func configurationForTranscription(locale: Locale) -> SFSpeechLanguageModel.Configuration? {
|
||||
guard Self.isChineseLocale(locale) else { return nil }
|
||||
return lock.withLock { () -> SFSpeechLanguageModel.Configuration? in
|
||||
if let cachedConfiguration {
|
||||
return cachedConfiguration
|
||||
}
|
||||
if let loaded = Self.loadCachedConfigurationFromDisk() {
|
||||
cachedConfiguration = loaded
|
||||
state = .ready
|
||||
return loaded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
public func currentState() -> PrepareState {
|
||||
lock.withLock { state }
|
||||
}
|
||||
|
||||
/// Fire-and-forget preparation for the host app. Safe to call repeatedly.
|
||||
/// Retries after exponential backoff when a prior attempt failed.
|
||||
public func prepareInBackgroundIfNeeded() {
|
||||
guard AppGroup.isAvailable else { return }
|
||||
|
||||
let shouldStart = lock.withLock { () -> Bool in
|
||||
if case .preparing = state { return false }
|
||||
if cachedConfiguration != nil { return false }
|
||||
if let loaded = Self.loadCachedConfigurationFromDisk() {
|
||||
cachedConfiguration = loaded
|
||||
state = .ready
|
||||
Self.clearRetryState()
|
||||
return false
|
||||
}
|
||||
if prepareTask != nil { return false }
|
||||
|
||||
if case .failed = state {
|
||||
guard Self.canRetryAfterFailure() else { return false }
|
||||
} else if !Self.canRetryAfterFailure() {
|
||||
return false
|
||||
}
|
||||
|
||||
state = .preparing
|
||||
return true
|
||||
}
|
||||
guard shouldStart else { return }
|
||||
|
||||
prepareTask = Task.detached(priority: .utility) { [weak self] in
|
||||
guard let self else { return }
|
||||
defer {
|
||||
self.lock.withLock { self.prepareTask = nil }
|
||||
}
|
||||
do {
|
||||
_ = try await self.prepareIfNeeded()
|
||||
} catch {
|
||||
Self.recordFailure()
|
||||
self.lock.withLock {
|
||||
self.state = .failed(error.localizedDescription)
|
||||
}
|
||||
Self.log(
|
||||
"prepare failed (attempt \(Self.storedAttemptCount())): \(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepares the bundled training asset into the App Group container.
|
||||
@discardableResult
|
||||
public func prepareIfNeeded() async throws -> SFSpeechLanguageModel.Configuration? {
|
||||
if let existing = configurationForTranscription(locale: Locale(identifier: "zh-Hans")) {
|
||||
lock.withLock { state = .ready }
|
||||
Self.clearRetryState()
|
||||
return existing
|
||||
}
|
||||
|
||||
guard Self.canRetryAfterFailure() else {
|
||||
throw PrepareError.retryBudgetExhausted
|
||||
}
|
||||
|
||||
guard let manifest = Self.bundledManifest() else {
|
||||
throw PrepareError.missingManifest
|
||||
}
|
||||
guard let assetURL = Self.bundledTrainingAssetURL() else {
|
||||
throw PrepareError.missingTrainingAsset
|
||||
}
|
||||
guard let preparedDir = Self.preparedDirectoryURL() else {
|
||||
throw PrepareError.missingAppGroupContainer
|
||||
}
|
||||
|
||||
let fingerprint = Self.fingerprint(for: manifest)
|
||||
if Self.storedFingerprint() == fingerprint,
|
||||
let cached = Self.loadCachedConfigurationFromDisk() {
|
||||
lock.withLock {
|
||||
cachedConfiguration = cached
|
||||
state = .ready
|
||||
}
|
||||
Self.clearRetryState()
|
||||
return cached
|
||||
}
|
||||
|
||||
lock.withLock { state = .preparing }
|
||||
|
||||
let languageModelURL = preparedDir.appendingPathComponent("LM")
|
||||
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
|
||||
try Self.removeItemIfExists(at: languageModelURL)
|
||||
try Self.removeItemIfExists(at: vocabularyURL)
|
||||
|
||||
let configuration = SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL
|
||||
)
|
||||
|
||||
Self.log("preparing custom LM (\(manifest.bin_bytes) byte asset)…")
|
||||
try await Self.prepareLanguageModel(assetURL: assetURL, configuration: configuration)
|
||||
|
||||
guard FileManager.default.fileExists(atPath: languageModelURL.path),
|
||||
FileManager.default.fileExists(atPath: vocabularyURL.path) else {
|
||||
throw PrepareError.missingPreparedArtifacts
|
||||
}
|
||||
|
||||
AppGroup.defaultsIfAvailable?.set(fingerprint, forKey: Storage.fingerprintKey)
|
||||
AppGroup.defaultsIfAvailable?.set(Date().timeIntervalSince1970, forKey: Storage.preparedAtKey)
|
||||
Self.clearRetryState()
|
||||
|
||||
lock.withLock {
|
||||
cachedConfiguration = configuration
|
||||
state = .ready
|
||||
}
|
||||
|
||||
Self.log("custom LM ready at \(preparedDir.path)")
|
||||
return configuration
|
||||
}
|
||||
|
||||
// MARK: - DictationTranscriber factory
|
||||
|
||||
public static func makeDictationTranscriber(
|
||||
locale: Locale,
|
||||
lmConfiguration: SFSpeechLanguageModel.Configuration?
|
||||
) -> DictationTranscriber {
|
||||
let preset = DictationTranscriber.Preset.progressiveLongDictation
|
||||
guard let lmConfiguration, isChineseLocale(locale) else {
|
||||
return DictationTranscriber(locale: locale, preset: preset)
|
||||
}
|
||||
|
||||
let contentHints = preset.contentHints.union([
|
||||
.customizedLanguage(modelConfiguration: lmConfiguration),
|
||||
])
|
||||
return DictationTranscriber(
|
||||
locale: locale,
|
||||
contentHints: contentHints,
|
||||
transcriptionOptions: preset.transcriptionOptions,
|
||||
reportingOptions: preset.reportingOptions,
|
||||
attributeOptions: preset.attributeOptions
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Bundle / disk helpers
|
||||
|
||||
private static var resourceBundle: Bundle {
|
||||
Bundle(for: CustomLanguageModelManager.self)
|
||||
}
|
||||
|
||||
static func bundledTrainingAssetURL() -> URL? {
|
||||
if let url = resourceBundle.url(
|
||||
forResource: "OSGKeyboardCLM",
|
||||
withExtension: "bin",
|
||||
subdirectory: Storage.subdirectory
|
||||
) {
|
||||
return url
|
||||
}
|
||||
return resourceBundle.url(forResource: "OSGKeyboardCLM", withExtension: "bin")
|
||||
}
|
||||
|
||||
static func bundledManifest() -> BundledManifest? {
|
||||
let manifestURL =
|
||||
resourceBundle.url(
|
||||
forResource: "compiled-manifest",
|
||||
withExtension: "json",
|
||||
subdirectory: Storage.subdirectory
|
||||
)
|
||||
?? resourceBundle.url(forResource: "compiled-manifest", withExtension: "json")
|
||||
guard let manifestURL,
|
||||
let data = try? Data(contentsOf: manifestURL),
|
||||
let manifest = try? JSONDecoder().decode(BundledManifest.self, from: data)
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
static func preparedDirectoryURL() -> URL? {
|
||||
guard let container = FileManager.default.containerURL(
|
||||
forSecurityApplicationGroupIdentifier: AppGroup.identifier
|
||||
) else {
|
||||
return nil
|
||||
}
|
||||
let directory = container.appendingPathComponent(Storage.subdirectory, isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
return directory
|
||||
}
|
||||
|
||||
static func loadCachedConfigurationFromDisk() -> SFSpeechLanguageModel.Configuration? {
|
||||
guard let manifest = bundledManifest(),
|
||||
storedFingerprint() == fingerprint(for: manifest),
|
||||
let preparedDir = preparedDirectoryURL()
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let languageModelURL = preparedDir.appendingPathComponent("LM")
|
||||
let vocabularyURL = preparedDir.appendingPathComponent("Vocab")
|
||||
let fm = FileManager.default
|
||||
guard fm.fileExists(atPath: languageModelURL.path),
|
||||
fm.fileExists(atPath: vocabularyURL.path) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL
|
||||
)
|
||||
}
|
||||
|
||||
static func isChineseLocale(_ locale: Locale) -> Bool {
|
||||
locale.identifier(.bcp47).lowercased().hasPrefix("zh")
|
||||
}
|
||||
|
||||
private static func fingerprint(for manifest: BundledManifest) -> String {
|
||||
"\(manifest.identifier)|\(manifest.version)|\(manifest.bin_bytes)"
|
||||
}
|
||||
|
||||
private static func storedFingerprint() -> String? {
|
||||
AppGroup.defaultsIfAvailable?.string(forKey: Storage.fingerprintKey)
|
||||
}
|
||||
|
||||
private static func removeItemIfExists(at url: URL) throws {
|
||||
let fm = FileManager.default
|
||||
if fm.fileExists(atPath: url.path) {
|
||||
try fm.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
|
||||
private static func prepareLanguageModel(
|
||||
assetURL: URL,
|
||||
configuration: SFSpeechLanguageModel.Configuration
|
||||
) async throws {
|
||||
try await withCheckedThrowingContinuation {
|
||||
(continuation: CheckedContinuation<Void, Error>) in
|
||||
SFSpeechLanguageModel.prepareCustomLanguageModel(
|
||||
for: assetURL,
|
||||
configuration: configuration
|
||||
) { error in
|
||||
if let error {
|
||||
continuation.resume(throwing: error)
|
||||
} else {
|
||||
continuation.resume()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Retry / backoff
|
||||
|
||||
private static func storedAttemptCount() -> Int {
|
||||
AppGroup.defaultsIfAvailable?.integer(forKey: Storage.attemptCountKey) ?? 0
|
||||
}
|
||||
|
||||
private static func storedLastFailureAt() -> TimeInterval? {
|
||||
let value = AppGroup.defaultsIfAvailable?.double(forKey: Storage.lastFailureAtKey) ?? 0
|
||||
return value > 0 ? value : nil
|
||||
}
|
||||
|
||||
private static func recordFailure() {
|
||||
guard let defaults = AppGroup.defaultsIfAvailable else { return }
|
||||
let nextAttempt = storedAttemptCount() + 1
|
||||
defaults.set(nextAttempt, forKey: Storage.attemptCountKey)
|
||||
defaults.set(Date().timeIntervalSince1970, forKey: Storage.lastFailureAtKey)
|
||||
}
|
||||
|
||||
private static func clearRetryState() {
|
||||
guard let defaults = AppGroup.defaultsIfAvailable else { return }
|
||||
defaults.removeObject(forKey: Storage.attemptCountKey)
|
||||
defaults.removeObject(forKey: Storage.lastFailureAtKey)
|
||||
}
|
||||
|
||||
/// Returns false when retry budget is exhausted or backoff has not elapsed.
|
||||
private static func canRetryAfterFailure() -> Bool {
|
||||
let attempts = storedAttemptCount()
|
||||
guard attempts > 0 else { return true }
|
||||
guard attempts <= Storage.maxRetryAttempts else { return false }
|
||||
|
||||
guard let lastFailureAt = storedLastFailureAt() else { return true }
|
||||
let backoffIndex = min(attempts - 1, Storage.backoffIntervals.count - 1)
|
||||
let requiredDelay = Storage.backoffIntervals[backoffIndex]
|
||||
let elapsed = Date().timeIntervalSince1970 - lastFailureAt
|
||||
return elapsed >= requiredDelay
|
||||
}
|
||||
|
||||
private static func log(_ message: String) {
|
||||
OSGLog.clm.info("\(message, privacy: .public)")
|
||||
}
|
||||
|
||||
enum PrepareError: LocalizedError {
|
||||
case missingManifest
|
||||
case missingTrainingAsset
|
||||
case missingAppGroupContainer
|
||||
case missingPreparedArtifacts
|
||||
case retryBudgetExhausted
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missingManifest:
|
||||
return "Missing bundled custom language model manifest."
|
||||
case .missingTrainingAsset:
|
||||
return "Missing bundled custom language model training asset."
|
||||
case .missingAppGroupContainer:
|
||||
return "App Group container unavailable for custom language model preparation."
|
||||
case .missingPreparedArtifacts:
|
||||
return "Custom language model preparation did not produce LM/Vocab artifacts."
|
||||
case .retryBudgetExhausted:
|
||||
return "Custom language model preparation retry budget exhausted."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
// DictationBridge.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Lightweight App Group bridge for host-app dictation handoff:
|
||||
// keyboard extension -> open host app for recording
|
||||
// host app -> writes final transcript
|
||||
// keyboard extension -> consumes pending transcript and inserts text
|
||||
//
|
||||
// STATUS (v0.1.2): Retained. Consumed by `KeyboardViewController` for
|
||||
// the "one-shot" host-app dictation path (where the keyboard extension
|
||||
// launches the host app, the user records there, and the resulting
|
||||
// text is consumed back by the extension). The *continuous* path goes
|
||||
// through `FlowSessionBridge` + `FlowSessionManager` instead.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum DictationBridge {
|
||||
public enum Status: String, Sendable, Equatable {
|
||||
case idle
|
||||
case requested
|
||||
case recording
|
||||
case transcribing
|
||||
case done
|
||||
case cancelled
|
||||
case error
|
||||
}
|
||||
|
||||
private enum Key {
|
||||
static let pendingText = "dictation.pendingText"
|
||||
static let polishWarning = "dictation.polishWarning"
|
||||
static let updatedAt = "dictation.updatedAt"
|
||||
static let status = "dictation.status"
|
||||
static let statusUpdatedAt = "dictation.statusUpdatedAt"
|
||||
static let statusMessage = "dictation.statusMessage"
|
||||
}
|
||||
|
||||
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
|
||||
if let defaults {
|
||||
return defaults
|
||||
}
|
||||
return AppGroup.isAvailable ? AppGroup.defaults : .standard
|
||||
}
|
||||
|
||||
public static func setStatus(
|
||||
_ status: Status,
|
||||
message: String? = nil,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(status.rawValue, forKey: Key.status)
|
||||
store.set(Date().timeIntervalSince1970, forKey: Key.statusUpdatedAt)
|
||||
if let message, !message.isEmpty {
|
||||
store.set(message, forKey: Key.statusMessage)
|
||||
} else {
|
||||
store.removeObject(forKey: Key.statusMessage)
|
||||
}
|
||||
}
|
||||
|
||||
public static func currentStatus(
|
||||
defaults: UserDefaults? = nil
|
||||
) -> (status: Status, message: String?, updatedAt: TimeInterval) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
let raw = store.string(forKey: Key.status) ?? Status.idle.rawValue
|
||||
let status = Status(rawValue: raw) ?? .idle
|
||||
let message = store.string(forKey: Key.statusMessage)
|
||||
let updatedAt = store.double(forKey: Key.statusUpdatedAt)
|
||||
return (status, message, updatedAt)
|
||||
}
|
||||
|
||||
public static func markRequested(defaults: UserDefaults? = nil) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.removeObject(forKey: Key.pendingText)
|
||||
setStatus(.requested, defaults: store)
|
||||
}
|
||||
|
||||
/// Store a transcript for the keyboard extension to consume.
|
||||
public static func storePendingTranscript(
|
||||
_ text: String,
|
||||
polishWarning: String? = nil,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(trimmed, forKey: Key.pendingText)
|
||||
store.set(Date().timeIntervalSince1970, forKey: Key.updatedAt)
|
||||
if let polishWarning, !polishWarning.isEmpty {
|
||||
store.set(polishWarning, forKey: Key.polishWarning)
|
||||
} else {
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
}
|
||||
setStatus(.done, defaults: store)
|
||||
}
|
||||
|
||||
/// Returns and clears the pending transcript if present.
|
||||
public static func consumePendingTranscript(
|
||||
maxAge: TimeInterval = 180,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> String? {
|
||||
consumePendingDelivery(maxAge: maxAge, defaults: defaults)?.text
|
||||
}
|
||||
|
||||
/// Returns and clears the pending delivery (text + optional polish
|
||||
/// warning) if present.
|
||||
public static func consumePendingDelivery(
|
||||
maxAge: TimeInterval = 180,
|
||||
defaults: UserDefaults? = nil
|
||||
) -> TranscriptionDelivery? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard let text = store.string(forKey: Key.pendingText) else {
|
||||
return nil
|
||||
}
|
||||
if maxAge > 0 {
|
||||
let ts = store.double(forKey: Key.updatedAt)
|
||||
if ts > 0, Date().timeIntervalSince1970 - ts > maxAge {
|
||||
clear(defaults: store)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
let warning = store.string(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.pendingText)
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.updatedAt)
|
||||
setStatus(.idle, defaults: store)
|
||||
return TranscriptionDelivery(text: text, polishWarning: warning)
|
||||
}
|
||||
|
||||
public static func clear(defaults: UserDefaults? = nil) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.removeObject(forKey: Key.pendingText)
|
||||
store.removeObject(forKey: Key.polishWarning)
|
||||
store.removeObject(forKey: Key.updatedAt)
|
||||
setStatus(.idle, defaults: store)
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
// FlowAppLifecycle.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Tracks whether the host app process is in the foreground.
|
||||
// Retained for any future GPU-backed paths; CoreML ASR does not require it.
|
||||
|
||||
import Foundation
|
||||
|
||||
public final class FlowAppLifecycle: @unchecked Sendable {
|
||||
|
||||
public static let shared = FlowAppLifecycle()
|
||||
|
||||
private let lock = NSLock()
|
||||
private var isForeground = true
|
||||
|
||||
private init() {}
|
||||
|
||||
/// `true` when the host app scene is active (`.active`).
|
||||
public var allowsGPUInference: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return isForeground
|
||||
}
|
||||
|
||||
public func setForeground(_ foreground: Bool) {
|
||||
lock.lock()
|
||||
isForeground = foreground
|
||||
lock.unlock()
|
||||
}
|
||||
|
||||
/// Blocks until foreground or cancellation.
|
||||
public func waitUntilForeground() async -> Bool {
|
||||
while !allowsGPUInference {
|
||||
if Task.isCancelled { return false }
|
||||
try? await Task.sleep(nanoseconds: 200_000_000)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -2,15 +2,31 @@
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// TypeWhisper-style Flow session bridge: keyboard writes recording
|
||||
// signals; host app writes transcription results. Legacy one-shot
|
||||
// dictation handoff remains in `DictationBridge`.
|
||||
// signals; host app writes transcription results.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct FlowTranscriptionError: Equatable, Sendable {
|
||||
public let message: String
|
||||
public let kind: FlowSessionKeys.TranscriptionErrorKind
|
||||
|
||||
public init(message: String, kind: FlowSessionKeys.TranscriptionErrorKind) {
|
||||
self.message = message
|
||||
self.kind = kind
|
||||
}
|
||||
}
|
||||
|
||||
public enum FlowSessionBridge {
|
||||
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
|
||||
if let defaults { return defaults }
|
||||
return AppGroup.isAvailable ? AppGroup.defaults : .standard
|
||||
guard let available = AppGroup.defaultsIfAvailable else {
|
||||
#if DEBUG
|
||||
fatalError("App Group unavailable — inject UserDefaults in tests or fix entitlements.")
|
||||
#else
|
||||
fatalError("App Group unavailable.")
|
||||
#endif
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/// Force cross-process visibility. Must only be called on the main thread.
|
||||
@@ -70,7 +86,6 @@ public enum FlowSessionBridge {
|
||||
/// background while the continuous audio session is frozen.
|
||||
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
|
||||
|
||||
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
|
||||
@@ -81,7 +96,6 @@ public enum FlowSessionBridge {
|
||||
/// actively processing). Used for auto-start heuristics, not gating record.
|
||||
public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard isSessionActive(defaults: store) else { return false }
|
||||
|
||||
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
|
||||
@@ -144,6 +158,7 @@ public enum FlowSessionBridge {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
if let polishWarning, !polishWarning.isEmpty {
|
||||
store.set(polishWarning, forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
} else {
|
||||
@@ -151,16 +166,46 @@ public enum FlowSessionBridge {
|
||||
}
|
||||
setRecordingState(.idle, defaults: store)
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Host app: publish pipelined ASR partial while recording or finalizing.
|
||||
public static func storeTranscriptionPartial(
|
||||
_ text: String,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let store = resolvedDefaults(defaults)
|
||||
if trimmed.isEmpty {
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
} else {
|
||||
store.set(trimmed, forKey: FlowSessionKeys.transcriptionPartial)
|
||||
}
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Keyboard: read the latest partial without clearing it.
|
||||
public static func transcriptionPartial(defaults: UserDefaults? = nil) -> String? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard let text = store.string(forKey: FlowSessionKeys.transcriptionPartial),
|
||||
!text.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
public static func storeTranscriptionError(
|
||||
_ message: String,
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind = .generic,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(message, forKey: FlowSessionKeys.transcriptionError)
|
||||
store.set(kind.rawValue, forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
setRecordingState(.idle, defaults: store)
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription result, if any.
|
||||
@@ -174,7 +219,6 @@ public enum FlowSessionBridge {
|
||||
defaults: UserDefaults? = nil
|
||||
) -> TranscriptionDelivery? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
@@ -186,20 +230,21 @@ public enum FlowSessionBridge {
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription error, if any.
|
||||
public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> String? {
|
||||
public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> FlowTranscriptionError? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard let message = store.string(forKey: FlowSessionKeys.transcriptionError), !message.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
let kindRaw = store.string(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
let kind = FlowSessionKeys.TranscriptionErrorKind(rawValue: kindRaw ?? "") ?? .generic
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
flush(store)
|
||||
return message
|
||||
return FlowTranscriptionError(message: message, kind: kind)
|
||||
}
|
||||
|
||||
public static func audioLevels(defaults: UserDefaults? = nil) -> [Float] {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [Double], !levels.isEmpty {
|
||||
return levels.map { Float($0) }
|
||||
}
|
||||
@@ -240,7 +285,9 @@ public enum FlowSessionBridge {
|
||||
|
||||
private static func clearTranscription(defaults: UserDefaults) {
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import Foundation
|
||||
|
||||
public enum FlowSessionDarwin {
|
||||
public static let notificationName = "com.osgkeyboard.flow.session.changed"
|
||||
/// Posted when the host app writes a transcription result or error.
|
||||
public static let transcriptionNotificationName = "com.osgkeyboard.flow.transcription.changed"
|
||||
|
||||
public static func postSessionChanged() {
|
||||
CFNotificationCenterPostNotification(
|
||||
@@ -18,6 +20,16 @@ public enum FlowSessionDarwin {
|
||||
true
|
||||
)
|
||||
}
|
||||
|
||||
public static func postTranscriptionChanged() {
|
||||
CFNotificationCenterPostNotification(
|
||||
CFNotificationCenterGetDarwinNotifyCenter(),
|
||||
CFNotificationName(transcriptionNotificationName as CFString),
|
||||
nil,
|
||||
nil,
|
||||
true
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Observes Darwin notifications on a background thread; invokes
|
||||
|
||||
@@ -13,9 +13,13 @@ public enum FlowSessionKeys {
|
||||
public static let keyboardRecordingState = "flow.keyboardRecordingState"
|
||||
public static let transcriptionLanguage = "flow.transcriptionLanguage"
|
||||
public static let transcriptionResult = "flow.transcriptionResult"
|
||||
/// Live pipelined ASR partial for the keyboard transcript line.
|
||||
public static let transcriptionPartial = "flow.transcriptionPartial"
|
||||
/// Soft warning when polish failed but raw transcript was delivered.
|
||||
public static let transcriptionPolishWarning = "flow.transcriptionPolishWarning"
|
||||
public static let transcriptionError = "flow.transcriptionError"
|
||||
/// Structured kind paired with `transcriptionError` for keyboard UI.
|
||||
public static let transcriptionErrorKind = "flow.transcriptionErrorKind"
|
||||
public static let audioLevels = "flow.audioLevels"
|
||||
|
||||
/// Heartbeat older than this while the host is foreground → likely killed.
|
||||
@@ -36,15 +40,7 @@ public enum FlowSessionKeys {
|
||||
/// Keyboard watchdog after the user stops recording (not utterance max length).
|
||||
/// Must cover worst-case post-stop backlog: remaining SpeechAnalyzer chunks
|
||||
/// plus cloud LLM polish (see `PolishingService.effectiveTimeout` cap).
|
||||
///
|
||||
/// As of v0.2.0 the local engine uses iOS `SpeechAnalyzer` only, so the
|
||||
/// previous Qwen3-specific timeout (240 s) collapses into the shared
|
||||
/// local path. We keep `localASRBackend` on the signature for symmetry
|
||||
/// with other shared helpers.
|
||||
public static func keyboardResultTimeout(
|
||||
engineMode: String,
|
||||
localASRBackend: LocalASRBackend
|
||||
) -> TimeInterval {
|
||||
public static func keyboardResultTimeout(engineMode: String) -> TimeInterval {
|
||||
if engineMode == "local" {
|
||||
return 180
|
||||
}
|
||||
@@ -58,4 +54,13 @@ public enum FlowSessionKeys {
|
||||
case processing
|
||||
case aborted
|
||||
}
|
||||
|
||||
/// Structured host → keyboard transcription failure kind.
|
||||
public enum TranscriptionErrorKind: String, Sendable, Equatable {
|
||||
case noSpeech
|
||||
case recognitionInterrupted
|
||||
case audioUnavailable
|
||||
case asrFailed
|
||||
case generic
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,26 +35,37 @@ public final class KeyboardState: ObservableObject {
|
||||
case asr(String)
|
||||
case llm(LLMError)
|
||||
case appGroupUnavailable
|
||||
/// Keyboard extension lacks Full Access for host-app jumps.
|
||||
case fullAccessRequired
|
||||
/// Auto-jump to the host app failed; user must open it manually.
|
||||
case manualOpenRequired
|
||||
/// Host delivered raw transcript; polish step failed or was skipped.
|
||||
case polishDegraded(String)
|
||||
/// Host ASR finished with no usable speech.
|
||||
case noSpeechDetected
|
||||
/// Host ASR was interrupted before a final transcript arrived.
|
||||
case recognitionInterrupted
|
||||
/// Host could not start background audio capture.
|
||||
case hostAudioUnavailable
|
||||
/// Host ASR or pipeline failed with a user-facing message.
|
||||
case hostTranscriptionFailed(String)
|
||||
/// Flow result did not arrive before the keyboard watchdog expired.
|
||||
case flowResultTimeout
|
||||
/// Host Flow session ended while the keyboard was idle.
|
||||
case flowSessionExpired
|
||||
case unknown(String)
|
||||
}
|
||||
|
||||
public enum Reason: Equatable { case mic, speech }
|
||||
}
|
||||
|
||||
/// Voice input always runs through polish; legacy off/transcribe modes removed.
|
||||
public enum InputMode: String, CaseIterable, Identifiable {
|
||||
case off
|
||||
case transcribe
|
||||
case polish
|
||||
|
||||
public var id: String { rawValue }
|
||||
|
||||
public var labelKey: String {
|
||||
switch self {
|
||||
case .off: return "mode.off"
|
||||
case .transcribe: return "mode.transcribe"
|
||||
case .polish: return "mode.polish"
|
||||
}
|
||||
}
|
||||
public var labelKey: String { "mode.polish" }
|
||||
}
|
||||
|
||||
@Published public var phase: Phase = .idle
|
||||
@@ -78,20 +89,6 @@ public final class KeyboardState: ObservableObject {
|
||||
@Published public var micDisabledHint: String = ""
|
||||
/// "local" → on-device ASR only. "cloud" → ASR + LLM polish.
|
||||
@Published public var engineMode: String = "cloud"
|
||||
/// Which on-device ASR engine to use when `engineMode == "local"`.
|
||||
/// Mirrored from `ProviderConfig.localASRBackend` for UI display
|
||||
/// and for `state` consumers that want a single source of truth.
|
||||
@Published public var localASRBackend: LocalASRBackend = .speechAnalyzer
|
||||
/// v0.2.0: kept for source compatibility with the previous Qwen3
|
||||
/// CoreML local engine. Always `true` now — iOS `SpeechAnalyzer`
|
||||
/// ships with iOS 26 and has no per-user weights to download or
|
||||
/// preload. Existing read sites will see `true` and behave the
|
||||
/// same as the "stack ready" branch did.
|
||||
@Published public var localModelsReady: Bool = true
|
||||
/// v0.2.0: kept for source compatibility with the previous Qwen3
|
||||
/// CoreML local engine. Always `false` now — there are no weights
|
||||
/// for the host app to preload.
|
||||
@Published public var localModelsLoaded: Bool = false
|
||||
/// v0.2.1 follow-up: derived — translation is on iff a target
|
||||
/// locale has been selected (mirrors `ProviderConfig.translationEnabled`
|
||||
/// so the chip / pipeline read the same source of truth).
|
||||
@@ -103,9 +100,6 @@ public final class KeyboardState: ObservableObject {
|
||||
/// Defaults to `offLocaleId` so the keyboard boots in the "off"
|
||||
/// state on first install.
|
||||
@Published public var translationTargetLocaleId: String = TranslationLanguageCatalog.offLocaleId
|
||||
/// v0.2.0: mirrored from App Group — kept for source compatibility.
|
||||
/// Local engine always runs built-in polish; the flag is ignored.
|
||||
@Published public var localModeCloudPolishEnabled: Bool = true
|
||||
/// Mirrored from App Group — swaps delete / return on the bottom row.
|
||||
@Published public var handednessPreference: HandednessPreference = .left
|
||||
/// Press-and-drag pads beside the mic for four-way caret movement.
|
||||
@@ -159,7 +153,6 @@ public final class KeyboardState: ObservableObject {
|
||||
public var setMode: (InputMode) -> Void = { _ in }
|
||||
public var setLocale: (String) -> Void = { _ in }
|
||||
public var setEngineMode: (String) -> Void = { _ in }
|
||||
public var setLocalASRBackend: (LocalASRBackend) -> Void = { _ in }
|
||||
/// v0.2.1 follow-up: only the locale picker remains — `enabled`
|
||||
/// is derived from the locale id, so there's no separate toggle to
|
||||
/// persist. Wired in `KeyboardViewController.installStateActions`.
|
||||
@@ -209,4 +202,20 @@ public final class KeyboardState: ObservableObject {
|
||||
return s
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
extension KeyboardState.Phase.ErrorKind {
|
||||
/// Maps a host-app Flow transcription failure into a keyboard error kind.
|
||||
public static func fromFlowTranscription(_ error: FlowTranscriptionError) -> Self {
|
||||
switch error.kind {
|
||||
case .noSpeech:
|
||||
return .noSpeechDetected
|
||||
case .recognitionInterrupted:
|
||||
return .recognitionInterrupted
|
||||
case .audioUnavailable:
|
||||
return .hostAudioUnavailable
|
||||
case .asrFailed, .generic:
|
||||
return .hostTranscriptionFailed(error.message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
// This class is still imported by:
|
||||
// - `OSGKeyboard/Views/PreviewASRController.swift` (typealias)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (in-app preview)
|
||||
// - `OSGKeyboard/Views/DictationCaptureView.swift` (host-app fallback)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (host-app ASR preview)
|
||||
// - `OSGKeyboardTests/PreviewASRControllerStateTests.swift`
|
||||
//
|
||||
// Do NOT remove without updating those call sites. The earlier
|
||||
@@ -104,14 +104,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
private var didInstallTap = false
|
||||
|
||||
public init(asr: ASRService? = nil) {
|
||||
// Resolve through the factory so the user's `LocalASRBackend`
|
||||
// selection is honoured. Tests can pass a stub `asr` directly
|
||||
// to bypass the factory and exercise the controller in
|
||||
// isolation.
|
||||
self.asr = asr ?? ASRServiceFactory.make(
|
||||
engineMode: ProviderConfig.shared.engineMode,
|
||||
localBackend: ProviderConfig.shared.localASRBackend
|
||||
)
|
||||
self.asr = asr ?? ASRServiceFactory.make()
|
||||
}
|
||||
|
||||
/// Start dictation using a persisted settings locale id (`auto`, `zh-Hans`, …).
|
||||
|
||||
Reference in New Issue
Block a user