rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(asr): speed up local Flow dictation and land CLM/keyboard refactor

Browse Source 
Reduce perceived latency from key release to final text:
- Adaptive chunking: 2.5s first chunk + 5s follow-ups so short
  utterances start on-device recognition while still recording.
- Session-level ASR warmup and audio-format cache reuse to remove
  per-utterance cold-start of SpeechAnalyzer.
- Mirror live pipelined partials to the keyboard transcript line via
  a new flow.transcriptionPartial App Group key + Darwin ping.

Also commits the accumulated custom language model, Flow session,
keyboard extension restructure, and Xiaomi MiMo provider work in
progress on this branch.
This commit is contained in:
Rocky
2026-07-06 00:00:19 +08:00
parent cfbfb542cc
commit 537a68552a
76 changed files with 3456 additions and 121086 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardTests/AppGroupConfigurationTests.swift
+109
View File
@@ -0,0 +1,109 @@
// AppGroupConfigurationTests.swift
// OSGKeyboardTests
import XCTest
@testable import OSGKeyboardShared
final class AppGroupConfigurationTests: XCTestCase {
private func makeDefaults() -> UserDefaults {
let suite = "group.com.osgkeyboard.shared.tests.config.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suite)!
defaults.removePersistentDomain(forName: suite)
return defaults
}
func testLoadDefaultsWhenSuiteIsEmpty() {
let defaults = makeDefaults()
let config = AppGroupConfiguration.load(fromAvailable: defaults)
XCTAssertEqual(config.providerId, "openai")
XCTAssertEqual(config.modeId, "polish")
XCTAssertEqual(config.localeId, "auto")
XCTAssertEqual(config.engineMode, "cloud")
XCTAssertFalse(config.hasCompletedOnboarding)
XCTAssertEqual(config.onboardingPage, 0)
XCTAssertFalse(config.hasAcknowledgedCloudSharing)
XCTAssertEqual(config.translationTargetLocaleId, TranslationLanguageCatalog.offLocaleId)
XCTAssertFalse(config.translationEnabled)
XCTAssertEqual(config.handednessPreference, .left)
XCTAssertTrue(config.cursorDragNavigationEnabled)
XCTAssertEqual(config.polishIntensity, .default)
XCTAssertTrue(config.personalDictionary.entries.isEmpty)
}
func testSaveAndLoadRoundTrip() {
let defaults = makeDefaults()
var config = AppGroupConfiguration.load(fromAvailable: defaults)
config.providerId = "anthropic"
config.baseURL = "https://example.com/v1"
config.model = "claude-test"
config.modeId = "polish"
config.localeId = "zh-Hans"
config.engineMode = "local"
config.hasCompletedOnboarding = true
config.onboardingPage = 2
config.hasAcknowledgedCloudSharing = true
config.uiLanguage = .chinese
config.translationTargetLocaleId = "en"
config.handednessPreference = .right
config.cursorDragNavigationEnabled = false
config.polishIntensity = .light
config.save(to: defaults)
let loaded = AppGroupConfiguration.load(fromAvailable: defaults)
XCTAssertEqual(loaded.providerId, "anthropic")
XCTAssertEqual(loaded.baseURL, "https://example.com/v1")
XCTAssertEqual(loaded.model, "claude-test")
XCTAssertEqual(loaded.localeId, "zh-Hans")
XCTAssertEqual(loaded.engineMode, "local")
XCTAssertTrue(loaded.hasCompletedOnboarding)
XCTAssertEqual(loaded.onboardingPage, 2)
XCTAssertTrue(loaded.hasAcknowledgedCloudSharing)
XCTAssertEqual(loaded.uiLanguage, .chinese)
XCTAssertEqual(loaded.translationTargetLocaleId, "en")
XCTAssertTrue(loaded.translationEnabled)
XCTAssertEqual(loaded.handednessPreference, .right)
XCTAssertFalse(loaded.cursorDragNavigationEnabled)
XCTAssertEqual(loaded.polishIntensity, .light)
}
func testTranslationEnabledDerivedFromTargetLocale() {
let defaults = makeDefaults()
var config = AppGroupConfiguration.load(fromAvailable: defaults)
XCTAssertFalse(config.translationEnabled)
config.translationTargetLocaleId = "ja"
XCTAssertTrue(config.translationEnabled)
config.translationTargetLocaleId = TranslationLanguageCatalog.offLocaleId
XCTAssertFalse(config.translationEnabled)
}
func testCloudDeepSeekProviderMigratesToOpenAI() {
let defaults = makeDefaults()
defaults.set("deepseek", forKey: AppGroupConfiguration.Keys.providerId)
defaults.set("cloud", forKey: AppGroupConfiguration.Keys.engineMode)
let config = AppGroupConfiguration.load(fromAvailable: defaults)
XCTAssertEqual(config.providerId, "openai")
XCTAssertEqual(defaults.string(forKey: AppGroupConfiguration.Keys.providerId), "openai")
}
func testPolishIntensityLegacyOffMigratesToMedium() {
let defaults = makeDefaults()
defaults.set(PolishIntensity.legacyOffRawValue, forKey: AppGroupConfiguration.Keys.polishIntensity)
let config = AppGroupConfiguration.load(fromAvailable: defaults)
XCTAssertEqual(config.polishIntensity, .medium)
XCTAssertEqual(defaults.string(forKey: AppGroupConfiguration.Keys.polishIntensity), PolishIntensity.medium.rawValue)
}
func testLoadFromNilUsesAppGroupWhenAvailable() {
if AppGroup.defaultsIfAvailable != nil {
XCTAssertNotNil(AppGroupConfiguration.load(from: nil))
} else {
XCTAssertNil(AppGroupConfiguration.load(from: nil))
}
}
}
OSGKeyboardTests/ChunkedUtterancePipelineTests.swift
+8 -7
View File
@@ -47,10 +47,11 @@ final class ChunkedUtterancePipelineTests: XCTestCase {
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.finish()
var partials: [String] = []
let partialsLock = OSAllocatedUnfairLock(initialState: [String]())
let outcome = await pipeline.transcribe(stream: stream) { partial in
partials.append(partial)
partialsLock.withLock { $0.append(partial) }
}
let partials = partialsLock.withLock { $0 }
guard case .success(let success) = outcome else {
return XCTFail("expected success, got \(outcome)")
@@ -89,8 +90,7 @@ final class ChunkedUtterancePipelineTests: XCTestCase {
}
private struct FailingSecondChunkASR: ASRService, @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var index = 0
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
@@ -103,9 +103,10 @@ private struct FailingSecondChunkASR: ASRService, @unchecked Sendable {
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = lock.withLock {
defer { index += 1 }
return index
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 1 {
return .failure("simulated chunk error")
OSGKeyboardTests/DictationBridgeTests.swift
-59
View File
@@ -1,59 +0,0 @@
// DictationBridgeTests.swift
// OSGKeyboardTests
import XCTest
@testable import OSGKeyboardShared
final class DictationBridgeTests: XCTestCase {
private func makeDefaults() -> UserDefaults {
let suite = "group.com.osgkeyboard.shared.tests.dictation.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suite)!
defaults.removePersistentDomain(forName: suite)
return defaults
}
func testStoreAndConsumeTranscript() {
let defaults = makeDefaults()
DictationBridge.storePendingTranscript(" hello ", defaults: defaults)
let consumed = DictationBridge.consumePendingTranscript(defaults: defaults)
XCTAssertEqual(consumed, "hello")
XCTAssertNil(DictationBridge.consumePendingTranscript(defaults: defaults))
}
func testConsumeIgnoresExpiredTranscript() {
let defaults = makeDefaults()
DictationBridge.storePendingTranscript("stale", defaults: defaults)
// maxAge = 1ms, then delay to force expiry
usleep(2_000)
let consumed = DictationBridge.consumePendingTranscript(maxAge: 0.001, defaults: defaults)
XCTAssertNil(consumed)
}
func testStatusLifecycle() {
let defaults = makeDefaults()
DictationBridge.markRequested(defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .requested)
DictationBridge.setStatus(.recording, defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .recording)
DictationBridge.storePendingTranscript("ok", defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .done)
_ = DictationBridge.consumePendingTranscript(defaults: defaults)
XCTAssertEqual(DictationBridge.currentStatus(defaults: defaults).status, .idle)
}
func testStatusMessageAndTimestamp() {
let defaults = makeDefaults()
DictationBridge.setStatus(.error, message: "fail", defaults: defaults)
let snapshot = DictationBridge.currentStatus(defaults: defaults)
XCTAssertEqual(snapshot.status, .error)
XCTAssertEqual(snapshot.message, "fail")
XCTAssertGreaterThan(snapshot.updatedAt, 0)
}
}
OSGKeyboardTests/FlowSessionBridgeTests.swift
+21
View File
@@ -84,6 +84,27 @@ final class FlowSessionBridgeTests: XCTestCase {
XCTAssertNil(FlowSessionBridge.remainingSessionDuration(defaults: defaults))
}
func testConsumeTranscriptionErrorIncludesKind() {
let defaults = makeDefaults()
FlowSessionBridge.storeTranscriptionError(
"no speech",
kind: .noSpeech,
defaults: defaults
)
let error = FlowSessionBridge.consumeTranscriptionError(defaults: defaults)
XCTAssertEqual(error?.message, "no speech")
XCTAssertEqual(error?.kind, .noSpeech)
XCTAssertNil(FlowSessionBridge.consumeTranscriptionError(defaults: defaults))
}
func testTranscriptionPartialRoundTrip() {
let defaults = makeDefaults()
FlowSessionBridge.storeTranscriptionPartial("你好世界", defaults: defaults)
XCTAssertEqual(FlowSessionBridge.transcriptionPartial(defaults: defaults), "你好世界")
FlowSessionBridge.storeTranscriptionResult("final", defaults: defaults)
XCTAssertNil(FlowSessionBridge.transcriptionPartial(defaults: defaults))
}
func testDarwinNotificationPostsWithoutCrashing() {
FlowSessionDarwin.postSessionChanged()
}
OSGKeyboardTests/IntelligentPolishTests.swift
+5 -5
View File
@@ -122,10 +122,7 @@ final class IntelligentPolishTests: XCTestCase {
func testPolishServiceMissingAPIKeyThrows() async {
store.setEngineMode("cloud")
let service = PolishingService(
store: store,
client: EchoLLMClient()
)
let service = PolishingService(store: store)
do {
_ = try await service.polish("hello world", context: PolishContext(intensity: .medium))
XCTFail("Expected missingAPIKey")
@@ -218,7 +215,10 @@ final class IntelligentPolishTests: XCTestCase {
store.setEngineMode("local")
let captured = CapturingLLMClient()
let service = PolishingService(store: store, client: captured)
_ = try await service.polish("hello", context: PolishContext(intensity: .medium))
_ = try await service.polish(
"今天我们部署 k8s 集群",
context: PolishContext(intensity: .medium)
)
XCTAssertTrue(
captured.lastPrompt.contains("全局输出契约"),
"Local engine should get the Chinese prompt via DeepSeek. Got prefix: \(captured.lastPrompt.prefix(80))"
OSGKeyboardTests/KeyboardOnboardingOverlayTests.swift
+12 -6
View File
@@ -39,30 +39,36 @@ final class KeyboardOnboardingOverlayTests: XCTestCase {
func testOnboardingFlagsRoundTrip() {
store.onboardingPage = 3
store.hasCompletedOnboarding = true
XCTAssertEqual(store.onboardingPage, 3)
store.hasCompletedOnboarding = true
XCTAssertTrue(store.hasCompletedOnboarding)
// Completing onboarding clears the in-progress page index.
XCTAssertEqual(store.onboardingPage, 0)
}
func testOnboardingFlagsSurviveReconstruct() {
store.onboardingPage = 4
store.hasCompletedOnboarding = true
// Simulate the keyboard extension being torn down and rebuilt
// (which is what happens on every `viewDidLoad` cycle).
let store2 = AppGroupStore(defaults: defaults)
var store2 = AppGroupStore(defaults: defaults)
XCTAssertEqual(store2.onboardingPage, 4)
XCTAssertTrue(store2.hasCompletedOnboarding)
store2.hasCompletedOnboarding = true
let store3 = AppGroupStore(defaults: defaults)
XCTAssertTrue(store3.hasCompletedOnboarding)
XCTAssertEqual(store3.onboardingPage, 0)
}
// MARK: - App context detection round-trip
func testDetectedAppContextRoundTrip() {
func testDetectedAppContextRoundTrip() throws {
let now = Date()
store.setDetectedAppContext(.code, at: now)
let result = store.detectedAppContext
XCTAssertEqual(result?.context, .code)
XCTAssertEqual(result?.observedAt.timeIntervalSinceReferenceDate,
let observedAt = try XCTUnwrap(result?.observedAt)
XCTAssertEqual(observedAt.timeIntervalSinceReferenceDate,
now.timeIntervalSinceReferenceDate,
accuracy: 0.001)
}
OSGKeyboardTests/LLMClientTests.swift
+11 -12
View File
@@ -237,13 +237,13 @@ final class LLMClientTests: XCTestCase {
/// Cross-process App Group contract: what `ProviderConfig` writes must
/// be readable through `AppGroupStore` on the same suite.
func testAppGroupCrossProcessAndOffModeShortCircuit() async {
func testAppGroupCrossProcessLegacyOffModeMigratesToPolish() async {
let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suiteName)!
defaults.removePersistentDomain(forName: suiteName)
defer { defaults.removePersistentDomain(forName: suiteName) }
// Writer side: ProviderConfig (main App) writes API key + mode = off.
// Writer side: ProviderConfig (main App) writes API key + legacy off mode.
let config = ProviderConfig(defaults: defaults)
config.apiKey = "sk-test-1234"
config.model = "gpt-4o-mini"
@@ -254,7 +254,7 @@ final class LLMClientTests: XCTestCase {
// same suite.
let store = AppGroupStore(defaults: defaults)
XCTAssertEqual(store.apiKey, "sk-test-1234", "API key did not survive the cross-process boundary")
XCTAssertEqual(store.modeId, "off")
XCTAssertEqual(store.modeId, "polish", "legacy off mode migrates to polish")
XCTAssertEqual(store.model, "gpt-4o-mini")
}
@@ -317,20 +317,19 @@ final class LLMClientTests: XCTestCase {
XCTAssertEqual(calls, 1, "cloud engine must polish even with legacy modeId=off")
}
/// Local engine is ASR-only and never calls the cloud `LLMClient`.
func testPolisherReturnsRawWhenEngineLocal() async throws {
/// Local engine always runs the built-in DeepSeek polish step.
func testPolisherInvokesLLMWhenEngineLocal() async throws {
let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suiteName)!
defaults.removePersistentDomain(forName: suiteName)
defer { defaults.removePersistentDomain(forName: suiteName) }
defaults.set("local", forKey: "config.engineMode")
defaults.set("off", forKey: "config.modeId")
defaults.set("polish", forKey: "config.modeId")
let counter = CallCounter()
let countingClient = CountingLLMClient(counter: counter) { _, _ in
XCTFail("cloud LLMClient must not run under local engine")
return ""
let countingClient = CountingLLMClient(counter: counter) { raw, _ in
"POLISHED: \(raw)"
}
let store = AppGroupStore(defaults: defaults)
@@ -340,10 +339,10 @@ final class LLMClientTests: XCTestCase {
timeout: 1
)
let result = try await polisher.polish(" hello ")
XCTAssertEqual(result, "hello")
let result = try await polisher.polish("hello world")
XCTAssertEqual(result, "POLISHED: hello world")
let calls = await counter.value()
XCTAssertEqual(calls, 0)
XCTAssertEqual(calls, 1, "local engine must always invoke the polish LLM step")
}
/// Local engine pins DeepSeek — cloud-provider URL/model in App Group
OSGKeyboardTests/UtteranceStreamChunkerTests.swift
+24
View File
@@ -24,6 +24,30 @@ final class UtteranceStreamChunkerTests: XCTestCase {
XCTAssertLessThanOrEqual(split, config.maxChunkSamples + config.pauseExtensionSamples)
}
func testFirstChunkUsesShorterWindow() async {
let config = FlowUtteranceChunkConfig(
firstChunkDurationSeconds: 0.5,
subsequentChunkDurationSeconds: 1.0,
overlapDurationSeconds: 0,
pauseExtensionMaxSeconds: 0,
pauseRMSThreshold: 0.02,
sampleRate: 1_000
)
let firstChunkSamples = config.maxChunkSamples(forChunkIndex: 0) + 50
let samples = [Float](repeating: 0.05, count: firstChunkSamples)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: samples, sampleRate: Double(config.sampleRate)))
continuation.finish()
var received: [UtteranceAudioChunk] = []
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
received.append(chunk)
}
XCTAssertGreaterThanOrEqual(received.count, 2)
XCTAssertLessThanOrEqual(received[0].samples.count, config.maxChunkSamples(forChunkIndex: 0) + 50)
}
func testChunksEmitMultipleSegmentsForLongStream() async {
let sampleCount = config.maxChunkSamples * 2 + 100
let samples = [Float](repeating: 0.05, count: sampleCount)