feat(keyboard): improve typing, voice flow, and polish reliability

Reduce extension memory pressure and delivery races while adding richer candidates, tactile feedback, and safer two-level creative polishing.
This commit is contained in:
Rocky
2026-08-05 21:39:31 +08:00
parent 38e5ad570d
commit 31f5937a7f
177 changed files with 8343 additions and 3904 deletions
@@ -13,7 +13,7 @@ final class CandidatePanelExpandTests: XCTestCase {
func testChevronRequiresChineseAndAtLeastTwoCandidates() {
let engine = StubRimeEngine()
let typing = TypingSessionController(engine: engine)
let typing = TypingSessionController(engine: { engine })
XCTAssertFalse(typing.canExpandCandidatePanel)
@@ -36,7 +36,7 @@ final class CandidatePanelExpandTests: XCTestCase {
func testToggleExpandAndSelectCollapses() {
let engine = StubRimeEngine()
let typing = TypingSessionController(engine: engine)
let typing = TypingSessionController(engine: { engine })
_ = typing.handleKey("n")
XCTAssertTrue(typing.canExpandCandidatePanel)
@@ -57,7 +57,7 @@ final class CandidatePanelExpandTests: XCTestCase {
func testPanelCollapsesWhenCandidatesDropBelowTwo() {
let engine = StubRimeEngine()
let typing = TypingSessionController(engine: engine)
let typing = TypingSessionController(engine: { engine })
_ = typing.handleKey("n")
typing.toggleCandidatePanelExpanded()
@@ -75,7 +75,7 @@ final class CandidatePanelExpandTests: XCTestCase {
func testLeaveTypingModeCollapsesPanel() {
let engine = StubRimeEngine()
let typing = TypingSessionController(engine: engine)
let typing = TypingSessionController(engine: { engine })
_ = typing.handleKey("n")
typing.toggleCandidatePanelExpanded()
XCTAssertTrue(typing.isCandidatePanelExpanded)
@@ -1,371 +0,0 @@
// ChunkedUtterancePipelineTests.swift
// OSGKeyboardExtTests
//
// Hostless Shared-pipeline tests (no OSGKeyboard.app TEST_HOST).
// Durations are seconds at sampleRate 1000, 0.01s == 10 samples.
import XCTest
import os
@testable import OSGKeyboardShared
private struct StubChunkASR: ASRService, @unchecked Sendable {
let labels: @Sendable ([Float]) -> String
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
return .success(labels(samples))
}
}
final class ChunkedUtterancePipelineTests: XCTestCase {
/// 50-sample chunks @ 1 kHz; overlap / min-final expressed in seconds.
private func config(
maxChunkSeconds: TimeInterval = 0.05,
overlapSeconds: TimeInterval = 0,
minFinalSeconds: TimeInterval = 0.05
) -> FlowUtteranceChunkConfig {
FlowUtteranceChunkConfig(
maxChunkDurationSeconds: maxChunkSeconds,
overlapDurationSeconds: overlapSeconds,
pauseExtensionMaxSeconds: 0,
pauseRMSThreshold: 1.0,
minFinalChunkDurationSeconds: minFinalSeconds,
sampleRate: 1_000
)
}
func testPipelineStitchesQueuedChunks() async {
let asr = StubChunkASR { samples in
samples.isEmpty ? "" : "seg\(samples.count)"
}
let pipeline = ChunkedUtterancePipeline(
asr: asr,
locale: Locale(identifier: "zh-Hans"),
config: config(overlapSeconds: 0)
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.finish()
let partialsLock = OSAllocatedUnfairLock(initialState: [String]())
let outcome = await pipeline.transcribe(stream: stream) { partial in
partialsLock.withLock { $0.append(partial) }
}
let partials = partialsLock.withLock { $0 }
guard case .success(let success) = outcome else {
return XCTFail("expected success, got \(outcome)")
}
XCTAssertTrue(success.text.contains("seg"))
XCTAssertFalse(partials.isEmpty)
}
func testPipelineRetriesTransientMiddleChunkFailure() async {
let pipeline = ChunkedUtterancePipeline(
asr: FailingSecondChunkASR(),
locale: Locale(identifier: "zh-Hans"),
config: config(overlapSeconds: 0)
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
guard case .success(let success) = outcome else {
return XCTFail("expected partial success, got \(outcome)")
}
XCTAssertTrue(success.text.contains("recovered-middle"), "got \(success.text)")
XCTAssertTrue(success.chunkWarnings.isEmpty)
}
func testPipelineWarnsAfterMiddleChunkRetryAlsoFails() async {
let pipeline = ChunkedUtterancePipeline(
asr: PermanentlyFailingMiddleChunkASR(),
locale: Locale(identifier: "zh-Hans"),
config: config(overlapSeconds: 0)
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(
AudioBufferSnapshot(
samples: [Float](repeating: 0.1, count: 160),
sampleRate: 1_000
)
)
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
guard case .success(let success) = outcome else {
return XCTFail("expected partial success, got \(outcome)")
}
XCTAssertFalse(success.text.isEmpty)
XCTAssertEqual(success.chunkWarnings.count, 1)
}
func testPipelineRetranscribesShortFinalChunkWithPriorOverlap() async {
// overlap = 10 samples, minFinal = 50 samples @ 1 kHz
let cfg = config(overlapSeconds: 0.01, minFinalSeconds: 0.05)
XCTAssertEqual(cfg.overlapSamples, 10)
XCTAssertEqual(cfg.minFinalChunkSamples, 50)
let asr = ShortFinalMergeStubASR()
let pipeline = ChunkedUtterancePipeline(
asr: asr,
locale: Locale(identifier: "zh-Hans"),
config: cfg
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
// 80 emit 50 head; leftover 30. +20 50 exactly mid-chunk, then
// empty last marker OR short tail via exact boundary use 80+15 so
// final leftover after mid split stays < minFinal.
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 15), sampleRate: 1_000))
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
guard case .success(let success) = outcome else {
return XCTFail("expected success, got \(outcome)")
}
XCTAssertTrue(success.text.contains("merged"), "got \(success.text)")
}
func testPipelineRetriesEmptyFinalChunkWithOverlap() async {
// Final chunk must be minFinal so emptyRetry runs (not preMerge).
let cfg = config(overlapSeconds: 0.01, minFinalSeconds: 0.01)
XCTAssertEqual(cfg.minFinalChunkSamples, 10)
let asr = EmptyFinalRetryStubASR()
let pipeline = ChunkedUtterancePipeline(
asr: asr,
locale: Locale(identifier: "zh-Hans"),
config: cfg
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
guard case .success(let success) = outcome else {
return XCTFail("expected success, got \(outcome)")
}
XCTAssertTrue(success.text.contains("recovered-tail"), "got \(success.text)")
}
/// Deterministic AC327 regression: short final preMerge empty must keep "head".
///
/// Layout @ 1 kHz:
/// - maxChunk = 100, overlap = 20, minFinal = 80
/// - yield 100 chunk0 ASR "head"
/// - yield 30 final (30 < 80) preMerge samples = 20+30
func testPipelineKeepsPriorTextWhenPreMergeReturnsEmpty() async {
let cfg = config(
maxChunkSeconds: 0.1,
overlapSeconds: 0.02,
minFinalSeconds: 0.08
)
XCTAssertEqual(cfg.maxChunkSamples, 100)
XCTAssertEqual(cfg.overlapSamples, 20)
XCTAssertEqual(cfg.minFinalChunkSamples, 80)
let asr = RecordingEmptyPreMergeASR()
let pipeline = ChunkedUtterancePipeline(
asr: asr,
locale: Locale(identifier: "zh-Hans"),
config: cfg
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(
AudioBufferSnapshot(samples: [Float](repeating: 0.2, count: 100), sampleRate: 1_000)
)
continuation.yield(
AudioBufferSnapshot(samples: [Float](repeating: 0.2, count: 30), sampleRate: 1_000)
)
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
let sampleCounts = asr.sampleCountsSnapshot()
guard case .success(let success) = outcome else {
return XCTFail("expected success keeping prior text, got \(outcome); calls=\(sampleCounts)")
}
XCTAssertEqual(
sampleCounts.count,
2,
"expected head chunk + one preMerge call, got \(sampleCounts)"
)
XCTAssertEqual(sampleCounts[0], 100)
XCTAssertEqual(
sampleCounts[1],
50,
"preMerge should be overlap(20)+tail(30), got \(sampleCounts[1])"
)
XCTAssertTrue(
success.text.contains("head"),
"empty preMerge must not wipe prior segment, got \(success.text)"
)
XCTAssertFalse(success.text.isEmpty)
}
}
private struct FailingSecondChunkASR: ASRService, @unchecked Sendable {
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 1 {
return .failure("simulated chunk error")
}
if current == 2 {
return .success("recovered-middle")
}
return .success("seg\(samples.count)")
}
}
private struct PermanentlyFailingMiddleChunkASR: ASRService, @unchecked Sendable {
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 1 || current == 2 {
return .failure("persistent simulated chunk error")
}
return .success("seg\(samples.count)")
}
}
private struct ShortFinalMergeStubASR: ASRService, @unchecked Sendable {
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 0 {
return .success("head")
}
// preMerge feeds overlap+tail (> first-pass short chunk size)
if samples.count > 15 {
return .success("merged-tail")
}
return .success("short")
}
}
private struct EmptyFinalRetryStubASR: ASRService, @unchecked Sendable {
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 0 {
return .success("head")
}
if current == 1 {
return .success("")
}
return .success("recovered-tail")
}
}
/// Records sample counts; first call "head", later calls empty (preMerge wipe trap).
private final class RecordingEmptyPreMergeASR: ASRService, @unchecked Sendable {
private let lock = OSAllocatedUnfairLock(initialState: [Int]())
func sampleCountsSnapshot() -> [Int] {
lock.withLock { $0 }
}
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let callIndex = lock.withLock { state -> Int in
state.append(samples.count)
return state.count - 1
}
if callIndex == 0 {
return .success("head")
}
return .success("")
}
}
@@ -17,18 +17,75 @@ final class KeyboardSurfaceStateTests: XCTestCase {
func testStandardLayoutHasQwertyTopRow() {
let layout = StandardTypingLayout()
let rows = layout.rows(for: .letters, shiftActive: false)
let rows = layout.rows(for: .letters, language: .english, shiftActive: false)
XCTAssertEqual(rows.first, ["q", "w", "e", "r", "t", "y", "u", "i", "o", "p"])
let shifted = layout.rows(for: .letters, shiftActive: true)
let shifted = layout.rows(for: .letters, language: .english, shiftActive: true)
XCTAssertEqual(shifted.first?.first, "Q")
}
func testEnglishNumberAndSymbolPagesMatchIOSUS() {
let layout = StandardTypingLayout()
XCTAssertEqual(
layout.rows(for: .numbers, language: .english, shiftActive: false),
[
["1", "2", "3", "4", "5", "6", "7", "8", "9", "0"],
["-", "/", ":", ";", "(", ")", "$", "&", "@", "\""],
["#+=", ".", ",", "?", "!", "'", ""]
]
)
XCTAssertEqual(
layout.rows(for: .symbols, language: .english, shiftActive: false),
[
["[", "]", "{", "}", "#", "%", "^", "*", "+", "="],
["_", "\\", "|", "~", "<", ">", "", "£", "¥", "·"],
["123", ".", ",", "?", "!", "'", ""]
]
)
}
func testChineseNumberAndSymbolPagesMatchIOSSimplified() {
let layout = StandardTypingLayout()
XCTAssertEqual(
layout.rows(for: .numbers, language: .chinese, shiftActive: false),
[
["1", "2", "3", "4", "5", "6", "7", "8", "9", "0"],
["-", "/", "", "", "", "", "", "@", "", ""],
["#+=", "", "", "", "", "", ".", ""]
]
)
XCTAssertEqual(
layout.rows(for: .symbols, language: .chinese, shiftActive: false),
[
["", "", "", "", "#", "%", "^", "*", "+", "="],
["_", "\\", "|", "~", "", "", "", "£", "¥", "·"],
["123", "", "", "", "", "", ".", ""]
]
)
}
func testKeyRowsFollowTypingLanguageOnNumberPage() {
let typing = TypingSessionController()
typing.setPage(.numbers)
XCTAssertEqual(typing.keyRows[1], ["-", "/", "", "", "", "", "", "@", "", ""])
_ = typing.setLanguage(.english)
typing.setPage(.numbers)
XCTAssertEqual(typing.keyRows[1], ["-", "/", ":", ";", "(", ")", "$", "&", "@", "\""])
}
func testVoiceAndTypingChromeShareStableDimensions() {
XCTAssertEqual(KeyboardChromeLayout.totalHeight, 281)
XCTAssertEqual(KeyboardChromeLayout.actionKeyHeight, 50)
XCTAssertEqual(KeyboardChromeLayout.actionKeyCornerRadius, 10)
XCTAssertEqual(KeyboardChromeLayout.sideActionKeyWidth, 86)
XCTAssertEqual(KeyboardChromeLayout.actionKeySpacing, 8)
XCTAssertEqual(KeyboardChromeLayout.sideActionKeyFraction, 0.2)
XCTAssertEqual(KeyboardChromeLayout.centerActionKeyFraction, 0.6)
XCTAssertEqual(KeyboardChromeLayout.horizontalInset, 8)
XCTAssertEqual(KeyboardChromeLayout.contentMaxWidth, 700)
let widths = KeyboardChromeLayout.actionKeyWidths(availableWidth: 374)
XCTAssertEqual(widths.side, 71.6, accuracy: 0.001)
XCTAssertEqual(widths.center, 214.8, accuracy: 0.001)
}
func testSharedCapsuleCanSelectSpecificTypingLanguage() {
@@ -49,6 +106,52 @@ final class KeyboardSurfaceStateTests: XCTestCase {
_ = typing.handleKey("")
XCTAssertTrue(typing.shiftActive)
XCTAssertTrue(typing.isShiftEnabled)
XCTAssertEqual(typing.keyRows.first?.first, "Q")
}
func testManualShiftSurvivesAutocapitalizationSync() {
let typing = TypingSessionController()
// Providers must be set before language switch (which syncs autocap).
typing.precedingTextProvider = { "hello " } // mid-sentence: auto stays off
typing.autocapitalizationModeProvider = { .sentences }
_ = typing.setLanguage(.english)
XCTAssertFalse(typing.shiftActive)
_ = typing.handleKey("")
XCTAssertTrue(typing.shiftActive)
typing.syncAutocapitalization()
XCTAssertTrue(typing.shiftActive, "manual one-shot must outrank autocap sync")
XCTAssertEqual(typing.keyRows.first?.first, "Q")
}
func testShiftHoldTypesUppercaseWithoutEnteringCapsLock() {
let typing = TypingSessionController()
_ = typing.setLanguage(.english)
typing.precedingTextProvider = { "hello " }
typing.autocapitalizationModeProvider = { .sentences }
typing.beginShiftHold()
XCTAssertTrue(typing.shiftHeld)
XCTAssertEqual(typing.keyRows.first?.first, "Q")
_ = typing.handleKey("S")
_ = typing.handleKey("m")
XCTAssertFalse(typing.capsLock)
typing.endShiftHold()
XCTAssertFalse(typing.shiftHeld)
XCTAssertFalse(typing.capsLock)
XCTAssertFalse(typing.shiftActive)
}
func testShiftHoldWithoutTypingActsAsTap() {
let typing = TypingSessionController()
typing.beginShiftHold()
typing.endShiftHold()
XCTAssertTrue(typing.shiftActive)
XCTAssertFalse(typing.shiftHeld)
XCTAssertFalse(typing.capsLock)
}
}
@@ -15,7 +15,7 @@ final class LibrimeIntegrationTests: XCTestCase {
try fileManager.createDirectory(at: shared, withIntermediateDirectories: true)
try fileManager.createDirectory(at: user, withIntermediateDirectories: true)
let bundle = try XCTUnwrap(Bundle(identifier: "com.osgkeyboard.ios.shared"))
let bundle = Bundle(for: LibrimeIntegrationTests.self)
let dictionary = try XCTUnwrap(
bundle.url(forResource: "osg_pinyin.dict", withExtension: "yaml")
)
@@ -51,6 +51,7 @@ final class RimeSchemaGeneratorTests: XCTestCase {
XCTAssertEqual(configuration.schema, .fullPinyin)
XCTAssertTrue(configuration.fuzzyPairs.isEmpty)
XCTAssertFalse(configuration.defaultToTyping)
XCTAssertFalse(configuration.rememberLastSurface)
}
@MainActor
@@ -60,11 +61,58 @@ final class RimeSchemaGeneratorTests: XCTestCase {
defer { defaults.removePersistentDomain(forName: suiteName) }
XCTAssertFalse(TypingInputConfiguration.prefersTypingOnOpen(defaults: defaults))
XCTAssertEqual(
TypingInputConfiguration.preferredSurfaceOnOpen(defaults: defaults),
.voice
)
let configuration = TypingInputConfiguration(defaults: defaults)
configuration.defaultToTyping = true
XCTAssertTrue(TypingInputConfiguration.prefersTypingOnOpen(defaults: defaults))
XCTAssertTrue(TypingInputConfiguration(defaults: defaults).defaultToTyping)
XCTAssertEqual(
TypingInputConfiguration.preferredSurfaceOnOpen(defaults: defaults),
.typing
)
}
@MainActor
func testRememberLastSurfaceOverridesDefaultToTyping() {
let suiteName = "TypingInputConfigurationTests.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suiteName)!
defer { defaults.removePersistentDomain(forName: suiteName) }
let configuration = TypingInputConfiguration(defaults: defaults)
configuration.defaultToTyping = true
configuration.rememberLastSurface = true
TypingInputConfiguration.persistLastSurface(.voice, defaults: defaults)
XCTAssertEqual(
TypingInputConfiguration.preferredSurfaceOnOpen(defaults: defaults),
.voice
)
TypingInputConfiguration.persistLastSurface(.typing, defaults: defaults)
XCTAssertEqual(
TypingInputConfiguration.preferredSurfaceOnOpen(defaults: defaults),
.typing
)
}
@MainActor
func testRememberLastSurfaceFallsBackWhenNothingPersisted() {
let suiteName = "TypingInputConfigurationTests.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suiteName)!
defer { defaults.removePersistentDomain(forName: suiteName) }
let configuration = TypingInputConfiguration(defaults: defaults)
configuration.defaultToTyping = true
configuration.rememberLastSurface = true
XCTAssertEqual(
TypingInputConfiguration.preferredSurfaceOnOpen(defaults: defaults),
.typing
)
}
}