feat(keyboard): improve typing, voice flow, and polish reliability
Reduce extension memory pressure and delivery races while adding richer candidates, tactile feedback, and safer two-level creative polishing.
This commit is contained in:
Rocky
@@ -0,0 +1,81 @@
|
||||
// StubURLProtocol.swift
|
||||
// OSGKeyboardTests · TestSupport
|
||||
//
|
||||
// Shared URLProtocol stub for hermetic HTTP client tests (LLM + Cloud ASR).
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Per-test stub config holder. Tests set these via `StubURLProtocolStorage.config =`
|
||||
/// before invoking the code under test, then reset to nil in cleanup.
|
||||
enum StubURLProtocolStorage {
|
||||
nonisolated(unsafe) static var config: (statusCode: Int, body: Data)?
|
||||
nonisolated(unsafe) static var delaySeconds: Double = 0
|
||||
nonisolated(unsafe) static var lastRequest: URLRequest?
|
||||
}
|
||||
|
||||
final class StubURLProtocol: URLProtocol, @unchecked Sendable {
|
||||
override class func canInit(with request: URLRequest) -> Bool { true }
|
||||
override class func canonicalRequest(for request: URLRequest) -> URLRequest { request }
|
||||
|
||||
override func startLoading() {
|
||||
let cfg = StubURLProtocolStorage.config ?? (statusCode: 200, body: Data())
|
||||
let delay = StubURLProtocolStorage.delaySeconds
|
||||
// Materialize body here (once). Doing it in `canonicalRequest` consumes
|
||||
// the stream before `startLoading` can capture it for assertions.
|
||||
StubURLProtocolStorage.lastRequest = Self.materializingBody(of: request)
|
||||
|
||||
// Simulate a slow transport. We honour URLProtocol.stopLoading() so
|
||||
// cancellation doesn't leave the test hanging, and we yield to the
|
||||
// run loop so `URLSession.data(for:)` actually observes the delay
|
||||
// (a busy-wait would never let the cooperative scheduler time out).
|
||||
DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + delay) { [weak self] in
|
||||
guard let self else { return }
|
||||
guard self.client != nil else { return }
|
||||
let response = HTTPURLResponse(
|
||||
url: self.request.url!,
|
||||
statusCode: cfg.statusCode,
|
||||
httpVersion: "HTTP/1.1",
|
||||
headerFields: ["Content-Type": "application/json"]
|
||||
)!
|
||||
self.client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed)
|
||||
self.client?.urlProtocol(self, didLoad: cfg.body)
|
||||
self.client?.urlProtocolDidFinishLoading(self)
|
||||
}
|
||||
}
|
||||
|
||||
override func stopLoading() {}
|
||||
|
||||
private static func materializingBody(of request: URLRequest) -> URLRequest {
|
||||
var req = request
|
||||
if req.httpBody == nil, let stream = req.httpBodyStream {
|
||||
stream.open()
|
||||
defer { stream.close() }
|
||||
var data = Data()
|
||||
let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: 4_096)
|
||||
defer { buffer.deallocate() }
|
||||
while stream.hasBytesAvailable {
|
||||
let read = stream.read(buffer, maxLength: 4_096)
|
||||
if read <= 0 { break }
|
||||
data.append(buffer, count: read)
|
||||
}
|
||||
if !data.isEmpty {
|
||||
req.httpBody = data
|
||||
}
|
||||
}
|
||||
return req
|
||||
}
|
||||
}
|
||||
|
||||
extension StubURLProtocol {
|
||||
static func makeEphemeralSession() -> URLSession {
|
||||
let cfg = URLSessionConfiguration.ephemeral
|
||||
cfg.protocolClasses = [StubURLProtocol.self]
|
||||
return URLSession(configuration: cfg)
|
||||
}
|
||||
|
||||
static func reset() {
|
||||
StubURLProtocolStorage.config = nil
|
||||
StubURLProtocolStorage.delaySeconds = 0
|
||||
StubURLProtocolStorage.lastRequest = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
// VoicePipelinePerfHarness.swift
|
||||
// OSGKeyboardTests · TestSupport
|
||||
//
|
||||
// Hermetic voice → polish → bridge deliver harness with per-stage timings.
|
||||
// No mic / no live network: synthetic PCM + stub ASR + injected LLM.
|
||||
|
||||
import Foundation
|
||||
import os
|
||||
@testable import OSGKeyboardShared
|
||||
@testable import OSGKeyboardHostSupport
|
||||
|
||||
/// Wall-clock stage breakdown for one utterance finalize path.
|
||||
struct VoicePipelineStageTimings: Sendable, Equatable {
|
||||
/// Build + yield synthetic PCM into an `AsyncStream`.
|
||||
var pcmFeedSeconds: TimeInterval = 0
|
||||
/// `ChunkedUtterancePipeline.transcribe` (chunk → ASR → stitch).
|
||||
var chunkASRSeconds: TimeInterval = 0
|
||||
/// `UtteranceTranscriptGuard.resolve`.
|
||||
var transcriptGuardSeconds: TimeInterval = 0
|
||||
/// Optional full-PCM batch ASR when policy fires.
|
||||
var batchFallbackSeconds: TimeInterval = 0
|
||||
/// `PolishingService.polishWithOutcome`.
|
||||
var polishSeconds: TimeInterval = 0
|
||||
/// App Group bridge write + keyboard-side match read-back.
|
||||
var bridgeDeliverSeconds: TimeInterval = 0
|
||||
/// End-to-end wall clock (pcm feed → bridge deliver).
|
||||
var totalSeconds: TimeInterval = 0
|
||||
|
||||
var didRunBatchFallback: Bool = false
|
||||
var asrText: String = ""
|
||||
var guardedText: String = ""
|
||||
var polishedText: String = ""
|
||||
var deliveredText: String = ""
|
||||
|
||||
/// Ordered rows for XCT attachments / console reports.
|
||||
var rows: [(stage: String, seconds: TimeInterval)] {
|
||||
var list: [(String, TimeInterval)] = [
|
||||
("pcm_feed", pcmFeedSeconds),
|
||||
("chunk_asr", chunkASRSeconds),
|
||||
("transcript_guard", transcriptGuardSeconds),
|
||||
]
|
||||
if didRunBatchFallback {
|
||||
list.append(("batch_fallback", batchFallbackSeconds))
|
||||
}
|
||||
list.append(contentsOf: [
|
||||
("polish", polishSeconds),
|
||||
("bridge_deliver", bridgeDeliverSeconds),
|
||||
("total_e2e", totalSeconds),
|
||||
])
|
||||
return list
|
||||
}
|
||||
|
||||
func reportText() -> String {
|
||||
let body = rows
|
||||
.map { row in
|
||||
let ms = row.seconds * 1_000
|
||||
return "\(row.stage.padding(toLength: 18, withPad: " ", startingAt: 0)) \(String(format: "%8.3f", ms)) ms"
|
||||
}
|
||||
.joined(separator: "\n")
|
||||
return """
|
||||
Voice pipeline stage timings
|
||||
asr=\(asrText)
|
||||
guarded=\(guardedText)
|
||||
polished=\(polishedText)
|
||||
delivered=\(deliveredText)
|
||||
batch_fallback=\(didRunBatchFallback)
|
||||
---
|
||||
\(body)
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic non-silent PCM for chunker / RMS-style paths.
|
||||
enum SyntheticPCM {
|
||||
static func tone(
|
||||
durationSeconds: TimeInterval,
|
||||
sampleRate: Double,
|
||||
amplitude: Float = 0.2,
|
||||
frequencyHz: Double = 440
|
||||
) -> [Float] {
|
||||
let count = max(1, Int((durationSeconds * sampleRate).rounded()))
|
||||
return (0..<count).map { i in
|
||||
let t = Double(i) / sampleRate
|
||||
return amplitude * Float(sin(2 * Double.pi * frequencyHz * t))
|
||||
}
|
||||
}
|
||||
|
||||
static func stream(
|
||||
samples: [Float],
|
||||
sampleRate: Double,
|
||||
frameSize: Int = 80
|
||||
) -> AsyncStream<AudioBufferSnapshot> {
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
var index = 0
|
||||
while index < samples.count {
|
||||
let end = min(index + frameSize, samples.count)
|
||||
continuation.yield(
|
||||
AudioBufferSnapshot(
|
||||
samples: Array(samples[index..<end]),
|
||||
sampleRate: sampleRate
|
||||
)
|
||||
)
|
||||
index = end
|
||||
}
|
||||
continuation.finish()
|
||||
return stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunk ASR stub with optional per-chunk delay (for stage attribution).
|
||||
struct TimedStubChunkASR: ASRChunkTranscribing, ASRService, @unchecked Sendable {
|
||||
let transcript: String
|
||||
let delayNanoseconds: UInt64
|
||||
let batchTranscript: String?
|
||||
/// Only the first non-empty chunk emits `transcript` so stitch stays short.
|
||||
private let emittedLock = OSAllocatedUnfairLock(initialState: false)
|
||||
|
||||
init(
|
||||
transcript: String = "今天部署完成了",
|
||||
delayNanoseconds: UInt64 = 0,
|
||||
batchTranscript: String? = nil
|
||||
) {
|
||||
self.transcript = transcript
|
||||
self.delayNanoseconds = delayNanoseconds
|
||||
self.batchTranscript = batchTranscript
|
||||
}
|
||||
|
||||
func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
) -> AsyncStream<ASREvent> {
|
||||
AsyncStream { $0.finish() }
|
||||
}
|
||||
|
||||
func cancel() {}
|
||||
|
||||
func resetForNewUtterance() {
|
||||
emittedLock.withLock { $0 = false }
|
||||
}
|
||||
|
||||
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
||||
_ = locale
|
||||
if delayNanoseconds > 0 {
|
||||
try? await Task.sleep(nanoseconds: delayNanoseconds)
|
||||
}
|
||||
if samples.isEmpty { return .success("") }
|
||||
// Full-utterance batch calls are much longer than a single chunk.
|
||||
if let batchTranscript, samples.count > 200 {
|
||||
return .success(batchTranscript)
|
||||
}
|
||||
let shouldEmit = emittedLock.withLock { emitted -> Bool in
|
||||
if emitted { return false }
|
||||
emitted = true
|
||||
return true
|
||||
}
|
||||
return .success(shouldEmit ? transcript : "")
|
||||
}
|
||||
}
|
||||
|
||||
/// Injected polish client with optional delay.
|
||||
final class TimedStubLLMClient: LLMClient, @unchecked Sendable {
|
||||
let requestTimeout: TimeInterval = 15
|
||||
let polished: String
|
||||
let delayNanoseconds: UInt64
|
||||
|
||||
init(polished: String, delayNanoseconds: UInt64 = 0) {
|
||||
self.polished = polished
|
||||
self.delayNanoseconds = delayNanoseconds
|
||||
}
|
||||
|
||||
func polish(_ text: String, systemPrompt: String, timeout: TimeInterval?) async throws -> String {
|
||||
_ = text
|
||||
_ = systemPrompt
|
||||
_ = timeout
|
||||
if delayNanoseconds > 0 {
|
||||
try await Task.sleep(nanoseconds: delayNanoseconds)
|
||||
}
|
||||
return polished
|
||||
}
|
||||
}
|
||||
|
||||
enum VoicePipelinePerfHarness {
|
||||
struct Config: Sendable {
|
||||
var sampleRate: Int = 1_000
|
||||
var utteranceDurationSeconds: TimeInterval = 0.25
|
||||
var asrTranscript: String = "今天部署完成了"
|
||||
var polishedTranscript: String = "今天部署已经全部完成。"
|
||||
var asrDelayNanoseconds: UInt64 = 0
|
||||
var polishDelayNanoseconds: UInt64 = 0
|
||||
/// When non-nil and longer than ASR text, forces guard/batch path.
|
||||
var partialSnapshotOverride: String? = nil
|
||||
var batchTranscript: String? = nil
|
||||
var runBatchFallbackIfNeeded: Bool = true
|
||||
}
|
||||
|
||||
@MainActor
|
||||
static func run(config: Config = Config()) async throws -> VoicePipelineStageTimings {
|
||||
var timings = VoicePipelineStageTimings()
|
||||
let totalStart = ContinuousClock.now
|
||||
let rate = Double(config.sampleRate)
|
||||
|
||||
let chunkConfig = FlowUtteranceChunkConfig(
|
||||
maxChunkDurationSeconds: 0.05,
|
||||
overlapDurationSeconds: 0,
|
||||
pauseExtensionMaxSeconds: 0,
|
||||
pauseRMSThreshold: 1.0,
|
||||
minFinalChunkDurationSeconds: 0.05,
|
||||
sampleRate: config.sampleRate
|
||||
)
|
||||
|
||||
let asr = TimedStubChunkASR(
|
||||
transcript: config.asrTranscript,
|
||||
delayNanoseconds: config.asrDelayNanoseconds,
|
||||
batchTranscript: config.batchTranscript
|
||||
)
|
||||
|
||||
// --- pcm_feed ---
|
||||
let pcmStart = ContinuousClock.now
|
||||
let samples = SyntheticPCM.tone(
|
||||
durationSeconds: config.utteranceDurationSeconds,
|
||||
sampleRate: rate
|
||||
)
|
||||
let stream = SyntheticPCM.stream(samples: samples, sampleRate: rate)
|
||||
timings.pcmFeedSeconds = elapsedSeconds(since: pcmStart)
|
||||
|
||||
// --- chunk_asr ---
|
||||
let asrStart = ContinuousClock.now
|
||||
let partialsLock = OSAllocatedUnfairLock(initialState: "")
|
||||
let pipeline = ChunkedUtterancePipeline(
|
||||
asr: asr,
|
||||
locale: Locale(identifier: "zh-Hans"),
|
||||
config: chunkConfig
|
||||
)
|
||||
let outcome = await pipeline.transcribe(stream: stream) { partial in
|
||||
partialsLock.withLock { $0 = partial }
|
||||
}
|
||||
timings.chunkASRSeconds = elapsedSeconds(since: asrStart)
|
||||
|
||||
let stitched: String
|
||||
switch outcome {
|
||||
case .success(let success):
|
||||
stitched = success.text
|
||||
case .cancelled:
|
||||
throw NSError(
|
||||
domain: "VoicePipelinePerfHarness",
|
||||
code: 2,
|
||||
userInfo: [NSLocalizedDescriptionKey: "pipeline cancelled"]
|
||||
)
|
||||
case .failure(let message):
|
||||
throw NSError(
|
||||
domain: "VoicePipelinePerfHarness",
|
||||
code: 1,
|
||||
userInfo: [NSLocalizedDescriptionKey: message]
|
||||
)
|
||||
}
|
||||
timings.asrText = stitched
|
||||
|
||||
let lastPartial = partialsLock.withLock { $0 }
|
||||
let partialSnapshot = config.partialSnapshotOverride ?? lastPartial
|
||||
|
||||
// --- transcript_guard ---
|
||||
let guardStart = ContinuousClock.now
|
||||
var guarded = UtteranceTranscriptGuard.resolve(
|
||||
stitchedFinal: stitched,
|
||||
partialSnapshot: partialSnapshot
|
||||
)
|
||||
timings.transcriptGuardSeconds = elapsedSeconds(since: guardStart)
|
||||
timings.guardedText = guarded
|
||||
|
||||
// --- batch_fallback (optional) ---
|
||||
if config.runBatchFallbackIfNeeded,
|
||||
UtteranceBatchFallbackPolicy.shouldRunBatchFallback(
|
||||
stitchedFinal: stitched,
|
||||
partialSnapshot: partialSnapshot
|
||||
) {
|
||||
timings.didRunBatchFallback = true
|
||||
let batchStart = ContinuousClock.now
|
||||
let batch = await asr.transcribeChunk(samples: samples, locale: Locale(identifier: "zh-Hans"))
|
||||
let batchText: String
|
||||
if case .success(let text) = batch {
|
||||
batchText = text
|
||||
} else {
|
||||
batchText = ""
|
||||
}
|
||||
guarded = UtteranceBatchFallbackPolicy.preferredTranscript(
|
||||
batch: batchText,
|
||||
stitchedFinal: stitched,
|
||||
partialSnapshot: partialSnapshot,
|
||||
current: guarded
|
||||
)
|
||||
timings.batchFallbackSeconds = elapsedSeconds(since: batchStart)
|
||||
timings.guardedText = guarded
|
||||
}
|
||||
|
||||
// --- polish ---
|
||||
let suiteName = "group.com.osgkeyboard.shared.tests.perf.\(UUID().uuidString)"
|
||||
let defaults = UserDefaults(suiteName: suiteName)!
|
||||
defaults.removePersistentDomain(forName: suiteName)
|
||||
defer { defaults.removePersistentDomain(forName: suiteName) }
|
||||
defaults.set("local", forKey: "config.engineMode")
|
||||
let store = AppGroupStore(defaults: defaults)
|
||||
let llm = TimedStubLLMClient(
|
||||
polished: config.polishedTranscript,
|
||||
delayNanoseconds: config.polishDelayNanoseconds
|
||||
)
|
||||
let polishStart = ContinuousClock.now
|
||||
let polishService = PolishingService(store: store, client: llm)
|
||||
let polishOutcome = try await polishService.polishWithOutcome(
|
||||
guarded,
|
||||
context: PolishContext()
|
||||
)
|
||||
timings.polishSeconds = elapsedSeconds(since: polishStart)
|
||||
timings.polishedText = polishOutcome.text
|
||||
|
||||
// --- bridge_deliver ---
|
||||
let bridgeStart = ContinuousClock.now
|
||||
let sessionId = UUID()
|
||||
let utteranceId = UUID()
|
||||
let result = FlowResult(
|
||||
sessionId: sessionId,
|
||||
utteranceId: utteranceId,
|
||||
commandSeq: 1,
|
||||
status: .final,
|
||||
text: polishOutcome.text
|
||||
)
|
||||
FlowSessionBridge.writeResult(result, defaults: defaults)
|
||||
let latest = FlowSessionBridge.latestResult(defaults: defaults)
|
||||
let matched = FlowKeyboardResultMatcher.matchingResult(
|
||||
latest: latest,
|
||||
activeSessionId: sessionId,
|
||||
currentUtteranceId: utteranceId
|
||||
)
|
||||
timings.bridgeDeliverSeconds = elapsedSeconds(since: bridgeStart)
|
||||
timings.deliveredText = matched?.text ?? ""
|
||||
|
||||
timings.totalSeconds = elapsedSeconds(since: totalStart)
|
||||
return timings
|
||||
}
|
||||
|
||||
private static func elapsedSeconds(since start: ContinuousClock.Instant) -> TimeInterval {
|
||||
let duration = start.duration(to: .now)
|
||||
return Double(duration.components.seconds)
|
||||
+ Double(duration.components.attoseconds) / 1e18
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user