feat(keyboard): ship AI hint carousel, home library cards, and clipboard polish
Rotate AI idle suggestions with optional remote packs, move history/dictionary onto self-sizing Home preview cards, harden clipboard capture/prompting, and simplify keyboard chrome by dropping most liquid-glass shadows.
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// SevenDayUsageChart.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// 7-day dictation bar chart. Platform shells wrap this in their own page
|
||||
// layout; the chart itself only needs points + UI language.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// SupportDeveloperSection.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Optional voluntary tip block for Settings. Does not gate features.
|
||||
|
||||
|
||||
@@ -1,29 +1,32 @@
|
||||
// UsageStatsCluster.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Cross-platform home / dashboard stats: 7-day chart + cumulative metrics.
|
||||
// Callers observe their store and pass plain values — Shared stays unbound
|
||||
// from platform singletons.
|
||||
//
|
||||
// Optional `header` sits above the 7-day chart inside the same surface card
|
||||
// (iOS Home glass preview field). Mac / plain call sites keep `EmptyView`.
|
||||
|
||||
import SwiftUI
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public struct UsageStatsCluster: View {
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
public enum Layout: Sendable, Equatable {
|
||||
/// Chart left, 2×2 `UsageStatCard` grid right (Mac / iPad).
|
||||
case split
|
||||
/// Chart above a compact single-card 2×2 grid (iPhone).
|
||||
case stacked
|
||||
}
|
||||
public enum UsageStatsClusterLayout: Sendable, Equatable {
|
||||
/// Chart left, 2×2 `UsageStatCard` grid right (Mac / iPad).
|
||||
case split
|
||||
/// Chart above a compact single-card 2×2 grid (iPhone).
|
||||
case stacked
|
||||
|
||||
/// 手机端 2×2 统计网格的紧凑固定高度(沿用旧版 HomeStatsCard 数值)。
|
||||
static let compactGridHeight: CGFloat = 166
|
||||
public static let compactGridHeight: CGFloat = 166
|
||||
}
|
||||
|
||||
public let layout: Layout
|
||||
public struct UsageStatsCluster<Header: View>: View {
|
||||
@Environment(\.themePalette) private var palette
|
||||
|
||||
public let layout: UsageStatsClusterLayout
|
||||
public let language: AppUILanguage
|
||||
public let points: [UsageStatisticsStore.DailyUsagePoint]
|
||||
public let dictationCharacterCount: Int
|
||||
@@ -32,16 +35,18 @@ public struct UsageStatsCluster: View {
|
||||
public let dictionaryTermCount: Int
|
||||
/// 小屏(如 iPhone SE)收紧 stacked 图表高度,把空间让给下方的输入框。
|
||||
public let compact: Bool
|
||||
private let header: Header
|
||||
|
||||
public init(
|
||||
layout: Layout,
|
||||
layout: UsageStatsClusterLayout,
|
||||
language: AppUILanguage,
|
||||
points: [UsageStatisticsStore.DailyUsagePoint],
|
||||
dictationCharacterCount: Int,
|
||||
dictationDurationSeconds: TimeInterval,
|
||||
translationCharacterCount: Int,
|
||||
dictionaryTermCount: Int,
|
||||
compact: Bool = false
|
||||
compact: Bool = false,
|
||||
@ViewBuilder header: () -> Header
|
||||
) {
|
||||
self.layout = layout
|
||||
self.language = language
|
||||
@@ -51,6 +56,7 @@ public struct UsageStatsCluster: View {
|
||||
self.translationCharacterCount = translationCharacterCount
|
||||
self.dictionaryTermCount = dictionaryTermCount
|
||||
self.compact = compact
|
||||
self.header = header()
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
@@ -66,7 +72,7 @@ public struct UsageStatsCluster: View {
|
||||
|
||||
private var splitBody: some View {
|
||||
HStack(alignment: .top, spacing: Spacing.md) {
|
||||
SevenDayUsageChart(points: points, language: language)
|
||||
chartCard
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
splitStatGrid
|
||||
.frame(maxWidth: .infinity)
|
||||
@@ -112,13 +118,34 @@ public struct UsageStatsCluster: View {
|
||||
|
||||
private var stackedBody: some View {
|
||||
VStack(spacing: compact ? Spacing.sm : Spacing.md) {
|
||||
chartCard
|
||||
compactStatGrid
|
||||
}
|
||||
}
|
||||
|
||||
/// Chart surface; when `header` is present it sits above the bars in the same card.
|
||||
@ViewBuilder
|
||||
private var chartCard: some View {
|
||||
if Header.self == EmptyView.self {
|
||||
SevenDayUsageChart(
|
||||
points: points,
|
||||
language: language,
|
||||
chartMinHeight: compact ? 72 : 96,
|
||||
expands: false
|
||||
expands: layout == .split
|
||||
)
|
||||
compactStatGrid
|
||||
} else {
|
||||
UsageSurfaceCard(padding: Spacing.md) {
|
||||
VStack(alignment: .leading, spacing: Spacing.sm) {
|
||||
header
|
||||
SevenDayUsageChart(
|
||||
points: points,
|
||||
language: language,
|
||||
chartMinHeight: compact ? 72 : 96,
|
||||
embedsInCard: false,
|
||||
expands: layout == .split
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +183,7 @@ public struct UsageStatsCluster: View {
|
||||
}
|
||||
}
|
||||
// 锁定紧凑固定高度(对齐旧版 HomeStatsCard 的 166pt),避免格子按内容撑高。
|
||||
.frame(height: UsageStatsCluster.compactGridHeight)
|
||||
.frame(height: UsageStatsClusterLayout.compactGridHeight)
|
||||
.background(palette.surface)
|
||||
.clipShape(RoundedRectangle(cornerRadius: Radius.xl, style: .continuous))
|
||||
.overlay(
|
||||
@@ -197,3 +224,28 @@ public struct UsageStatsCluster: View {
|
||||
.padding(Spacing.md)
|
||||
}
|
||||
}
|
||||
|
||||
extension UsageStatsCluster where Header == EmptyView {
|
||||
public init(
|
||||
layout: UsageStatsClusterLayout,
|
||||
language: AppUILanguage,
|
||||
points: [UsageStatisticsStore.DailyUsagePoint],
|
||||
dictationCharacterCount: Int,
|
||||
dictationDurationSeconds: TimeInterval,
|
||||
translationCharacterCount: Int,
|
||||
dictionaryTermCount: Int,
|
||||
compact: Bool = false
|
||||
) {
|
||||
self.init(
|
||||
layout: layout,
|
||||
language: language,
|
||||
points: points,
|
||||
dictationCharacterCount: dictationCharacterCount,
|
||||
dictationDurationSeconds: dictationDurationSeconds,
|
||||
translationCharacterCount: translationCharacterCount,
|
||||
dictionaryTermCount: dictionaryTermCount,
|
||||
compact: compact,
|
||||
header: { EmptyView() }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// ASRChunkTranscribing.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Minimal ASR surface for pipelined utterance chunking. Keeps
|
||||
// `ChunkedUtterancePipeline` independent of iOS-only `SpeechAnalyzer`.
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
// ASRService.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Speech-to-text abstraction. As of iOS 26 being the minimum
|
||||
// deployment target, the only ASR backend is `SpeechAnalyzer` +
|
||||
// `DictationTranscriber` — always on-device, no cloud fallback, no
|
||||
// `requiresOnDevice` toggle. The previous legacy recognizer path is
|
||||
// gone; if a future platform ever needs it back,
|
||||
// reintroduce as a sibling class in `ASRServiceFactory.make()`.
|
||||
// Speech-to-text abstraction for foreground host flows. Local mode uses
|
||||
// iOS 26 `SpeechAnalyzer` + `DictationTranscriber`; cloud mode uses the
|
||||
// provider selected in the user's configuration.
|
||||
//
|
||||
// Lives in `OSGKeyboardShared` (not the keyboard extension target) so
|
||||
// that the host app's `KeyboardPreviewSheet` can run the same ASR
|
||||
// pipeline against real iOS audio — without it, the in-app preview
|
||||
// was a static mock that never actually called `SFSpeechRecognizer`,
|
||||
// and "did you actually wire up ASR?" was a fair review note.
|
||||
// Lives in `OSGKeyboardHostSupport` because the foreground host owns
|
||||
// audio capture and recognition. The keyboard extension receives
|
||||
// completed results through the Flow bridge instead of running ASR.
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
@@ -36,9 +31,8 @@ extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {}
|
||||
public protocol ASRService: ASRChunkTranscribing, Sendable {
|
||||
/// Start a transcription session. The returned stream emits `.partial`
|
||||
/// updates and exactly one `.final` (or `.error`) before finishing.
|
||||
/// `SpeechAnalyzer` is always fully on-device, so there is no
|
||||
/// `requiresOnDevice` flag — that legacy cloud-fallback control
|
||||
/// doesn't apply to the iOS 26 `SpeechAnalyzer` path.
|
||||
/// The local `SpeechAnalyzer` path is fully on-device, so this
|
||||
/// abstraction does not expose the legacy `requiresOnDevice` flag.
|
||||
func transcribe(
|
||||
stream: AsyncStream<AudioBufferSnapshot>,
|
||||
locale: Locale
|
||||
@@ -385,8 +379,8 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService`
|
||||
/// / `PreviewASRController` before it reaches SpeechAnalyzer.
|
||||
/// Canonical 16 kHz mono Float32 format produced by the host capture
|
||||
/// pipelines before samples reach SpeechAnalyzer.
|
||||
private static let captureFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: 16_000,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// ChunkedUtterancePipeline.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Pipelined Flow utterance ASR: split PCM while recording, transcribe chunks
|
||||
// serially on a background queue, stitch partials for display and delivery.
|
||||
@@ -336,7 +336,8 @@ public actor ChunkedUtterancePipeline {
|
||||
|
||||
FlowTrace.warn(
|
||||
"pipeline.chunk.retry",
|
||||
"chunk=\(chunkIndex) samples=\(samples.count) error=\(message)"
|
||||
"chunk=\(chunkIndex) samples=\(samples.count) "
|
||||
+ "errorCategory=asrFailure errorBytes=\(message.utf8.count)"
|
||||
)
|
||||
do {
|
||||
try await Task.sleep(nanoseconds: 150_000_000)
|
||||
@@ -362,7 +363,10 @@ public actor ChunkedUtterancePipeline {
|
||||
FlowTrace.transcript("asr.chunk", trimmed, audio)
|
||||
}
|
||||
case .failure(let message):
|
||||
FlowTrace.warn("pipeline.chunk.failed", "\(audio) error=\(message)")
|
||||
FlowTrace.warn(
|
||||
"pipeline.chunk.failed",
|
||||
"\(audio) errorCategory=asrFailure errorBytes=\(message.utf8.count)"
|
||||
)
|
||||
case .cancelled:
|
||||
FlowTrace.pipeline("chunk.cancelled", audio)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// AlibabaVocabularySync.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Syncs PersonalDictionary → DashScope custom vocabulary (Fun-ASR Flash).
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// BailianRealtimeASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Alibaba Cloud Bailian / DashScope realtime ASR over the classic inference
|
||||
// WebSocket (`/api-ws/v1/inference`). Utterance-level duplex session with
|
||||
@@ -145,6 +145,8 @@ struct BailianRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable
|
||||
static func sendText(_ text: String, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.string(text))
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
throw CancellationError()
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
@@ -153,6 +155,8 @@ struct BailianRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable
|
||||
static func sendBinary(_ data: Data, task: URLSessionWebSocketTask) async throws {
|
||||
do {
|
||||
try await task.send(.data(data))
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
throw CancellationError()
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// CloudASRClients.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Provider-specific cloud ASR backends with personal-dictionary bias.
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// CloudASRConnectionCheck.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Settings "validate connection" probe shared by iOS and macOS.
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// CloudASRService.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Cloud-engine ASR: uploads PCM to the user's configured provider with
|
||||
// personal-dictionary bias. Streaming-capable providers use one utterance
|
||||
@@ -11,6 +11,10 @@ import os
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
/// Uploads PCM only on the user-selected cloud engine path; provider clients
|
||||
/// reject missing credentials before network transmission. Personal dictionary
|
||||
/// entries are sent as recognition bias. Mutable client/cancellation state is
|
||||
/// lock-protected, which is the basis for `@unchecked Sendable`.
|
||||
public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
private let store: any ConfigurationStore
|
||||
private let session: URLSession
|
||||
@@ -56,7 +60,9 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
do {
|
||||
try await client.prepare(dictionary: store.personalDictionary)
|
||||
} catch {
|
||||
OSGLog.asr.warning("cloud ASR vocabulary prepare failed: \(error.localizedDescription, privacy: .public)")
|
||||
OSGLog.asr.warning(
|
||||
"cloud ASR vocabulary prepare failed: \(CloudASRLogMetadata.describe(error), privacy: .public)"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +95,7 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s"
|
||||
)
|
||||
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
||||
} catch is CancellationError {
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
FlowTrace.asr("cloud.chunk.cancelled", "samples=\(samples.count)")
|
||||
return .cancelled
|
||||
} catch {
|
||||
@@ -97,7 +103,7 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
"asr.cloud.chunk.failed",
|
||||
"provider=\(store.asrProviderId) samples=\(samples.count) "
|
||||
+ "rms=\(FlowTrace.rms(samples)) elapsed=\(FlowTrace.seconds(since: startedAt))s "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
+ "\(CloudASRLogMetadata.describe(error))"
|
||||
)
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
@@ -129,9 +135,11 @@ public final class CloudASRService: ASRService, @unchecked Sendable {
|
||||
dictionary: store.personalDictionary,
|
||||
onPartial: onPartial
|
||||
)
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
return .cancelled
|
||||
} catch {
|
||||
OSGLog.asr.warning(
|
||||
"streaming ASR session open failed, using chunked batch: \(error.localizedDescription, privacy: .public)"
|
||||
"streaming ASR session open failed, using chunked batch: \(CloudASRLogMetadata.describe(error), privacy: .public)"
|
||||
)
|
||||
let pipeline = ChunkedUtterancePipeline(asr: self, locale: locale)
|
||||
return await pipeline.transcribe(stream: stream, onPartial: onPartial)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// CloudASRStreaming.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Utterance-scoped cloud ASR sessions: one long-lived connection per press,
|
||||
// streaming PCM up and interim text down. Chunked batch ASR remains the
|
||||
@@ -10,6 +10,35 @@ import Foundation
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
enum CloudASRLogMetadata {
|
||||
static func describe(_ error: Error) -> String {
|
||||
if let cloudError = error as? CloudASRError {
|
||||
switch cloudError {
|
||||
case .noAPIKey:
|
||||
return "category=noAPIKey"
|
||||
case .invalidURL:
|
||||
return "category=invalidURL"
|
||||
case .http(let status, let message):
|
||||
return "category=http status=\(status) detailBytes=\(message?.utf8.count ?? 0)"
|
||||
case .decoding(let detail):
|
||||
return "category=decoding detailBytes=\(detail.utf8.count)"
|
||||
case .transport(let detail):
|
||||
return "category=transport detailBytes=\(detail.utf8.count)"
|
||||
case .emptyTranscript:
|
||||
return "category=emptyTranscript"
|
||||
case .audioTooLong:
|
||||
return "category=audioTooLong"
|
||||
case .providerUnsupported:
|
||||
return "category=providerUnsupported"
|
||||
}
|
||||
}
|
||||
if let urlError = error as? URLError {
|
||||
return "category=url code=\(urlError.code.rawValue)"
|
||||
}
|
||||
return "category=\(String(reflecting: type(of: error)))"
|
||||
}
|
||||
}
|
||||
|
||||
/// Long-lived cloud ASR session for one Flow utterance.
|
||||
public protocol CloudASRStreamingSession: Sendable {
|
||||
/// Append 16 kHz mono Float32 PCM captured while the mic is open.
|
||||
@@ -125,7 +154,7 @@ public actor StreamingUtterancePipeline {
|
||||
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s"
|
||||
)
|
||||
return .success(ChunkedUtteranceSuccess(text: finalText))
|
||||
} catch is CancellationError {
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
activeSession?.cancel()
|
||||
activeSession = nil
|
||||
FlowTrace.asr("cloud.stream.cancelled", "uploadedSamples=\(uploadedSamples)")
|
||||
@@ -138,7 +167,7 @@ public actor StreamingUtterancePipeline {
|
||||
"asr.cloud.stream.failed",
|
||||
"uploadedSamples=\(uploadedSamples) "
|
||||
+ "elapsed=\(FlowTrace.seconds(since: startedAt))s "
|
||||
+ "error=\(error.localizedDescription)"
|
||||
+ "\(CloudASRLogMetadata.describe(error))"
|
||||
)
|
||||
return .failure(error.localizedDescription)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// OpenAIRealtimeASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// OpenAI Realtime transcription (WebSocket). Streams PCM and transcript
|
||||
// deltas for utterance-level ASR. Batch `/audio/transcriptions` remains the
|
||||
@@ -94,10 +94,17 @@ struct OpenAIRealtimeASRClient: CloudASRTranscribing, CloudASRStreamingCapable {
|
||||
)
|
||||
session.cancel()
|
||||
} catch {
|
||||
guard Self.shouldFallbackToBatch(afterProbeError: error) else {
|
||||
throw CancellationError()
|
||||
}
|
||||
try await batchClient.probeConnection()
|
||||
}
|
||||
}
|
||||
|
||||
static func shouldFallbackToBatch(afterProbeError error: Error) -> Bool {
|
||||
!ProviderToolCancellation.matches(error)
|
||||
}
|
||||
|
||||
private var resolvedRealtimeModel: String {
|
||||
let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty || trimmed.hasPrefix("gpt-4o") || trimmed == "whisper-1" {
|
||||
@@ -326,6 +333,8 @@ private final class OpenAIRealtimeStreamingSession: CloudASRStreamingSession, @u
|
||||
}
|
||||
do {
|
||||
try await wsTask.send(.string(string))
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
throw CancellationError()
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// VolcengineCloudASRClient.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Volcengine SAUC bigmodel ASR client. Utterance-level WebSocket session with
|
||||
// enable_nonstream (official two-pass): interim text for on-screen partials,
|
||||
@@ -363,9 +363,12 @@ private final class VolcengineStreamingSession: CloudASRStreamingSession, @unche
|
||||
|
||||
guard let frame = VolcengineFrame.parse(data) else { continue }
|
||||
if frame.messageType == .errorMessage {
|
||||
let body = String(data: frame.payload, encoding: .utf8) ?? ""
|
||||
let code = frame.errorCode ?? 0
|
||||
publishFailure(CloudASRError.transport("ASR error \(code): \(body)"))
|
||||
publishFailure(
|
||||
CloudASRError.transport(
|
||||
"ASR error code=\(code) responseBytes=\(frame.payload.count)"
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
guard frame.messageType == .fullServerResponse else { continue }
|
||||
@@ -397,6 +400,8 @@ private final class VolcengineStreamingSession: CloudASRStreamingSession, @unche
|
||||
private func send(_ data: Data) async throws {
|
||||
do {
|
||||
try await wsTask.send(.data(data))
|
||||
} catch where ProviderToolCancellation.matches(error) {
|
||||
throw CancellationError()
|
||||
} catch {
|
||||
throw CloudASRError.transport(error.localizedDescription)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// CustomLanguageModelManager.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Prepares the bundled SFCustomLanguageModelData asset on device and shares
|
||||
// the compiled LM + Vocab through the App Group container. Both the host app
|
||||
// and keyboard extension read the same prepared configuration for
|
||||
// DictationTranscriber content hints.
|
||||
// Prepares the bundled SFCustomLanguageModelData asset for the host app's
|
||||
// iOS SpeechAnalyzer pipeline and caches the compiled LM + Vocab in the
|
||||
// App Group container. The keyboard extension does not run ASR or load
|
||||
// these assets.
|
||||
|
||||
import Foundation
|
||||
import Speech
|
||||
@@ -250,8 +250,8 @@ public final class CustomLanguageModelManager: @unchecked Sendable {
|
||||
// MARK: - Bundle / disk helpers
|
||||
|
||||
private static var resourceBundle: Bundle {
|
||||
// CLM assets ship in the host app bundle (not the extension Shared
|
||||
// framework) so the keyboard process never mmaps the training bin.
|
||||
// CLM assets ship in the host app bundle, so the keyboard process
|
||||
// never mmaps the training bin.
|
||||
Bundle.main
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,9 @@
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum FlowAudioRouteRecoveryPolicy {
|
||||
public static func shouldRebuild(
|
||||
@@ -41,6 +43,9 @@ public final class FlowAudioEngineHandle: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-wide owner of AVAudioSession and Flow's audio engines. Its serial
|
||||
/// queue orders every category, activation, route snapshot, and engine
|
||||
/// start/stop; that queue confinement justifies `@unchecked Sendable`.
|
||||
public final class FlowAudioSessionCoordinator: @unchecked Sendable {
|
||||
private struct CaptureActivation: Sendable {
|
||||
let snapshot: FlowAudioSessionSnapshot
|
||||
|
||||
@@ -9,7 +9,9 @@
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
#if canImport(OSGKeyboardShared)
|
||||
import OSGKeyboardShared
|
||||
#endif
|
||||
|
||||
public enum FlowCaptureVoiceProcessing {
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// FlowContinuousCapture.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// TypeWhisper-style continuous mic capture for Flow sessions: one
|
||||
// AVAudioEngine + input tap for the entire session. Utterances gate
|
||||
@@ -413,6 +413,9 @@ private final class AdaptiveDownsampler: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the session-long capture graph on the main actor. The realtime tap
|
||||
/// must not touch UserDefaults, log, or invoke actor callbacks; it exchanges
|
||||
/// snapshots through lock-protected relays, and callbacks return on MainActor.
|
||||
@MainActor
|
||||
public final class FlowContinuousCapture {
|
||||
|
||||
|
||||
@@ -1,40 +1,27 @@
|
||||
// LiveDictationController.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Unified on-device dictation session: mic capture + iOS 26 SpeechAnalyzer.
|
||||
// Used by the keyboard preview sheet, host-app dictation handoff, and any
|
||||
// other foreground surface that needs live ASR without duplicating pipeline code.
|
||||
// Retained for foreground preview and one-shot handoff surfaces that need
|
||||
// live ASR without duplicating the host-owned audio pipeline.
|
||||
//
|
||||
// STATUS (v0.1.2): Retained as a "preview / one-shot handoff" path.
|
||||
// The *primary* voice-session path is `FlowSessionManager` +
|
||||
// `FlowContinuousCapture` (TypeWhisper-style continuous capture shared
|
||||
// between host app and keyboard extension). The keyboard extension
|
||||
// consumes results through `FlowSessionBridge`.
|
||||
// `FlowContinuousCapture`; the keyboard extension consumes its results
|
||||
// through `FlowSessionBridge`.
|
||||
//
|
||||
// This class is still imported by:
|
||||
// - `OSGKeyboard/Views/PreviewASRController.swift` (typealias)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (in-app preview)
|
||||
// - `OSGKeyboard/Views/KeyboardPreviewSheet.swift` (host-app ASR preview)
|
||||
// - `OSGKeyboardTests/PreviewASRControllerStateTests.swift`
|
||||
//
|
||||
// Do NOT remove without updating those call sites. The earlier
|
||||
// `OSGKeyboardExt/Services/AudioCaptureService.swift` *was* a true
|
||||
// dead duplicate and has been deleted (see AUDIT_APPSTORE.md P0-3).
|
||||
// Do NOT remove without updating those call sites.
|
||||
// Owns its own AVAudioEngine + AVAudioSession, downsamples to 16 kHz
|
||||
// mono Float32 on the audio thread (same as `AudioCaptureService`), and
|
||||
// feeds `AudioBufferSnapshot` to the shared `ASRService` (the same
|
||||
// pipeline the real keyboard extension
|
||||
// uses, so the preview exercises the *real* iOS speech APIs, not a
|
||||
// stub). Without this the in-app preview was a hardcoded transcript
|
||||
// and "did you actually call SFSpeechRecognizer?" was a fair review
|
||||
// note.
|
||||
// mono Float32 on the audio thread, and feeds `AudioBufferSnapshot` to
|
||||
// the HostSupport `ASRService`, so previews exercise the real iOS
|
||||
// speech APIs instead of a stub.
|
||||
//
|
||||
// Why not reuse `AudioCaptureService` from the extension? It lives in
|
||||
// `OSGKeyboardExt`, an `app-extension` target — the main app can't
|
||||
// import its symbols. We could move it to `OSGKeyboardShared`, but
|
||||
// `AVAudioSession` lifecycle differs enough between a keyboard
|
||||
// extension (no background, no recording entitlement surprise) and a
|
||||
// foreground app that a copy here is the lesser evil.
|
||||
// This stays in HostSupport because foreground `AVAudioSession`
|
||||
// lifecycle and recording ownership do not belong in the keyboard
|
||||
// extension or the platform-neutral Shared target.
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
@@ -411,7 +398,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
controller.phase = .idle
|
||||
}
|
||||
case .failure(let message):
|
||||
controller.debug("asr error: \(message)")
|
||||
controller.debug("asr failed messageLen=\(message.count)")
|
||||
controller.teardownCapturePipeline()
|
||||
controller.errorMessage = message
|
||||
controller.phase = .error(message)
|
||||
@@ -442,7 +429,7 @@ public final class LiveDictationController: ObservableObject {
|
||||
// `requestAuthorization` callback were re-typed in the
|
||||
// `@MainActor` context of the caller, and the runtime
|
||||
// assertion came right back — same crash, different symbol:
|
||||
// `closure #1 in closure #2 in PreviewASRController.start(locale:)`.
|
||||
// `closure #1 in closure #2 in LiveDictationController.start(locale:)`.
|
||||
//
|
||||
// The fix that survives inlining is the *function-reference*
|
||||
// pattern, the same one used for `installTap` in
|
||||
@@ -526,8 +513,8 @@ public final class LiveDictationController: ObservableObject {
|
||||
let meter = min(Double(rms) * 4.0, 1.0)
|
||||
onMeter(meter)
|
||||
|
||||
// 2) Downsample to 16 kHz mono Float32 for ASR (matches
|
||||
// `AudioCaptureService` and Apple's `considering:` hint).
|
||||
// 2) Downsample to the 16 kHz mono Float32 format expected by
|
||||
// HostSupport ASR and Apple's `considering:` hint.
|
||||
let outFrames = AVAudioFrameCount(
|
||||
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// TipPurchaseManager.swift
|
||||
// OSGKeyboard · Shared
|
||||
// OSGKeyboard · HostSupport
|
||||
//
|
||||
// Optional ¥30 consumable tip via StoreKit 2. Voluntary support only —
|
||||
// no feature gates, no App Group sync, no restore (Apple consumable rules).
|
||||
|
||||
Reference in New Issue
Block a user