rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(keyboard): add clipboard AI skills, hint keywords, and voice session fixes

Browse Source 
Idle chips show entities with category icons; a fresh copy surfaces Reply/Summarize/Translate; abort/cancel/empty-tap no longer leave the mic stuck.
This commit is contained in:
Rocky
2026-08-13 15:47:32 +08:00
parent 9f308fadd2
commit 631617ee53
48 changed files with 1627 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboard/Services/AIHintRefreshService.swift
+3 -2
View File
@@ -1,8 +1,9 @@
// AIHintRefreshService.swift
// OSGKeyboard · Main App
//
// Silent 12h refresh: fetch remote packs, compress titles with polish LLM,
// merge local evergreen cards, write App Group ready packs for the keyboard.
// Silent 12h refresh: fetch remote packs, extract keywords (LLM only
// for leftovers), merge local evergreen cards, write App Group ready
// packs for the keyboard.
import Foundation
import OSGKeyboardShared
OSGKeyboard/Services/FlowSessionManager.swift
+125 -55
View File
@@ -1069,8 +1069,20 @@ final class FlowSessionManager: ObservableObject {
FlowSessionBridge.clearResult()
// Ack clears the terminal result; republish ready immediately so the
// keyboard does not linger on awaitingDelivery / starting between the
// 500 ms poll and the next heartbeat.
refreshHostReady()
// 500 ms poll and the next heartbeat. If processing is still true for
// this same utterance, the writer forgot to drop the gate — heal here.
if FlowHostAckGatePolicy.shouldDropProcessingGate(
ackUtteranceId: ack.utteranceId,
currentUtteranceId: currentUtteranceId,
isUtteranceProcessing: isUtteranceProcessing
) {
completeFinalizeCleanup(
sessionId: ack.sessionId,
utteranceId: ack.utteranceId
)
} else {
refreshHostReady()
}
}
private func hasUnacknowledgedTerminalResult() -> Bool {
@@ -1280,6 +1292,10 @@ final class FlowSessionManager: ObservableObject {
case .cancelPrimeAudio:
cancelAudioPrime(command)
case .endAIConversation:
if currentUtteranceMode == .aiQuestion,
(isUtteranceProcessing || isUtteranceRecording) {
abortUtterance()
}
if let conversationID = command.aiConversationID {
Task { await aiConversations.removeConversation(conversationID) }
}
@@ -1371,25 +1387,30 @@ final class FlowSessionManager: ObservableObject {
utteranceId: UUID,
commandSeq: Int64
) async {
guard let conversationID else {
guard claimTerminal(utteranceId: utteranceId) else { return }
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
// Hint-card / prefilled questions never go through `finalizeUtterance`,
// so this path must drop the processing gate itself. Leaving it set
// keeps `reason=processing` forever and the keyboard mic unclickable.
defer {
completeFinalizeCleanup(
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq
utteranceId: utteranceId
)
}
guard let conversationID else {
claimAndStoreTerminal(utteranceId: utteranceId) {
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq
)
}
return
}
storeRawCandidate(
question,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq
)
// Do not publish the prompt as a "transcript" — clipboard skills
// send an XML envelope that must never appear above the mic.
let pipelineStore = AppGroupStore()
do {
let service = try AIQuestionService.configured(
@@ -1413,6 +1434,8 @@ final class FlowSessionManager: ObservableObject {
)
}
}
// Abort during the LLM await must not commit or deliver.
guard canStoreTerminal(for: utteranceId) else { return }
publishStreamingAIAnswerIfNeeded(
answer,
sessionId: sessionId,
@@ -1421,30 +1444,32 @@ final class FlowSessionManager: ObservableObject {
aiConversationID: conversationID,
force: true
)
guard claimTerminal(utteranceId: utteranceId) else { return }
await service.commitSuccessfulTurn(
question: question,
answer: answer,
conversationID: conversationID
)
storeFinalizedResult(
answer,
warning: nil,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq,
aiConversationID: conversationID
)
claimAndStoreTerminal(utteranceId: utteranceId) {
storeFinalizedResult(
answer,
warning: nil,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq,
aiConversationID: conversationID
)
}
} catch {
guard claimTerminal(utteranceId: utteranceId) else { return }
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq,
aiConversationID: conversationID
)
claimAndStoreTerminal(utteranceId: utteranceId) {
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
sessionId: sessionId,
utteranceId: utteranceId,
commandSeq: commandSeq,
aiConversationID: conversationID
)
}
}
}
@@ -1993,6 +2018,26 @@ final class FlowSessionManager: ObservableObject {
private func endUtterance() {
guard isUtteranceRecording else { return }
let duration = utteranceRecordingStartedAt.map { Date().timeIntervalSince($0) } ?? 0
if currentUtteranceMode != .editLastInput,
duration < FlowEmptyTapSkipPolicy.maxDurationSeconds {
let samples = capture.utterancePCMSnapshot()
let peak = FlowEmptyTapSkipPolicy.peakAbs(samples)
if FlowEmptyTapSkipPolicy.shouldSkip(
durationSeconds: duration,
sampleCount: samples.count,
peakAmplitude: peak
) {
FlowTrace.pipeline(
"utterance.skipEmptyTap",
"duration=\(String(format: "%.3f", duration))s "
+ "samples=\(samples.count) peak=\(String(format: "%.4f", peak))"
)
abortUtterance(kind: .discardedEmpty, message: "")
return
}
}
// Close the mic gate first, then mark processing before dropping the
// recording flag so the poll loop cannot start a second utterance.
isUtteranceRecording = false
@@ -2040,12 +2085,15 @@ final class FlowSessionManager: ObservableObject {
debug("utterance stopped, draining tail")
}
private func abortUtterance() {
private func abortUtterance(
kind: FlowSessionKeys.TranscriptionErrorKind = .recognitionInterrupted,
message: String? = nil
) {
let utteranceId = currentUtteranceId
if claimTerminal(utteranceId: utteranceId) {
storeCurrentError(
AppL10n.string("flow.error.recognitionInterrupted"),
kind: .recognitionInterrupted,
message ?? AppL10n.string("flow.error.recognitionInterrupted"),
kind: kind,
status: .aborted
)
}
@@ -2125,6 +2173,24 @@ final class FlowSessionManager: ObservableObject {
return true
}
/// After an `await`, refuse to write if this utterance was aborted or replaced.
private func canStoreTerminal(for utteranceId: UUID?) -> Bool {
guard let utteranceId else { return false }
return FlowTerminalStorePolicy.canStore(
currentUtteranceId: currentUtteranceId,
finishedUtteranceId: utteranceId,
alreadyTerminal: terminalUtteranceIds.contains(utteranceId)
)
}
/// Claim + write with no `await` between them, so abort cannot sneak a
/// second payload in after `claimTerminal`.
private func claimAndStoreTerminal(utteranceId: UUID?, store: () -> Void) {
guard canStoreTerminal(for: utteranceId) else { return }
guard claimTerminal(utteranceId: utteranceId) else { return }
store()
}
private func clearPendingInstructionState() {
pendingEditSourceText = nil
pendingSourceHistoryEntryID = nil
@@ -2358,6 +2424,8 @@ final class FlowSessionManager: ObservableObject {
)
}
}
// Abort during the LLM await must not commit or deliver.
guard canStoreTerminal(for: finalizeUtteranceId) else { return }
// Flush the final draft when throttle skipped the last characters.
publishStreamingAIAnswerIfNeeded(
answer,
@@ -2367,30 +2435,32 @@ final class FlowSessionManager: ObservableObject {
aiConversationID: aiConversationID,
force: true
)
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
await service.commitSuccessfulTurn(
question: question,
answer: answer,
conversationID: aiConversationID
)
storeFinalizedResult(
answer,
warning: chunkNote,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq,
aiConversationID: aiConversationID
)
claimAndStoreTerminal(utteranceId: finalizeUtteranceId) {
storeFinalizedResult(
answer,
warning: chunkNote,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq,
aiConversationID: aiConversationID
)
}
} catch {
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq,
aiConversationID: aiConversationID
)
claimAndStoreTerminal(utteranceId: finalizeUtteranceId) {
storeFinalizedError(
AppL10n.string("flow.error.aiQuestionFailed"),
kind: .generic,
sessionId: finalizeSessionId,
utteranceId: finalizeUtteranceId,
commandSeq: finalizeCommandSeq,
aiConversationID: aiConversationID
)
}
}
return
}
OSGKeyboard/Views/ClipboardHistoryDemoView.swift
+1 -1
View File
@@ -13,7 +13,7 @@ import OSGKeyboardShared
struct ClipboardHistoryDemoView: View {
private enum Layout {
static let micSize: CGFloat = 121
static let undoSize: CGFloat = 44
static let undoSize: CGFloat = 52
static let micToButtonGap: CGFloat = 8
static let actionClusterTopGap: CGFloat = Spacing.xl
static let micUpwardAdjustment: CGFloat =
OSGKeyboard/Views/Components/MinimalTabBar.swift
+48 -48
View File
@@ -1,9 +1,9 @@
// MinimalTabBar.swift
// OSGKeyboard · Main App
//
// Bottom tab bar — three icons, no labels.
// Capsule uses iOS 26 Liquid Glass (.regular.interactive) so content
// behind the dock refracts through on scroll.
// Bottom tab bar — icon + label for Home / Styles / Settings.
// The dock capsule is iOS 26 Liquid Glass; the selected tab is a green
// fill inside that capsule (Photos-style), not a second glass layer.
// History + dictionary live as Home cards (not dock tabs).
import SwiftUI
@@ -53,66 +53,66 @@ enum AppTab: Int, CaseIterable {
struct MinimalTabBar: View {
@Environment(\.themePalette) private var palette: ThemePalette
@Environment(\.colorScheme) private var colorScheme
@Namespace private var selectionGlassNamespace
@Namespace private var selectionNamespace
@Binding var selection: AppTab
var body: some View {
GlassEffectContainer(spacing: 0) {
HStack(spacing: 0) {
ForEach(AppTab.allCases, id: \.rawValue) { tab in
Button {
withAnimation(Motion.soft) {
selection = tab
}
} label: {
HStack(spacing: 0) {
ForEach(AppTab.allCases, id: \.rawValue) { tab in
Button {
withAnimation(Motion.soft) {
selection = tab
}
} label: {
VStack(spacing: 2) {
Group {
if let sfSymbol = tab.sfSymbol {
Image(systemName: sfSymbol)
.font(.system(size: 20, weight: .regular))
.font(.system(size: TabBarDockMetrics.iconSize, weight: .regular))
} else {
MaterialIcon(name: tab.icon, size: 24)
MaterialIcon(name: tab.icon, size: TabBarDockMetrics.iconSize)
}
}
.foregroundStyle(tabIconColor(for: tab))
.frame(maxWidth: .infinity)
.frame(height: 48)
.background {
if selection == tab {
// Capsule, not circle: a circle would inscribe to the
// smaller edge and leave the tab slot looking empty.
Color.clear
.frame(width: 52, height: 44)
.glassEffect(
.regular
.tint(palette.accent.opacity(0.18))
.interactive(),
in: .capsule
)
.glassEffectID(
"main-tab-selection",
in: selectionGlassNamespace
)
.glassEffectTransition(.matchedGeometry)
.matchedGeometryEffect(
id: "main-tab-selection",
in: selectionGlassNamespace
)
}
}
.contentShape(Rectangle())
Text(tab.accessibilityKey)
.font(TypeStyle.caption2)
.lineLimit(1)
.minimumScaleFactor(0.8)
}
.buttonStyle(.plain)
.accessibilityLabel(tab.accessibilityKey)
.accessibilityAddTraits(selection == tab ? .isSelected : [])
.foregroundStyle(tabIconColor(for: tab))
.frame(maxWidth: .infinity)
.frame(height: TabBarDockMetrics.itemHeight)
.background {
if selection == tab {
// Stretch into the dock padding so top/bottom
// leftover matches the side leftover (~5 pt).
Capsule()
.fill(palette.accentMuted)
.frame(maxWidth: .infinity, maxHeight: .infinity)
.padding(.horizontal, TabBarDockMetrics.selectionInset)
.padding(
.vertical,
TabBarDockMetrics.selectionInset
- TabBarDockMetrics.dockInsetVertical
)
.matchedGeometryEffect(
id: "main-tab-selection",
in: selectionNamespace
)
}
}
.contentShape(Rectangle())
}
.buttonStyle(.plain)
.accessibilityLabel(tab.accessibilityKey)
.accessibilityAddTraits(selection == tab ? .isSelected : [])
}
.padding(.horizontal, Spacing.md)
.padding(.vertical, Spacing.sm)
.glassEffect(.regular.interactive(), in: .capsule)
}
.padding(.horizontal, TabBarDockMetrics.dockInsetHorizontal)
.padding(.vertical, TabBarDockMetrics.dockInsetVertical)
.glassEffect(.regular.interactive(), in: .capsule)
.frame(maxWidth: 280)
.frame(maxWidth: .infinity, alignment: .center)
.padding(.bottom, Spacing.xs)
.padding(.bottom, TabBarDockMetrics.bottomPadding)
}
private func tabIconColor(for tab: AppTab) -> Color {
OSGKeyboard/Views/Components/TabBarVisibility.swift
+14 -2
View File
@@ -52,8 +52,20 @@ extension View {
}
enum TabBarDockMetrics {
/// Clearance above the floating dock (icon row + vertical padding + home indicator).
static let scrollClearance: CGFloat = 100
static let itemHeight: CGFloat = 52
static let iconSize: CGFloat = 24
/// Horizontal glass pad is 0: the selected capsule's `selectionInset`
/// is the only side gap. `Spacing.md` (16) plus that 5 pt was ~3–4×
/// the 5 pt top/bottom gap.
static let dockInsetHorizontal: CGFloat = 0
static let dockInsetVertical: CGFloat = Spacing.sm
/// Gap between the selected fill and the glass dock / neighbouring tabs.
static let selectionInset: CGFloat = 5
static let bottomPadding: CGFloat = Spacing.xs
/// Clearance above the floating dock (bar + home-indicator slack).
static var scrollClearance: CGFloat {
itemHeight + dockInsetVertical * 2 + bottomPadding + 20
}
}
private struct TabBarScrollBottomPaddingModifier: ViewModifier {
OSGKeyboard/Views/HomeView.swift
+4 -4
View File
@@ -205,8 +205,8 @@ struct HomeView: View {
// MARK: - History / dictionary cards
/// Two independent cards; each header mirrors the stats tiles (accent icon
/// + small uppercase label) and the body grows with its rows.
/// Two independent cards; each header is an accent icon + uppercase label
/// and the body grows with its rows.
private var homeLibrarySection: some View {
VStack(spacing: Spacing.md) {
historyCard
@@ -290,11 +290,11 @@ struct HomeView: View {
VStack(alignment: .leading, spacing: Spacing.xs) {
HStack(spacing: Spacing.xs) {
Image(systemName: systemImage)
.font(.system(size: 13, weight: .semibold))
.font(.system(size: 16, weight: .semibold))
.foregroundStyle(palette.accent)
.symbolRenderingMode(.hierarchical)
Text(titleKey)
.font(TypeStyle.caption2)
.font(.system(size: 13, weight: .semibold))
.tracking(0.6)
.textCase(.uppercase)
.foregroundStyle(palette.textTertiary)
OSGKeyboard/Views/MainTabView.swift
+1 -1
View File
@@ -44,7 +44,7 @@ struct MainTabView: View {
if !isTabBarHidden {
// Match floating dock + home-indicator clearance so
// page footers / scroll ends sit above MinimalTabBar.
Color.clear.frame(height: 100)
Color.clear.frame(height: TabBarDockMetrics.scrollClearance)
}
}
.onPreferenceChange(TabBarHiddenPreferenceKey.self) { hidden in
OSGKeyboard/en.lproj/Localizable.strings
+1 -1
View File
@@ -423,7 +423,7 @@
"home.engine.attentionBanner" = "Local ASR model is not ready yet. Download it and wait for warm-up before dictating.";
/* Tabs */
"tab.keyboard" = "Keyboard";
"tab.keyboard" = "Home";
"tab.history" = "History";
"tab.dictionary" = "Dictionary";
"tab.styles" = "Styles";
OSGKeyboard/zh-Hans.lproj/Localizable.strings
+1 -1
View File
@@ -422,7 +422,7 @@
"home.engine.attentionBanner" = "本地语音识别模型尚未就绪，请先下载并等待加载完成后再使用语音输入。";
/* Tabs */
"tab.keyboard" = "键盘";
"tab.keyboard" = "首页";
"tab.history" = "历史";
"tab.dictionary" = "词库";
"tab.styles" = "风格";