Merge pull request #39 from hkgood/feature/clipboard-voice-command
Feature/clipboard voice command
This commit is contained in:
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- **Clipboard voice command**: long-press the mic after copying eligible text to treat speech as an instruction over the clipboard snapshot and insert the generated result; short press stays dictation. / **剪贴板语音指令**:复制合格文本后长按麦克风,将语音视为对剪贴板材料的指令并插入生成结果;短按仍为听写。
|
||||||
|
- **Clipboard record confirm**: show recording UI only after the host confirms capture; enforce a short minimum record window; light-prewarm ASR when clipboard becomes eligible (no idle mic hold). / **剪贴板开录确认**:宿主确认采音后再显示录音态;确认后再保证最短有效录音;剪贴板刚合格时轻预热 ASR(空闲不占麦)。
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
- **Keep-alive settings & Live Activity**: remove Settings keep-alive mode picker and its note; delete the Dynamic Island Live Activity extension and all ActivityKit session code. Voice sessions stay on silent low-profile PiP only, with user-facing copy that never names Picture in Picture. / **保活设置与灵动岛**:移除设置中的保活方式选项及说明;删除灵动岛 Live Activity 扩展与全部 ActivityKit 会话代码。语音会话仅保留静默低感知 PiP,用户可见文案不再出现「画中画」。
|
- **Keep-alive settings & Live Activity**: remove Settings keep-alive mode picker and its note; delete the Dynamic Island Live Activity extension and all ActivityKit session code. Voice sessions stay on silent low-profile PiP only, with user-facing copy that never names Picture in Picture. / **保活设置与灵动岛**:移除设置中的保活方式选项及说明;删除灵动岛 Live Activity 扩展与全部 ActivityKit 会话代码。语音会话仅保留静默低感知 PiP,用户可见文案不再出现「画中画」。
|
||||||
|
|
||||||
|
|||||||
@@ -75,6 +75,10 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
private var terminalUtteranceIds: Set<UUID> = []
|
private var terminalUtteranceIds: Set<UUID> = []
|
||||||
/// Cursor context captured by the keyboard at the final insertion point.
|
/// Cursor context captured by the keyboard at the final insertion point.
|
||||||
private var pendingFieldContext: FlowFieldContext?
|
private var pendingFieldContext: FlowFieldContext?
|
||||||
|
/// Dictation vs clipboard-command for the live utterance (set on start).
|
||||||
|
private var currentUtteranceMode: FlowUtteranceMode = .dictation
|
||||||
|
private var pendingClipboardSnapshot: String?
|
||||||
|
private var pendingPreviousOutput: String?
|
||||||
private var pendingStopUtteranceId: UUID?
|
private var pendingStopUtteranceId: UUID?
|
||||||
private var currentCommandSeq: Int64 = 0
|
private var currentCommandSeq: Int64 = 0
|
||||||
private var lastHandledCommandSeq: Int64 = 0
|
private var lastHandledCommandSeq: Int64 = 0
|
||||||
@@ -988,6 +992,20 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
utteranceId: command.utteranceId,
|
utteranceId: command.utteranceId,
|
||||||
commandSeq: command.commandSeq
|
commandSeq: command.commandSeq
|
||||||
) else { return }
|
) else { return }
|
||||||
|
currentUtteranceMode = command.resolvedUtteranceMode
|
||||||
|
if currentUtteranceMode == .clipboardCommand {
|
||||||
|
pendingClipboardSnapshot = command.clipboardSnapshot.map {
|
||||||
|
ClipboardMaterialFilter.truncateSnapshot($0)
|
||||||
|
}
|
||||||
|
pendingPreviousOutput = command.previousOutput?
|
||||||
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
if pendingPreviousOutput?.isEmpty == true {
|
||||||
|
pendingPreviousOutput = nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pendingClipboardSnapshot = nil
|
||||||
|
pendingPreviousOutput = nil
|
||||||
|
}
|
||||||
guard let startUtteranceId = currentUtteranceId else { return }
|
guard let startUtteranceId = currentUtteranceId else { return }
|
||||||
let startToken = FlowUtteranceStartToken(
|
let startToken = FlowUtteranceStartToken(
|
||||||
generation: utteranceGeneration,
|
generation: utteranceGeneration,
|
||||||
@@ -1019,6 +1037,10 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
case .abort:
|
case .abort:
|
||||||
guard currentUtteranceId == command.utteranceId else { return }
|
guard currentUtteranceId == command.utteranceId else { return }
|
||||||
abortUtterance()
|
abortUtterance()
|
||||||
|
case .prewarm:
|
||||||
|
// No utterance identity — warm SpeechAnalyzer / cloud prep only.
|
||||||
|
scheduleASRWarmup()
|
||||||
|
FlowDiagnostics.log("prewarm ASR requested seq=\(command.commandSeq)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1043,7 +1065,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
text: trimmed,
|
text: trimmed,
|
||||||
rawText: trimmed,
|
rawText: trimmed,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision()
|
revision: Self.resultRevision(),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1065,7 +1088,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
warning: warning,
|
warning: warning,
|
||||||
rawText: trimmed,
|
rawText: trimmed,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision()
|
revision: Self.resultRevision(),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1085,7 +1109,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
text: message,
|
text: message,
|
||||||
errorKind: kind,
|
errorKind: kind,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision()
|
revision: Self.resultRevision(),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1583,6 +1608,9 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
) async {
|
) async {
|
||||||
let pipelineStarted = Date()
|
let pipelineStarted = Date()
|
||||||
let fieldContext = pendingFieldContext
|
let fieldContext = pendingFieldContext
|
||||||
|
let utteranceMode = currentUtteranceMode
|
||||||
|
let clipboardSnapshot = pendingClipboardSnapshot
|
||||||
|
let previousOutput = pendingPreviousOutput
|
||||||
// ALWAYS clear the processing gate for this utterance. The previous
|
// ALWAYS clear the processing gate for this utterance. The previous
|
||||||
// guard required currentUtteranceId to still match; a racing
|
// guard required currentUtteranceId to still match; a racing
|
||||||
// fail/abort/cancel path could nil the id (or leave processing stuck)
|
// fail/abort/cancel path could nil the id (or leave processing stuck)
|
||||||
@@ -1590,6 +1618,11 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
// while host logs still said "utterance finalized".
|
// while host logs still said "utterance finalized".
|
||||||
defer {
|
defer {
|
||||||
pendingFieldContext = nil
|
pendingFieldContext = nil
|
||||||
|
pendingClipboardSnapshot = nil
|
||||||
|
pendingPreviousOutput = nil
|
||||||
|
if currentUtteranceMode == utteranceMode {
|
||||||
|
currentUtteranceMode = .dictation
|
||||||
|
}
|
||||||
completeFinalizeCleanup(
|
completeFinalizeCleanup(
|
||||||
sessionId: finalizeSessionId,
|
sessionId: finalizeSessionId,
|
||||||
utteranceId: finalizeUtteranceId
|
utteranceId: finalizeUtteranceId
|
||||||
@@ -1707,15 +1740,51 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
|
|
||||||
var delivered = text
|
var delivered = text
|
||||||
let polishStarted = Date()
|
let polishStarted = Date()
|
||||||
let polishMode = pipelineStore.polishModeForPipeline
|
let isClipboardCommand = utteranceMode == .clipboardCommand
|
||||||
|
let polishMode: PolishingService.PolishMode = isClipboardCommand
|
||||||
|
? .polish
|
||||||
|
: pipelineStore.polishModeForPipeline
|
||||||
|
let clipboardPrompt: (system: String, user: String)? = {
|
||||||
|
guard isClipboardCommand,
|
||||||
|
let snapshot = clipboardSnapshot,
|
||||||
|
!snapshot.isEmpty else { return nil }
|
||||||
|
let bias = ClipboardCommandPromptComposer.styleBias(
|
||||||
|
styleID: pipelineStore.activePolishStyleId,
|
||||||
|
catalog: pipelineStore.polishStyleCatalog
|
||||||
|
)
|
||||||
|
let input = ClipboardCommandPromptComposer.Input(
|
||||||
|
snapshot: snapshot,
|
||||||
|
instruction: textForPolish,
|
||||||
|
previousOutput: previousOutput,
|
||||||
|
styleBias: bias
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
ClipboardCommandPromptComposer.compose(input),
|
||||||
|
ClipboardCommandPromptComposer.userMessage(input)
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
|
||||||
|
if isClipboardCommand, clipboardPrompt == nil {
|
||||||
|
FlowDiagnostics.log("clipboard command missing snapshot — failing closed")
|
||||||
|
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
|
||||||
|
storeFinalizedError(
|
||||||
|
AppL10n.string("flow.error.clipboardCommandFailed"),
|
||||||
|
kind: .generic,
|
||||||
|
sessionId: finalizeSessionId,
|
||||||
|
utteranceId: finalizeUtteranceId,
|
||||||
|
commandSeq: finalizeCommandSeq
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
FlowDiagnostics.log(
|
FlowDiagnostics.log(
|
||||||
"finalize LLM mode=\(Self.polishModeLogLabel(polishMode)) " +
|
"finalize LLM mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) " +
|
||||||
"translationTarget=\(pipelineStore.translationTargetLocaleId)"
|
"translationTarget=\(pipelineStore.translationTargetLocaleId)"
|
||||||
)
|
)
|
||||||
FlowTrace.transcript(
|
FlowTrace.transcript(
|
||||||
"polish.input",
|
"polish.input",
|
||||||
textForPolish,
|
textForPolish,
|
||||||
"mode=\(Self.polishModeLogLabel(polishMode)) engine=\(engineMode) "
|
"mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) engine=\(engineMode) "
|
||||||
+ "provider=\(pipelineStore.polishProviderIdOverride ?? "default") "
|
+ "provider=\(pipelineStore.polishProviderIdOverride ?? "default") "
|
||||||
+ "recordedSeconds=\(String(format: "%.2f", recordingDuration))"
|
+ "recordedSeconds=\(String(format: "%.2f", recordingDuration))"
|
||||||
)
|
)
|
||||||
@@ -1723,29 +1792,31 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
// If the finalize task was cancelled (cold-start churn / abort),
|
// If the finalize task was cancelled (cold-start churn / abort),
|
||||||
// skip the LLM round-trip and deliver the raw transcript so the
|
// skip the LLM round-trip and deliver the raw transcript so the
|
||||||
// keyboard is not left waiting on a result that never arrives.
|
// keyboard is not left waiting on a result that never arrives.
|
||||||
|
// Clipboard-command mode must never insert the instruction ASR.
|
||||||
if Task.isCancelled {
|
if Task.isCancelled {
|
||||||
throw CancellationError()
|
throw CancellationError()
|
||||||
}
|
}
|
||||||
let outcome = try await Self.polishWithHostTimeout(
|
let outcome = try await Self.polishWithHostTimeout(
|
||||||
polisher: polisher,
|
polisher: polisher,
|
||||||
text: textForPolish,
|
text: clipboardPrompt?.user ?? textForPolish,
|
||||||
mode: polishMode,
|
mode: polishMode,
|
||||||
|
systemPrompt: clipboardPrompt?.system,
|
||||||
providerIdOverride: pipelineStore.polishProviderIdOverride,
|
providerIdOverride: pipelineStore.polishProviderIdOverride,
|
||||||
context: polishContext
|
context: isClipboardCommand ? nil : polishContext
|
||||||
)
|
)
|
||||||
let polished = outcome.text
|
let polished = outcome.text
|
||||||
delivered = polished
|
delivered = polished
|
||||||
FlowTrace.transcript(
|
FlowTrace.transcript(
|
||||||
"polish.output",
|
"polish.output",
|
||||||
polished,
|
polished,
|
||||||
"mode=\(Self.polishModeLogLabel(polishMode)) inputLen=\(text.count) "
|
"mode=\(isClipboardCommand ? "clipboardCommand" : Self.polishModeLogLabel(polishMode)) inputLen=\(text.count) "
|
||||||
+ "changed=\(polished == text ? 0 : 1) "
|
+ "changed=\(polished == text ? 0 : 1) "
|
||||||
+ "elapsed=\(FlowTrace.seconds(since: polishStarted))s"
|
+ "elapsed=\(FlowTrace.seconds(since: polishStarted))s"
|
||||||
)
|
)
|
||||||
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
|
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
|
||||||
storeFinalizedResult(
|
storeFinalizedResult(
|
||||||
polished,
|
polished,
|
||||||
rawText: text,
|
rawText: isClipboardCommand ? nil : text,
|
||||||
warning: Self.combinedWarning(
|
warning: Self.combinedWarning(
|
||||||
chunkNote,
|
chunkNote,
|
||||||
outcome.qualityDegraded
|
outcome.qualityDegraded
|
||||||
@@ -1760,7 +1831,33 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
"polish done in \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s " +
|
"polish done in \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s " +
|
||||||
"total=\(String(format: "%.1f", Date().timeIntervalSince(pipelineStarted)))s"
|
"total=\(String(format: "%.1f", Date().timeIntervalSince(pipelineStarted)))s"
|
||||||
)
|
)
|
||||||
|
SpeechHistoryStore.shared.recordUtterance(
|
||||||
|
text: delivered,
|
||||||
|
engineMode: engineMode,
|
||||||
|
duration: recordingDuration,
|
||||||
|
wasTranslation: isClipboardCommand ? false : pipelineStore.isTranslationEffective
|
||||||
|
)
|
||||||
} catch {
|
} catch {
|
||||||
|
if isClipboardCommand {
|
||||||
|
FlowDiagnostics.log(
|
||||||
|
"clipboard command failed after \(String(format: "%.1f", Date().timeIntervalSince(polishStarted)))s: " +
|
||||||
|
"\(error.localizedDescription)"
|
||||||
|
)
|
||||||
|
FlowTrace.warn(
|
||||||
|
"clipboardCommand.failed",
|
||||||
|
"elapsed=\(FlowTrace.seconds(since: polishStarted))s "
|
||||||
|
+ "cancelled=\(error is CancellationError ? 1 : 0) "
|
||||||
|
+ "error=\(error.localizedDescription)"
|
||||||
|
)
|
||||||
|
guard claimTerminal(utteranceId: finalizeUtteranceId) else { return }
|
||||||
|
storeFinalizedError(
|
||||||
|
AppL10n.string("flow.error.clipboardCommandFailed"),
|
||||||
|
kind: .generic,
|
||||||
|
sessionId: finalizeSessionId,
|
||||||
|
utteranceId: finalizeUtteranceId,
|
||||||
|
commandSeq: finalizeCommandSeq
|
||||||
|
)
|
||||||
|
} else {
|
||||||
// CancellationError is common when the user jumps back via
|
// CancellationError is common when the user jumps back via
|
||||||
// startflow mid-polish; still deliver raw text. Other errors
|
// startflow mid-polish; still deliver raw text. Other errors
|
||||||
// keep the existing polish-warning fallback.
|
// keep the existing polish-warning fallback.
|
||||||
@@ -1796,14 +1893,14 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
utteranceId: finalizeUtteranceId,
|
utteranceId: finalizeUtteranceId,
|
||||||
commandSeq: finalizeCommandSeq
|
commandSeq: finalizeCommandSeq
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
SpeechHistoryStore.shared.recordUtterance(
|
SpeechHistoryStore.shared.recordUtterance(
|
||||||
text: delivered,
|
text: delivered,
|
||||||
engineMode: engineMode,
|
engineMode: engineMode,
|
||||||
duration: recordingDuration,
|
duration: recordingDuration,
|
||||||
wasTranslation: pipelineStore.isTranslationEffective
|
wasTranslation: pipelineStore.isTranslationEffective
|
||||||
)
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
currentPartial = ""
|
currentPartial = ""
|
||||||
lastFinal = ""
|
lastFinal = ""
|
||||||
@@ -1882,7 +1979,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
rawText: rawText,
|
rawText: rawText,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision(),
|
revision: Self.resultRevision(),
|
||||||
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext)
|
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1905,7 +2003,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
rawText: trimmed,
|
rawText: trimmed,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision(),
|
revision: Self.resultRevision(),
|
||||||
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext)
|
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -1935,7 +2034,8 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
errorKind: kind,
|
errorKind: kind,
|
||||||
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
hostGeneration: FlowSessionBridge.currentHostGeneration(),
|
||||||
revision: Self.resultRevision(),
|
revision: Self.resultRevision(),
|
||||||
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext)
|
fieldFingerprint: Self.fieldFingerprint(pendingFieldContext),
|
||||||
|
utteranceMode: currentUtteranceMode
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2069,6 +2169,7 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
polisher: PolishingService,
|
polisher: PolishingService,
|
||||||
text: String,
|
text: String,
|
||||||
mode: PolishingService.PolishMode,
|
mode: PolishingService.PolishMode,
|
||||||
|
systemPrompt: String? = nil,
|
||||||
providerIdOverride: String?,
|
providerIdOverride: String?,
|
||||||
context: PolishContext?
|
context: PolishContext?
|
||||||
) async throws -> PolishingService.PolishOutcome {
|
) async throws -> PolishingService.PolishOutcome {
|
||||||
@@ -2077,6 +2178,7 @@ final class FlowSessionManager: ObservableObject {
|
|||||||
try await polisher.polishWithOutcome(
|
try await polisher.polishWithOutcome(
|
||||||
text,
|
text,
|
||||||
mode: mode,
|
mode: mode,
|
||||||
|
systemPrompt: systemPrompt,
|
||||||
providerIdOverride: providerIdOverride,
|
providerIdOverride: providerIdOverride,
|
||||||
context: context
|
context: context
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -336,6 +336,7 @@
|
|||||||
|
|
||||||
/* Flow session */
|
/* Flow session */
|
||||||
"flow.error.noSpeech" = "No speech detected. Please try again.";
|
"flow.error.noSpeech" = "No speech detected. Please try again.";
|
||||||
|
"flow.error.clipboardCommandFailed" = "Couldn't process the clipboard. Please try again.";
|
||||||
"flow.error.recognitionInterrupted" = "Recognition did not finish. Please try again.";
|
"flow.error.recognitionInterrupted" = "Recognition did not finish. Please try again.";
|
||||||
"keyboard.denied.mic" = "Microphone access denied";
|
"keyboard.denied.mic" = "Microphone access denied";
|
||||||
"keyboard.denied.speech" = "Speech recognition denied";
|
"keyboard.denied.speech" = "Speech recognition denied";
|
||||||
|
|||||||
@@ -335,6 +335,7 @@
|
|||||||
|
|
||||||
/* Flow session */
|
/* Flow session */
|
||||||
"flow.error.noSpeech" = "未检测到语音,请重试。";
|
"flow.error.noSpeech" = "未检测到语音,请重试。";
|
||||||
|
"flow.error.clipboardCommandFailed" = "剪贴板处理失败,请重试。";
|
||||||
"flow.error.recognitionInterrupted" = "识别未完成,请再试一次。";
|
"flow.error.recognitionInterrupted" = "识别未完成,请再试一次。";
|
||||||
"keyboard.denied.mic" = "麦克风权限被拒绝";
|
"keyboard.denied.mic" = "麦克风权限被拒绝";
|
||||||
"keyboard.denied.speech" = "语音识别权限被拒绝";
|
"keyboard.denied.speech" = "语音识别权限被拒绝";
|
||||||
|
|||||||
@@ -160,6 +160,7 @@ public final class KeyboardViewController: UIInputViewController {
|
|||||||
KeyboardSetupBridge.markExtensionAppearance(hasFullAccess: hasFullAccess)
|
KeyboardSetupBridge.markExtensionAppearance(hasFullAccess: hasFullAccess)
|
||||||
state.debugHasFullAccess = hasFullAccess
|
state.debugHasFullAccess = hasFullAccess
|
||||||
flowCoordinator.refreshSessionState()
|
flowCoordinator.refreshSessionState()
|
||||||
|
flowCoordinator.refreshClipboardEligibility()
|
||||||
flowCoordinator.startSessionMonitor()
|
flowCoordinator.startSessionMonitor()
|
||||||
configSync.syncOnboardingStateFromAppGroup()
|
configSync.syncOnboardingStateFromAppGroup()
|
||||||
configSync.refreshConfigFromAppGroup()
|
configSync.refreshConfigFromAppGroup()
|
||||||
@@ -247,6 +248,7 @@ public final class KeyboardViewController: UIInputViewController {
|
|||||||
textInserter = KeyboardTextInserter(
|
textInserter = KeyboardTextInserter(
|
||||||
state: state,
|
state: state,
|
||||||
insertText: { [weak self] text in self?.textDocumentProxy.insertText(text) },
|
insertText: { [weak self] text in self?.textDocumentProxy.insertText(text) },
|
||||||
|
deleteBackward: { [weak self] in self?.textDocumentProxy.deleteBackward() },
|
||||||
contextBeforeInput: { [weak self] in self?.textDocumentProxy.documentContextBeforeInput },
|
contextBeforeInput: { [weak self] in self?.textDocumentProxy.documentContextBeforeInput },
|
||||||
scheduleAutoClearError: { [weak self] in self?.scheduleAutoClearError() }
|
scheduleAutoClearError: { [weak self] in self?.scheduleAutoClearError() }
|
||||||
)
|
)
|
||||||
@@ -285,6 +287,15 @@ public final class KeyboardViewController: UIInputViewController {
|
|||||||
state.beginRecording = { [weak self] in self?.flowCoordinator.pressBegan() }
|
state.beginRecording = { [weak self] in self?.flowCoordinator.pressBegan() }
|
||||||
state.endRecording = { [weak self] in self?.flowCoordinator.pressEnded() }
|
state.endRecording = { [weak self] in self?.flowCoordinator.pressEnded() }
|
||||||
state.tapMic = { [weak self] in self?.flowCoordinator.toggleRecording() }
|
state.tapMic = { [weak self] in self?.flowCoordinator.toggleRecording() }
|
||||||
|
state.beginClipboardCommand = { [weak self] in
|
||||||
|
self?.flowCoordinator.clipboardCommandPressBegan()
|
||||||
|
}
|
||||||
|
state.endClipboardCommand = { [weak self] in
|
||||||
|
self?.flowCoordinator.clipboardCommandPressEnded()
|
||||||
|
}
|
||||||
|
state.refreshClipboardEligibility = { [weak self] in
|
||||||
|
self?.flowCoordinator.refreshClipboardEligibility()
|
||||||
|
}
|
||||||
state.openSettings = { [weak self] in self?.openHostApp() }
|
state.openSettings = { [weak self] in self?.openHostApp() }
|
||||||
state.startFlowSession = { [weak self] in self?.flowCoordinator.beginFlowStart() }
|
state.startFlowSession = { [weak self] in self?.flowCoordinator.beginFlowStart() }
|
||||||
state.setMode = { [weak self] m in self?.configSync.persistMode(m) }
|
state.setMode = { [weak self] m in self?.configSync.persistMode(m) }
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
// ClipboardPasteboardReader.swift
|
||||||
|
// OSGKeyboard · Keyboard Extension
|
||||||
|
//
|
||||||
|
// Opportunity-read of UIPasteboard for clipboard-command eligibility.
|
||||||
|
|
||||||
|
import UIKit
|
||||||
|
import OSGKeyboardShared
|
||||||
|
|
||||||
|
enum ClipboardPasteboardReader {
|
||||||
|
static func sample() -> (changeCount: Int, text: String?) {
|
||||||
|
let board = UIPasteboard.general
|
||||||
|
let changeCount = board.changeCount
|
||||||
|
// Prefer plain strings; avoid forcing non-text pasteboard items.
|
||||||
|
let text = board.hasStrings ? board.string : nil
|
||||||
|
return (changeCount, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -67,6 +67,21 @@ final class KeyboardFlowCoordinator {
|
|||||||
/// Ignores single-frame "host dead" samples before allowing a cold-start jump
|
/// Ignores single-frame "host dead" samples before allowing a cold-start jump
|
||||||
/// from non-press recovery paths.
|
/// from non-press recovery paths.
|
||||||
private var coldStartDebouncer = FlowColdStartDebouncer()
|
private var coldStartDebouncer = FlowColdStartDebouncer()
|
||||||
|
/// Opportunity-read eligibility window (copy → 30s).
|
||||||
|
private var clipboardEligibility: ClipboardCommandEligibility?
|
||||||
|
/// Active clipboard-command task session (continuous rewrite).
|
||||||
|
private var clipboardTaskSession: ClipboardCommandTaskSession?
|
||||||
|
/// True while the live utterance is a clipboard-command hold-to-talk.
|
||||||
|
private var isClipboardCommandUtterance = false
|
||||||
|
/// Clipboard start sent; waiting for host `reason=recording` before red UI.
|
||||||
|
private var clipboardAwaitingHostRecordConfirm = false
|
||||||
|
/// Wall time when host confirmed real capture for this clipboard utterance.
|
||||||
|
private var clipboardHostRecordConfirmedAt: TimeInterval?
|
||||||
|
/// Finger already up — stop after confirm + minimum recording window.
|
||||||
|
private var clipboardStopRequested = false
|
||||||
|
private var clipboardDeferredStopTask: Task<Void, Never>?
|
||||||
|
/// Last pasteboard changeCount we already asked the host to light-prewarm.
|
||||||
|
private var clipboardPrewarmedChangeCount: Int?
|
||||||
|
|
||||||
init(
|
init(
|
||||||
state: KeyboardState,
|
state: KeyboardState,
|
||||||
@@ -165,6 +180,8 @@ final class KeyboardFlowCoordinator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
recomputeMicVoiceAvailability()
|
recomputeMicVoiceAvailability()
|
||||||
|
refreshClipboardEligibility()
|
||||||
|
promoteClipboardRecordingFromSnapshotIfNeeded()
|
||||||
startHostReadyWaitIfNeeded()
|
startHostReadyWaitIfNeeded()
|
||||||
// Proactive host auto-launch is disabled (FlowHandoffPolicy): a single
|
// Proactive host auto-launch is disabled (FlowHandoffPolicy): a single
|
||||||
// stale ready snapshot after finalize must never open startflow.
|
// stale ready snapshot after finalize must never open startflow.
|
||||||
@@ -295,10 +312,14 @@ final class KeyboardFlowCoordinator {
|
|||||||
flowStartDeadline = 0
|
flowStartDeadline = 0
|
||||||
stopHostReadyWait()
|
stopHostReadyWait()
|
||||||
isFlowRecording = true
|
isFlowRecording = true
|
||||||
|
if isClipboardCommandUtterance {
|
||||||
|
noteClipboardHostRecordingConfirmed()
|
||||||
|
} else {
|
||||||
state.phase = .recording
|
state.phase = .recording
|
||||||
if state.lastTranscript.isEmpty {
|
if state.lastTranscript.isEmpty {
|
||||||
state.lastTranscript = ""
|
state.lastTranscript = ""
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if let view = wakeLockView() {
|
if let view = wakeLockView() {
|
||||||
ExtensionScreenWakeLock.acquire(from: view)
|
ExtensionScreenWakeLock.acquire(from: view)
|
||||||
}
|
}
|
||||||
@@ -450,12 +471,200 @@ final class KeyboardFlowCoordinator {
|
|||||||
case .recording:
|
case .recording:
|
||||||
pressEnded()
|
pressEnded()
|
||||||
case .idle, .denied, .error:
|
case .idle, .denied, .error:
|
||||||
|
// Short press while a clipboard task is open → exit command mode, then dictation.
|
||||||
|
if clipboardTaskSession != nil {
|
||||||
|
endClipboardTaskSession()
|
||||||
|
}
|
||||||
|
isClipboardCommandUtterance = false
|
||||||
pressBegan()
|
pressBegan()
|
||||||
case .requestingPermissions, .processing:
|
case .requestingPermissions, .processing:
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Long-press reached 0.45s — start clipboard-command hold-to-talk immediately.
|
||||||
|
func clipboardCommandPressBegan() {
|
||||||
|
switch state.phase {
|
||||||
|
case .idle, .denied, .error:
|
||||||
|
break
|
||||||
|
case .processing:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
refreshClipboardEligibility()
|
||||||
|
let secure = fieldContextProvider()?.isSecureEntry == true
|
||||||
|
if secure {
|
||||||
|
endClipboardTaskSession()
|
||||||
|
publishClipboardUIState()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = Date().timeIntervalSince1970
|
||||||
|
if let session = clipboardTaskSession, session.isActive(at: now) {
|
||||||
|
isClipboardCommandUtterance = true
|
||||||
|
pressBegan()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let eligibility = clipboardEligibility, eligibility.isOpen(at: now) else {
|
||||||
|
// No open window — do not enter command mode.
|
||||||
|
isClipboardCommandUtterance = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let fingerprint = fieldContextProvider()?.deliveryFingerprint
|
||||||
|
clipboardTaskSession = ClipboardCommandTaskSession(
|
||||||
|
snapshot: eligibility.snapshot,
|
||||||
|
expiresAt: now + ClipboardMaterialFilter.sessionDuration,
|
||||||
|
fieldFingerprint: fingerprint
|
||||||
|
)
|
||||||
|
isClipboardCommandUtterance = true
|
||||||
|
publishClipboardUIState()
|
||||||
|
pressBegan()
|
||||||
|
}
|
||||||
|
|
||||||
|
func clipboardCommandPressEnded() {
|
||||||
|
guard isClipboardCommandUtterance else { return }
|
||||||
|
clipboardStopRequested = true
|
||||||
|
if clipboardAwaitingHostRecordConfirm {
|
||||||
|
// Finger up before host capture — keep preparing; stop after confirm + min window.
|
||||||
|
traceState("clipboard.stop.deferred", extra: "reason=awaitingHostConfirm")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if let confirmedAt = clipboardHostRecordConfirmedAt {
|
||||||
|
let elapsed = Date().timeIntervalSince1970 - confirmedAt
|
||||||
|
let minimum = ClipboardMaterialFilter.minimumRecordingAfterHostConfirm
|
||||||
|
if elapsed < minimum {
|
||||||
|
scheduleClipboardDeferredStop(after: minimum - elapsed)
|
||||||
|
traceState(
|
||||||
|
"clipboard.stop.deferred",
|
||||||
|
extra: "reason=minRecording remaining=\(String(format: "%.2f", minimum - elapsed))"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pressEnded()
|
||||||
|
}
|
||||||
|
|
||||||
|
func refreshClipboardEligibility() {
|
||||||
|
let secure = fieldContextProvider()?.isSecureEntry == true
|
||||||
|
guard hasFullAccess(), !secure else {
|
||||||
|
clipboardEligibility = nil
|
||||||
|
clipboardPrewarmedChangeCount = nil
|
||||||
|
publishClipboardUIState()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let sample = ClipboardPasteboardReader.sample()
|
||||||
|
let previousChangeCount = clipboardEligibility?.changeCount
|
||||||
|
clipboardEligibility = ClipboardCommandEligibilityTracker.refresh(
|
||||||
|
changeCount: sample.changeCount,
|
||||||
|
rawText: sample.text,
|
||||||
|
previous: clipboardEligibility
|
||||||
|
)
|
||||||
|
|
||||||
|
if let session = clipboardTaskSession, !session.isActive() {
|
||||||
|
endClipboardTaskSession()
|
||||||
|
}
|
||||||
|
publishClipboardUIState()
|
||||||
|
requestClipboardLightPrewarmIfNeeded(previousChangeCount: previousChangeCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func endClipboardTaskSession() {
|
||||||
|
clipboardDeferredStopTask?.cancel()
|
||||||
|
clipboardDeferredStopTask = nil
|
||||||
|
clipboardTaskSession = nil
|
||||||
|
isClipboardCommandUtterance = false
|
||||||
|
clipboardAwaitingHostRecordConfirm = false
|
||||||
|
clipboardHostRecordConfirmedAt = nil
|
||||||
|
clipboardStopRequested = false
|
||||||
|
publishClipboardUIState()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func publishClipboardUIState() {
|
||||||
|
let now = Date().timeIntervalSince1970
|
||||||
|
state.clipboardCommandEligible = clipboardEligibility?.isOpen(at: now) == true
|
||||||
|
state.clipboardCommandSessionActive = clipboardTaskSession?.isActive(at: now) == true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Promote preparing → recording when host publishes real capture; honor deferred stop.
|
||||||
|
private func noteClipboardHostRecordingConfirmed() {
|
||||||
|
let now = Date().timeIntervalSince1970
|
||||||
|
if clipboardAwaitingHostRecordConfirm || clipboardHostRecordConfirmedAt == nil {
|
||||||
|
clipboardAwaitingHostRecordConfirm = false
|
||||||
|
if clipboardHostRecordConfirmedAt == nil {
|
||||||
|
clipboardHostRecordConfirmedAt = now
|
||||||
|
}
|
||||||
|
state.phase = .recording
|
||||||
|
if state.lastTranscript == ExtL10n.string("keyboard.placeholder.preparingRecording")
|
||||||
|
|| state.lastTranscript == ExtL10n.string("keyboard.placeholder.preparing") {
|
||||||
|
state.lastTranscript = ""
|
||||||
|
}
|
||||||
|
KeyboardHapticFeedback.play(role: .action, intensity: state.keyboardHapticIntensity)
|
||||||
|
traceState("clipboard.hostRecording.confirmed")
|
||||||
|
}
|
||||||
|
if clipboardStopRequested {
|
||||||
|
let confirmedAt = clipboardHostRecordConfirmedAt ?? now
|
||||||
|
let elapsed = now - confirmedAt
|
||||||
|
let minimum = ClipboardMaterialFilter.minimumRecordingAfterHostConfirm
|
||||||
|
if elapsed >= minimum {
|
||||||
|
pressEnded()
|
||||||
|
} else {
|
||||||
|
scheduleClipboardDeferredStop(after: minimum - elapsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func scheduleClipboardDeferredStop(after delay: TimeInterval) {
|
||||||
|
clipboardDeferredStopTask?.cancel()
|
||||||
|
let seconds = max(0.05, delay)
|
||||||
|
clipboardDeferredStopTask = Task { @MainActor [weak self] in
|
||||||
|
try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
|
||||||
|
guard let self, !Task.isCancelled else { return }
|
||||||
|
guard self.isClipboardCommandUtterance, self.clipboardStopRequested else { return }
|
||||||
|
guard self.isFlowRecording else { return }
|
||||||
|
self.pressEnded()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Light prewarm: ASR assets only (no mic). Once per eligible pasteboard change.
|
||||||
|
private func requestClipboardLightPrewarmIfNeeded(previousChangeCount: Int?) {
|
||||||
|
guard let eligibility = clipboardEligibility, eligibility.isOpen() else { return }
|
||||||
|
if clipboardPrewarmedChangeCount == eligibility.changeCount { return }
|
||||||
|
// Only fire when eligibility newly appears or changeCount advances.
|
||||||
|
if previousChangeCount == eligibility.changeCount { return }
|
||||||
|
|
||||||
|
guard let sessionId = FlowSessionBridge.readySnapshot()?.sessionId ?? activeSessionId else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
clipboardPrewarmedChangeCount = eligibility.changeCount
|
||||||
|
let command = FlowCommand(
|
||||||
|
sessionId: sessionId,
|
||||||
|
utteranceId: UUID(),
|
||||||
|
commandSeq: nextCommandSeq(),
|
||||||
|
action: .prewarm,
|
||||||
|
localeId: state.localeId
|
||||||
|
)
|
||||||
|
FlowSessionBridge.writeCommand(command)
|
||||||
|
FlowTrace.keyboard(
|
||||||
|
"command.prewarm",
|
||||||
|
"seq=\(command.commandSeq) changeCount=\(eligibility.changeCount)"
|
||||||
|
)
|
||||||
|
traceState("clipboard.prewarm.requested", extra: "changeCount=\(eligibility.changeCount)")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// If we already sent start and host is recording our utterance, confirm UI.
|
||||||
|
private func promoteClipboardRecordingFromSnapshotIfNeeded() {
|
||||||
|
guard isClipboardCommandUtterance, clipboardAwaitingHostRecordConfirm else { return }
|
||||||
|
guard let snapshot = FlowSessionBridge.readySnapshot(),
|
||||||
|
snapshot.reason == .recording,
|
||||||
|
let busyId = snapshot.busyUtteranceId,
|
||||||
|
busyId == currentUtteranceId else { return }
|
||||||
|
noteClipboardHostRecordingConfirmed()
|
||||||
|
}
|
||||||
|
|
||||||
func pressBegan() {
|
func pressBegan() {
|
||||||
switch state.phase {
|
switch state.phase {
|
||||||
case .idle, .denied, .error:
|
case .idle, .denied, .error:
|
||||||
@@ -526,6 +735,10 @@ final class KeyboardFlowCoordinator {
|
|||||||
}
|
}
|
||||||
guard isFlowRecording else { return }
|
guard isFlowRecording else { return }
|
||||||
|
|
||||||
|
clipboardDeferredStopTask?.cancel()
|
||||||
|
clipboardDeferredStopTask = nil
|
||||||
|
clipboardAwaitingHostRecordConfirm = false
|
||||||
|
clipboardStopRequested = false
|
||||||
isFlowRecording = false
|
isFlowRecording = false
|
||||||
stopUtteranceCountdown()
|
stopUtteranceCountdown()
|
||||||
ExtensionScreenWakeLock.release()
|
ExtensionScreenWakeLock.release()
|
||||||
@@ -606,6 +819,7 @@ final class KeyboardFlowCoordinator {
|
|||||||
state.level = 0
|
state.level = 0
|
||||||
recomputeMicVoiceAvailability()
|
recomputeMicVoiceAvailability()
|
||||||
}
|
}
|
||||||
|
endClipboardTaskSession()
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Private
|
// MARK: - Private
|
||||||
@@ -618,18 +832,30 @@ final class KeyboardFlowCoordinator {
|
|||||||
|
|
||||||
private func writeCommand(_ action: FlowCommand.Action) {
|
private func writeCommand(_ action: FlowCommand.Action) {
|
||||||
guard let activeSessionId, let currentUtteranceId else { return }
|
guard let activeSessionId, let currentUtteranceId else { return }
|
||||||
|
let mode: FlowUtteranceMode? = isClipboardCommandUtterance ? .clipboardCommand : nil
|
||||||
|
let snapshot: String? = {
|
||||||
|
guard isClipboardCommandUtterance, action == .startRecording else { return nil }
|
||||||
|
return clipboardTaskSession?.snapshot
|
||||||
|
}()
|
||||||
|
let previous: String? = {
|
||||||
|
guard isClipboardCommandUtterance, action == .startRecording else { return nil }
|
||||||
|
return clipboardTaskSession?.previousOutput
|
||||||
|
}()
|
||||||
let command = FlowCommand(
|
let command = FlowCommand(
|
||||||
sessionId: activeSessionId,
|
sessionId: activeSessionId,
|
||||||
utteranceId: currentUtteranceId,
|
utteranceId: currentUtteranceId,
|
||||||
commandSeq: nextCommandSeq(),
|
commandSeq: nextCommandSeq(),
|
||||||
action: action,
|
action: action,
|
||||||
localeId: state.localeId,
|
localeId: state.localeId,
|
||||||
fieldContext: action == .stopRecording ? fieldContextProvider() : nil
|
fieldContext: action == .stopRecording ? fieldContextProvider() : nil,
|
||||||
|
utteranceMode: mode,
|
||||||
|
clipboardSnapshot: snapshot,
|
||||||
|
previousOutput: previous
|
||||||
)
|
)
|
||||||
FlowSessionBridge.writeCommand(command)
|
FlowSessionBridge.writeCommand(command)
|
||||||
debug(
|
debug(
|
||||||
"command \(action.rawValue) seq=\(command.commandSeq) " +
|
"command \(action.rawValue) seq=\(command.commandSeq) " +
|
||||||
"utterance=\(currentUtteranceId.uuidString) contextChars=" +
|
"utterance=\(currentUtteranceId.uuidString) mode=\(mode?.rawValue ?? "dictation") contextChars=" +
|
||||||
"\(command.fieldContext?.precedingText?.count ?? 0)/" +
|
"\(command.fieldContext?.precedingText?.count ?? 0)/" +
|
||||||
"\(command.fieldContext?.followingText?.count ?? 0)"
|
"\(command.fieldContext?.followingText?.count ?? 0)"
|
||||||
)
|
)
|
||||||
@@ -640,6 +866,7 @@ final class KeyboardFlowCoordinator {
|
|||||||
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
|
"seq=\(command.commandSeq) utterance=\(currentUtteranceId.uuidString.prefix(8)) "
|
||||||
+ "locale=\(state.localeId) engine=\(state.engineMode) "
|
+ "locale=\(state.localeId) engine=\(state.engineMode) "
|
||||||
+ "hostReady=\(FlowSessionBridge.isHostReady() ? 1 : 0) "
|
+ "hostReady=\(FlowSessionBridge.isHostReady() ? 1 : 0) "
|
||||||
|
+ "mode=\(mode?.rawValue ?? "dictation")"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -648,9 +875,39 @@ final class KeyboardFlowCoordinator {
|
|||||||
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
if let result = matchingResult(), result.status == .final, let text = result.text, !text.isEmpty {
|
||||||
isAwaitingFlowResult = false
|
isAwaitingFlowResult = false
|
||||||
stopFlowWatchdog()
|
stopFlowWatchdog()
|
||||||
|
let wasClipboard = result.resolvedUtteranceMode == .clipboardCommand
|
||||||
|
|| isClipboardCommandUtterance
|
||||||
|
let replacePrevious: String? = {
|
||||||
|
guard wasClipboard,
|
||||||
|
let session = clipboardTaskSession,
|
||||||
|
let previous = session.lastInsertedText,
|
||||||
|
!previous.isEmpty else { return nil }
|
||||||
|
let fingerprint = fieldContextProvider()?.deliveryFingerprint
|
||||||
|
if let expected = session.fieldFingerprint,
|
||||||
|
let fingerprint,
|
||||||
|
expected != fingerprint {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return previous
|
||||||
|
}()
|
||||||
textInserter.handleFlowTranscript(
|
textInserter.handleFlowTranscript(
|
||||||
TranscriptionDelivery(text: text, polishWarning: result.warning)
|
TranscriptionDelivery(text: text, polishWarning: result.warning),
|
||||||
|
replacePrevious: replacePrevious
|
||||||
)
|
)
|
||||||
|
if wasClipboard {
|
||||||
|
let fingerprint = fieldContextProvider()?.deliveryFingerprint
|
||||||
|
if clipboardTaskSession == nil {
|
||||||
|
clipboardTaskSession = ClipboardCommandTaskSession(
|
||||||
|
snapshot: clipboardEligibility?.snapshot ?? "",
|
||||||
|
expiresAt: Date().timeIntervalSince1970 + ClipboardMaterialFilter.sessionDuration,
|
||||||
|
fieldFingerprint: fingerprint
|
||||||
|
)
|
||||||
|
}
|
||||||
|
clipboardTaskSession?.noteSuccessfulInsert(text)
|
||||||
|
clipboardTaskSession?.fieldFingerprint = fingerprint
|
||||||
|
publishClipboardUIState()
|
||||||
|
}
|
||||||
|
isClipboardCommandUtterance = false
|
||||||
FlowSessionBridge.writeAck(
|
FlowSessionBridge.writeAck(
|
||||||
FlowAck(
|
FlowAck(
|
||||||
sessionId: result.sessionId,
|
sessionId: result.sessionId,
|
||||||
@@ -669,6 +926,7 @@ final class KeyboardFlowCoordinator {
|
|||||||
text,
|
text,
|
||||||
"utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
"utterance=\(result.utteranceId.uuidString.prefix(8)) "
|
||||||
+ "commandSeq=\(result.commandSeq) warning=\(result.warning == nil ? 0 : 1) "
|
+ "commandSeq=\(result.commandSeq) warning=\(result.warning == nil ? 0 : 1) "
|
||||||
|
+ "clipboard=\(wasClipboard ? 1 : 0)"
|
||||||
)
|
)
|
||||||
recomputeMicVoiceAvailability()
|
recomputeMicVoiceAvailability()
|
||||||
return
|
return
|
||||||
@@ -683,6 +941,8 @@ final class KeyboardFlowCoordinator {
|
|||||||
)
|
)
|
||||||
isAwaitingFlowResult = false
|
isAwaitingFlowResult = false
|
||||||
stopFlowWatchdog()
|
stopFlowWatchdog()
|
||||||
|
isClipboardCommandUtterance = false
|
||||||
|
publishClipboardUIState()
|
||||||
FlowSessionBridge.writeAck(
|
FlowSessionBridge.writeAck(
|
||||||
FlowAck(
|
FlowAck(
|
||||||
sessionId: result.sessionId,
|
sessionId: result.sessionId,
|
||||||
@@ -828,6 +1088,7 @@ final class KeyboardFlowCoordinator {
|
|||||||
private func deliverRawFallbackIfAvailable(reason: String) -> Bool {
|
private func deliverRawFallbackIfAvailable(reason: String) -> Bool {
|
||||||
FlowSessionBridge.reloadFromDisk()
|
FlowSessionBridge.reloadFromDisk()
|
||||||
guard let result = matchingResult(),
|
guard let result = matchingResult(),
|
||||||
|
result.allowsRawFallback,
|
||||||
result.status == .partial
|
result.status == .partial
|
||||||
|| result.status == .rawReady
|
|| result.status == .rawReady
|
||||||
|| (result.status == .final && result.rawText != nil),
|
|| (result.status == .final && result.rawText != nil),
|
||||||
@@ -953,15 +1214,31 @@ final class KeyboardFlowCoordinator {
|
|||||||
lastStoppedUtteranceId = nil
|
lastStoppedUtteranceId = nil
|
||||||
writeCommand(.startRecording)
|
writeCommand(.startRecording)
|
||||||
isFlowRecording = true
|
isFlowRecording = true
|
||||||
|
if isClipboardCommandUtterance {
|
||||||
|
clipboardAwaitingHostRecordConfirm = true
|
||||||
|
clipboardHostRecordConfirmedAt = nil
|
||||||
|
clipboardStopRequested = false
|
||||||
|
clipboardDeferredStopTask?.cancel()
|
||||||
|
clipboardDeferredStopTask = nil
|
||||||
|
state.phase = .requestingPermissions
|
||||||
|
state.lastTranscript = ExtL10n.string("keyboard.placeholder.preparingRecording")
|
||||||
|
} else {
|
||||||
|
clipboardAwaitingHostRecordConfirm = false
|
||||||
|
clipboardHostRecordConfirmedAt = nil
|
||||||
|
clipboardStopRequested = false
|
||||||
state.lastTranscript = ""
|
state.lastTranscript = ""
|
||||||
state.phase = .recording
|
state.phase = .recording
|
||||||
|
}
|
||||||
recomputeMicVoiceAvailability()
|
recomputeMicVoiceAvailability()
|
||||||
if let view = wakeLockView() {
|
if let view = wakeLockView() {
|
||||||
ExtensionScreenWakeLock.acquire(from: view)
|
ExtensionScreenWakeLock.acquire(from: view)
|
||||||
}
|
}
|
||||||
startUtteranceCountdown()
|
startUtteranceCountdown()
|
||||||
startFlowLevelWatchdog()
|
startFlowLevelWatchdog()
|
||||||
traceState("startFlowRecording.started")
|
traceState(
|
||||||
|
"startFlowRecording.started",
|
||||||
|
extra: isClipboardCommandUtterance ? "clipboardPreparing=1" : "clipboardPreparing=0"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func startUtteranceCountdown() {
|
private func startUtteranceCountdown() {
|
||||||
|
|||||||
@@ -10,28 +10,44 @@ import OSGKeyboardShared
|
|||||||
final class KeyboardTextInserter {
|
final class KeyboardTextInserter {
|
||||||
private let state: KeyboardState
|
private let state: KeyboardState
|
||||||
private let insertText: (String) -> Void
|
private let insertText: (String) -> Void
|
||||||
|
private let deleteBackward: () -> Void
|
||||||
private let contextBeforeInput: () -> String?
|
private let contextBeforeInput: () -> String?
|
||||||
private let scheduleAutoClearError: () -> Void
|
private let scheduleAutoClearError: () -> Void
|
||||||
|
|
||||||
init(
|
init(
|
||||||
state: KeyboardState,
|
state: KeyboardState,
|
||||||
insertText: @escaping (String) -> Void,
|
insertText: @escaping (String) -> Void,
|
||||||
|
deleteBackward: @escaping () -> Void,
|
||||||
contextBeforeInput: @escaping () -> String?,
|
contextBeforeInput: @escaping () -> String?,
|
||||||
scheduleAutoClearError: @escaping () -> Void
|
scheduleAutoClearError: @escaping () -> Void
|
||||||
) {
|
) {
|
||||||
self.state = state
|
self.state = state
|
||||||
self.insertText = insertText
|
self.insertText = insertText
|
||||||
|
self.deleteBackward = deleteBackward
|
||||||
self.contextBeforeInput = contextBeforeInput
|
self.contextBeforeInput = contextBeforeInput
|
||||||
self.scheduleAutoClearError = scheduleAutoClearError
|
self.scheduleAutoClearError = scheduleAutoClearError
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleFlowTranscript(_ delivery: TranscriptionDelivery) {
|
func handleFlowTranscript(
|
||||||
|
_ delivery: TranscriptionDelivery,
|
||||||
|
replacePrevious: String? = nil
|
||||||
|
) {
|
||||||
let trimmed = delivery.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
let trimmed = delivery.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
guard !trimmed.isEmpty else {
|
guard !trimmed.isEmpty else {
|
||||||
state.phase = .idle
|
state.phase = .idle
|
||||||
state.level = 0
|
state.level = 0
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let previous = replacePrevious?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||||
|
!previous.isEmpty,
|
||||||
|
let preceding = contextBeforeInput(),
|
||||||
|
preceding.hasSuffix(previous) {
|
||||||
|
for _ in 0..<previous.count {
|
||||||
|
deleteBackward()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Host app already polished when configured; keyboard only inserts.
|
// Host app already polished when configured; keyboard only inserts.
|
||||||
// Word-boundary hygiene: dictating "world" with the cursor right
|
// Word-boundary hygiene: dictating "world" with the cursor right
|
||||||
// after "Hello" must yield "Hello world", not "Helloworld".
|
// after "Hello" must yield "Hello world", not "Helloworld".
|
||||||
|
|||||||
@@ -132,6 +132,7 @@ public struct KeyboardRootView: View {
|
|||||||
transcript: state.lastTranscript,
|
transcript: state.lastTranscript,
|
||||||
micVoiceAvailability: state.micVoiceAvailability,
|
micVoiceAvailability: state.micVoiceAvailability,
|
||||||
micDisabledHint: state.micDisabledHint,
|
micDisabledHint: state.micDisabledHint,
|
||||||
|
clipboardCommandEligible: state.clipboardCommandEligible || state.clipboardCommandSessionActive,
|
||||||
cursorDragHintActive: state.cursorDragActive,
|
cursorDragHintActive: state.cursorDragActive,
|
||||||
openSettings: state.openSettings
|
openSettings: state.openSettings
|
||||||
)
|
)
|
||||||
@@ -182,7 +183,13 @@ public struct KeyboardRootView: View {
|
|||||||
level: state.level,
|
level: state.level,
|
||||||
remainingSeconds: state.phase == .recording ? state.utteranceRemainingSeconds : nil,
|
remainingSeconds: state.phase == .recording ? state.utteranceRemainingSeconds : nil,
|
||||||
isEnabled: !state.micDisabled,
|
isEnabled: !state.micDisabled,
|
||||||
onToggle: state.tapMic
|
onToggle: state.tapMic,
|
||||||
|
onClipboardLongPressBegan: (state.clipboardCommandEligible || state.clipboardCommandSessionActive)
|
||||||
|
? state.beginClipboardCommand
|
||||||
|
: nil,
|
||||||
|
onClipboardLongPressEnded: (state.clipboardCommandEligible || state.clipboardCommandSessionActive)
|
||||||
|
? state.endClipboardCommand
|
||||||
|
: nil
|
||||||
)
|
)
|
||||||
.frame(width: KeyboardLayoutMetrics.micSize, height: KeyboardLayoutMetrics.micSize)
|
.frame(width: KeyboardLayoutMetrics.micSize, height: KeyboardLayoutMetrics.micSize)
|
||||||
.offset(y: -KeyboardLayoutMetrics.micUpwardAdjustment)
|
.offset(y: -KeyboardLayoutMetrics.micUpwardAdjustment)
|
||||||
@@ -339,6 +346,7 @@ private struct TranscriptLine: View {
|
|||||||
let transcript: String
|
let transcript: String
|
||||||
let micVoiceAvailability: MicVoiceAvailability
|
let micVoiceAvailability: MicVoiceAvailability
|
||||||
let micDisabledHint: String
|
let micDisabledHint: String
|
||||||
|
let clipboardCommandEligible: Bool
|
||||||
let cursorDragHintActive: Bool
|
let cursorDragHintActive: Bool
|
||||||
let openSettings: () -> Void
|
let openSettings: () -> Void
|
||||||
|
|
||||||
@@ -363,7 +371,11 @@ private struct TranscriptLine: View {
|
|||||||
case .requestingPermissions:
|
case .requestingPermissions:
|
||||||
HStack(spacing: 6) {
|
HStack(spacing: 6) {
|
||||||
ProgressView().controlSize(.mini).tint(palette.textSecondary)
|
ProgressView().controlSize(.mini).tint(palette.textSecondary)
|
||||||
ExtL10n.text("keyboard.placeholder.preparing")
|
Text(
|
||||||
|
transcript.isEmpty
|
||||||
|
? ExtL10n.string("keyboard.placeholder.preparing")
|
||||||
|
: transcript
|
||||||
|
)
|
||||||
.font(TypeStyle.caption)
|
.font(TypeStyle.caption)
|
||||||
.foregroundStyle(palette.textSecondary)
|
.foregroundStyle(palette.textSecondary)
|
||||||
}
|
}
|
||||||
@@ -420,13 +432,25 @@ private struct TranscriptLine: View {
|
|||||||
Group {
|
Group {
|
||||||
switch micVoiceAvailability {
|
switch micVoiceAvailability {
|
||||||
case .ready:
|
case .ready:
|
||||||
|
if clipboardCommandEligible {
|
||||||
|
ExtL10n.text("keyboard.placeholder.idleClipboard")
|
||||||
|
} else {
|
||||||
ExtL10n.text("keyboard.placeholder.idle")
|
ExtL10n.text("keyboard.placeholder.idle")
|
||||||
|
}
|
||||||
case .unavailable(.missingAPIKey):
|
case .unavailable(.missingAPIKey):
|
||||||
Text(micDisabledHint)
|
Text(micDisabledHint)
|
||||||
case .unavailable(.hostNotReady):
|
case .unavailable(.hostNotReady):
|
||||||
|
if clipboardCommandEligible {
|
||||||
|
ExtL10n.text("keyboard.placeholder.idleClipboard")
|
||||||
|
} else {
|
||||||
ExtL10n.text("keyboard.placeholder.idle")
|
ExtL10n.text("keyboard.placeholder.idle")
|
||||||
|
}
|
||||||
case .unavailable(.preparingSession):
|
case .unavailable(.preparingSession):
|
||||||
|
if clipboardCommandEligible {
|
||||||
|
ExtL10n.text("keyboard.placeholder.idleClipboard")
|
||||||
|
} else {
|
||||||
ExtL10n.text("keyboard.placeholder.idle")
|
ExtL10n.text("keyboard.placeholder.idle")
|
||||||
|
}
|
||||||
case .unavailable(.noFullAccess):
|
case .unavailable(.noFullAccess):
|
||||||
ExtL10n.text("keyboard.error.fullAccessRequired")
|
ExtL10n.text("keyboard.error.fullAccessRequired")
|
||||||
case .unavailable(.appGroupUnavailable):
|
case .unavailable(.appGroupUnavailable):
|
||||||
|
|||||||
@@ -115,7 +115,9 @@
|
|||||||
|
|
||||||
/* Keyboard (ext) */
|
/* Keyboard (ext) */
|
||||||
"keyboard.placeholder.idle" = "Tap to talk";
|
"keyboard.placeholder.idle" = "Tap to talk";
|
||||||
|
"keyboard.placeholder.idleClipboard" = "Tap to talk, long-press for clipboard";
|
||||||
"keyboard.placeholder.preparing" = "Preparing";
|
"keyboard.placeholder.preparing" = "Preparing";
|
||||||
|
"keyboard.placeholder.preparingRecording" = "Preparing mic…";
|
||||||
"keyboard.placeholder.processing" = "Processing";
|
"keyboard.placeholder.processing" = "Processing";
|
||||||
"keyboard.placeholder.error" = "Polishing failed";
|
"keyboard.placeholder.error" = "Polishing failed";
|
||||||
"keyboard.placeholder.localBadge" = "On-device";
|
"keyboard.placeholder.localBadge" = "On-device";
|
||||||
|
|||||||
@@ -115,7 +115,9 @@
|
|||||||
|
|
||||||
/* Keyboard (ext) */
|
/* Keyboard (ext) */
|
||||||
"keyboard.placeholder.idle" = "点按说话";
|
"keyboard.placeholder.idle" = "点按说话";
|
||||||
|
"keyboard.placeholder.idleClipboard" = "点击说话,长按处理剪贴板";
|
||||||
"keyboard.placeholder.preparing" = "准备中…";
|
"keyboard.placeholder.preparing" = "准备中…";
|
||||||
|
"keyboard.placeholder.preparingRecording" = "准备录音…";
|
||||||
"keyboard.placeholder.processing" = "处理中…";
|
"keyboard.placeholder.processing" = "处理中…";
|
||||||
"keyboard.placeholder.error" = "润色失败";
|
"keyboard.placeholder.error" = "润色失败";
|
||||||
"keyboard.placeholder.localBadge" = "本地";
|
"keyboard.placeholder.localBadge" = "本地";
|
||||||
|
|||||||
@@ -24,21 +24,29 @@ public struct RecordButton: View {
|
|||||||
public let remainingSeconds: Int?
|
public let remainingSeconds: Int?
|
||||||
public let isEnabled: Bool
|
public let isEnabled: Bool
|
||||||
public let onToggle: () -> Void
|
public let onToggle: () -> Void
|
||||||
|
/// When non-nil, a 0.45s hold starts clipboard-command recording instead of toggle.
|
||||||
|
public let onClipboardLongPressBegan: (() -> Void)?
|
||||||
|
public let onClipboardLongPressEnded: (() -> Void)?
|
||||||
|
|
||||||
@State private var breath = false
|
@State private var breath = false
|
||||||
|
@State private var longPressArmed = false
|
||||||
|
|
||||||
public init(
|
public init(
|
||||||
phase: Phase,
|
phase: Phase,
|
||||||
level: Double,
|
level: Double,
|
||||||
remainingSeconds: Int? = nil,
|
remainingSeconds: Int? = nil,
|
||||||
isEnabled: Bool = true,
|
isEnabled: Bool = true,
|
||||||
onToggle: @escaping () -> Void
|
onToggle: @escaping () -> Void,
|
||||||
|
onClipboardLongPressBegan: (() -> Void)? = nil,
|
||||||
|
onClipboardLongPressEnded: (() -> Void)? = nil
|
||||||
) {
|
) {
|
||||||
self.phase = phase
|
self.phase = phase
|
||||||
self.level = level
|
self.level = level
|
||||||
self.remainingSeconds = remainingSeconds
|
self.remainingSeconds = remainingSeconds
|
||||||
self.isEnabled = isEnabled
|
self.isEnabled = isEnabled
|
||||||
self.onToggle = onToggle
|
self.onToggle = onToggle
|
||||||
|
self.onClipboardLongPressBegan = onClipboardLongPressBegan
|
||||||
|
self.onClipboardLongPressEnded = onClipboardLongPressEnded
|
||||||
}
|
}
|
||||||
|
|
||||||
private var isUrgent: Bool {
|
private var isUrgent: Bool {
|
||||||
@@ -131,14 +139,23 @@ public struct RecordButton: View {
|
|||||||
.animation(Motion.soft, value: remainingSeconds)
|
.animation(Motion.soft, value: remainingSeconds)
|
||||||
}
|
}
|
||||||
.contentShape(Circle())
|
.contentShape(Circle())
|
||||||
.onTapGesture {
|
.modifier(
|
||||||
guard phase != .processing else { return }
|
RecordButtonPressModifier(
|
||||||
guard isEnabled || phase == .idleUnavailable else { return }
|
phase: phase,
|
||||||
onToggle()
|
isEnabled: isEnabled,
|
||||||
}
|
supportsClipboardLongPress: onClipboardLongPressBegan != nil,
|
||||||
|
longPressArmed: $longPressArmed,
|
||||||
|
onToggle: onToggle,
|
||||||
|
onClipboardLongPressBegan: onClipboardLongPressBegan,
|
||||||
|
onClipboardLongPressEnded: onClipboardLongPressEnded
|
||||||
|
)
|
||||||
|
)
|
||||||
.onAppear { breath = (phase == .recording) }
|
.onAppear { breath = (phase == .recording) }
|
||||||
.onChange(of: phase) { _, new in
|
.onChange(of: phase) { _, new in
|
||||||
breath = (new == .recording)
|
breath = (new == .recording)
|
||||||
|
if new != .recording {
|
||||||
|
longPressArmed = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.accessibilityLabel(Text(SharedL10n.string("keyboard.tapToTalkA11y")))
|
.accessibilityLabel(Text(SharedL10n.string("keyboard.tapToTalkA11y")))
|
||||||
}
|
}
|
||||||
@@ -186,3 +203,55 @@ public struct RecordButton: View {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: - Press / long-press routing
|
||||||
|
|
||||||
|
private struct RecordButtonPressModifier: ViewModifier {
|
||||||
|
let phase: RecordButton.Phase
|
||||||
|
let isEnabled: Bool
|
||||||
|
let supportsClipboardLongPress: Bool
|
||||||
|
@Binding var longPressArmed: Bool
|
||||||
|
let onToggle: () -> Void
|
||||||
|
let onClipboardLongPressBegan: (() -> Void)?
|
||||||
|
let onClipboardLongPressEnded: (() -> Void)?
|
||||||
|
|
||||||
|
func body(content: Content) -> some View {
|
||||||
|
if supportsClipboardLongPress {
|
||||||
|
content.onLongPressGesture(
|
||||||
|
minimumDuration: ClipboardMaterialFilter.longPressDuration,
|
||||||
|
maximumDistance: 120,
|
||||||
|
pressing: { pressing in
|
||||||
|
if pressing {
|
||||||
|
longPressArmed = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Released.
|
||||||
|
if longPressArmed {
|
||||||
|
onClipboardLongPressEnded?()
|
||||||
|
longPressArmed = false
|
||||||
|
} else if phase != .processing, isEnabled || phase == .idleUnavailable {
|
||||||
|
// Short press → existing tap-toggle dictation.
|
||||||
|
guard phase != .recording else {
|
||||||
|
// If somehow recording without arming, end via toggle.
|
||||||
|
onToggle()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
onToggle()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
perform: {
|
||||||
|
guard phase != .processing else { return }
|
||||||
|
guard isEnabled || phase == .idleUnavailable else { return }
|
||||||
|
longPressArmed = true
|
||||||
|
onClipboardLongPressBegan?()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
content.onTapGesture {
|
||||||
|
guard phase != .processing else { return }
|
||||||
|
guard isEnabled || phase == .idleUnavailable else { return }
|
||||||
|
onToggle()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
// FlowUtteranceMode.swift
|
||||||
|
// OSGKeyboard · Shared
|
||||||
|
//
|
||||||
|
// Distinguishes dictation polish from clipboard-command generation on the
|
||||||
|
// Flow command / result wire (plan §11).
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
public enum FlowUtteranceMode: String, Codable, Equatable, Sendable {
|
||||||
|
/// ASR is draft text to polish and insert (default / legacy).
|
||||||
|
case dictation
|
||||||
|
/// ASR is an instruction over a frozen clipboard snapshot.
|
||||||
|
case clipboardCommand
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
// ClipboardCommandEligibility.swift
|
||||||
|
// OSGKeyboard · Shared
|
||||||
|
//
|
||||||
|
// Tracks clipboard open-window eligibility (30s from first sighting of a
|
||||||
|
// pasteboard change) for opportunity-read UI. Pure timing logic — no UIKit.
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
public struct ClipboardCommandEligibility: Equatable, Sendable {
|
||||||
|
public var changeCount: Int
|
||||||
|
public var snapshot: String
|
||||||
|
public var startedAt: TimeInterval
|
||||||
|
|
||||||
|
public init(changeCount: Int, snapshot: String, startedAt: TimeInterval = Date().timeIntervalSince1970) {
|
||||||
|
self.changeCount = changeCount
|
||||||
|
self.snapshot = snapshot
|
||||||
|
self.startedAt = startedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
public func isOpen(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
||||||
|
now - startedAt <= ClipboardMaterialFilter.eligibilityDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
public func remaining(at now: TimeInterval = Date().timeIntervalSince1970) -> TimeInterval {
|
||||||
|
max(0, ClipboardMaterialFilter.eligibilityDuration - (now - startedAt))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum ClipboardCommandEligibilityTracker: Sendable {
|
||||||
|
/// Update eligibility from an opportunity-read sample.
|
||||||
|
/// - Parameters:
|
||||||
|
/// - changeCount: `UIPasteboard.general.changeCount`
|
||||||
|
/// - rawText: pasteboard string (may be nil)
|
||||||
|
/// - previous: last known eligibility
|
||||||
|
/// - now: clock
|
||||||
|
public static func refresh(
|
||||||
|
changeCount: Int,
|
||||||
|
rawText: String?,
|
||||||
|
previous: ClipboardCommandEligibility?,
|
||||||
|
now: TimeInterval = Date().timeIntervalSince1970
|
||||||
|
) -> ClipboardCommandEligibility? {
|
||||||
|
guard let rawText else { return nil }
|
||||||
|
|
||||||
|
if let previous, previous.changeCount == changeCount {
|
||||||
|
return previous.isOpen(at: now) ? previous : nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch ClipboardMaterialFilter.evaluate(rawText) {
|
||||||
|
case .eligible(let snapshot):
|
||||||
|
return ClipboardCommandEligibility(
|
||||||
|
changeCount: changeCount,
|
||||||
|
snapshot: snapshot,
|
||||||
|
startedAt: now
|
||||||
|
)
|
||||||
|
case .rejected:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// In-memory clipboard-command task session (plan §8 layer B). Owned by the keyboard.
|
||||||
|
public struct ClipboardCommandTaskSession: Equatable, Sendable {
|
||||||
|
public var snapshot: String
|
||||||
|
public var previousOutput: String?
|
||||||
|
public var lastInsertedText: String?
|
||||||
|
public var expiresAt: TimeInterval
|
||||||
|
public var fieldFingerprint: String?
|
||||||
|
|
||||||
|
public init(
|
||||||
|
snapshot: String,
|
||||||
|
previousOutput: String? = nil,
|
||||||
|
lastInsertedText: String? = nil,
|
||||||
|
expiresAt: TimeInterval,
|
||||||
|
fieldFingerprint: String? = nil
|
||||||
|
) {
|
||||||
|
self.snapshot = snapshot
|
||||||
|
self.previousOutput = previousOutput
|
||||||
|
self.lastInsertedText = lastInsertedText
|
||||||
|
self.expiresAt = expiresAt
|
||||||
|
self.fieldFingerprint = fieldFingerprint
|
||||||
|
}
|
||||||
|
|
||||||
|
public func isActive(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
||||||
|
now <= expiresAt
|
||||||
|
}
|
||||||
|
|
||||||
|
public mutating func refreshExpiry(at now: TimeInterval = Date().timeIntervalSince1970) {
|
||||||
|
expiresAt = now + ClipboardMaterialFilter.sessionDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
public mutating func noteSuccessfulInsert(_ text: String, at now: TimeInterval = Date().timeIntervalSince1970) {
|
||||||
|
lastInsertedText = text
|
||||||
|
previousOutput = text
|
||||||
|
refreshExpiry(at: now)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
// ClipboardCommandPromptComposer.swift
|
||||||
|
// OSGKeyboard · Shared
|
||||||
|
//
|
||||||
|
// Prompt assembly for clipboard-command mode (plan §11).
|
||||||
|
// Intentionally separate from PolishPromptComposer — ASR is an instruction,
|
||||||
|
// not draft text (R6 must not apply).
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
public enum ClipboardCommandPromptComposer {
|
||||||
|
|
||||||
|
public struct Input: Equatable, Sendable {
|
||||||
|
public var snapshot: String
|
||||||
|
public var instruction: String
|
||||||
|
public var previousOutput: String?
|
||||||
|
/// Short style bias from the active Style Pack (B1).
|
||||||
|
public var styleBias: String?
|
||||||
|
|
||||||
|
public init(
|
||||||
|
snapshot: String,
|
||||||
|
instruction: String,
|
||||||
|
previousOutput: String? = nil,
|
||||||
|
styleBias: String? = nil
|
||||||
|
) {
|
||||||
|
self.snapshot = snapshot
|
||||||
|
self.instruction = instruction
|
||||||
|
self.previousOutput = previousOutput
|
||||||
|
self.styleBias = styleBias
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static func compose(_ input: Input, language: AppUILanguage? = nil) -> String {
|
||||||
|
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
|
||||||
|
var parts: [String] = [useChinese ? chineseCore : englishCore]
|
||||||
|
|
||||||
|
if let bias = normalized(input.styleBias), !bias.isEmpty {
|
||||||
|
let header = useChinese ? "# 语气底色(弱偏置;口述指令优先)" : "# Tone bias (weak; spoken instruction wins)"
|
||||||
|
parts.append(header)
|
||||||
|
// Keep bias short so it cannot drown the command contract.
|
||||||
|
parts.append(String(bias.prefix(800)))
|
||||||
|
}
|
||||||
|
return parts.joined(separator: "\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// User-turn payload (material / instruction / previous output).
|
||||||
|
public static func userMessage(_ input: Input, language: AppUILanguage? = nil) -> String {
|
||||||
|
let useChinese = (language ?? .auto).resolvedLanguageCode().hasPrefix("zh")
|
||||||
|
return userPayload(input, chinese: useChinese)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// B1: derive a short bias string from the active pack without shipping the
|
||||||
|
/// full dictation personality prompt.
|
||||||
|
public static func styleBias(
|
||||||
|
styleID: String,
|
||||||
|
catalog: PolishStyleCatalog,
|
||||||
|
maxCharacters: Int = 400
|
||||||
|
) -> String? {
|
||||||
|
let pack = PolishStylePackCatalog.resolve(id: styleID, userCatalog: catalog)
|
||||||
|
let personality = PolishStylePackCatalog.runtimePersonality(for: pack)
|
||||||
|
let trimmed = personality.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !trimmed.isEmpty else { return nil }
|
||||||
|
if trimmed.count <= maxCharacters { return trimmed }
|
||||||
|
let end = trimmed.index(trimmed.startIndex, offsetBy: maxCharacters)
|
||||||
|
return String(trimmed[..<end])
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Private
|
||||||
|
|
||||||
|
private static func userPayload(_ input: Input, chinese: Bool) -> String {
|
||||||
|
var lines: [String] = []
|
||||||
|
lines.append(chinese ? "【材料】" : "[Material]")
|
||||||
|
lines.append(ClipboardMaterialFilter.truncateSnapshot(input.snapshot))
|
||||||
|
lines.append("")
|
||||||
|
lines.append(chinese ? "【指令】" : "[Instruction]")
|
||||||
|
lines.append(input.instruction.trimmingCharacters(in: .whitespacesAndNewlines))
|
||||||
|
if let previous = normalized(input.previousOutput) {
|
||||||
|
lines.append("")
|
||||||
|
lines.append(chinese ? "【上一版结果】" : "[Previous output]")
|
||||||
|
lines.append(previous)
|
||||||
|
}
|
||||||
|
return lines.joined(separator: "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func normalized(_ value: String?) -> String? {
|
||||||
|
guard let value else { return nil }
|
||||||
|
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
return trimmed.isEmpty ? nil : trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
private static let chineseCore = """
|
||||||
|
你是输入法里的剪贴板写作助手。用户提供一段【材料】(剪贴板内容)和一条【指令】(语音转写)。
|
||||||
|
你的任务是按指令处理材料,输出用户可以直接发送或粘贴的最终文本。
|
||||||
|
|
||||||
|
# 全局契约(最高优先级)
|
||||||
|
C1 只输出最终文本:不解释、不加引号、不用 markdown 代码块、不写「好的,以下是…」之类前缀。
|
||||||
|
C2 【指令】优先于任何语气底色;指令要求的语气、目的、篇幅必须遵守。
|
||||||
|
C3 不要编造材料中没有的关键事实(人名、时间、金额、约定);语气发挥(安慰、拒绝等)允许,但不要捏造情节。
|
||||||
|
C4 若有【上一版结果】,在上一版基础上按新指令修订,不要重复堆叠无关内容。
|
||||||
|
C5 材料若注明已截断,只基于可见部分处理。
|
||||||
|
C6 输出语言跟随指令与材料的主导语言;指令要求翻译时才翻译。
|
||||||
|
"""
|
||||||
|
|
||||||
|
private static let englishCore = """
|
||||||
|
You are a clipboard writing assistant inside a keyboard. The user provides [Material] (clipboard text) and an [Instruction] (speech transcript).
|
||||||
|
Produce final text the user can send or paste immediately.
|
||||||
|
|
||||||
|
# Global contract (highest priority)
|
||||||
|
C1 Output final text only: no explanation, quotes, markdown fences, or preamble such as "Sure, here is…".
|
||||||
|
C2 The [Instruction] outranks any tone bias; honor requested tone, intent, and length.
|
||||||
|
C3 Do not invent key facts absent from the material (names, times, amounts, commitments). Tone (comfort, decline, etc.) may be creative without fabricating plot.
|
||||||
|
C4 If [Previous output] is present, revise that draft per the new instruction; do not stack unrelated duplicates.
|
||||||
|
C5 If material is marked truncated, use only the visible portion.
|
||||||
|
C6 Follow the dominant language of instruction and material; translate only when asked.
|
||||||
|
"""
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
// ClipboardMaterialFilter.swift
|
||||||
|
// OSGKeyboard · Shared
|
||||||
|
//
|
||||||
|
// Pure eligibility rules for clipboard-command mode (plan §4 R0–R6 content rules).
|
||||||
|
// Runtime gates (secure field, Full Access) live in the keyboard extension.
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
public enum ClipboardMaterialFilter: Sendable {
|
||||||
|
|
||||||
|
public static let minimumLength = 15
|
||||||
|
public static let maxSnapshotLength = 3_000
|
||||||
|
public static let eligibilityDuration: TimeInterval = 30
|
||||||
|
public static let sessionDuration: TimeInterval = 30
|
||||||
|
public static let longPressDuration: TimeInterval = 0.45
|
||||||
|
/// After the host confirms real capture, keep recording at least this long
|
||||||
|
/// before honoring finger-up (avoids near-silent cold-start tails).
|
||||||
|
public static let minimumRecordingAfterHostConfirm: TimeInterval = 0.70
|
||||||
|
|
||||||
|
public enum Rejection: String, Equatable, Sendable {
|
||||||
|
case empty
|
||||||
|
case phoneOrNumeric
|
||||||
|
case emojiOrSymbolOnly
|
||||||
|
case verificationCode
|
||||||
|
case tooShort
|
||||||
|
case repetitiveSpam
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum Verdict: Equatable, Sendable {
|
||||||
|
case eligible(String)
|
||||||
|
case rejected(Rejection)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evaluate trimmed clipboard text for command-mode entry.
|
||||||
|
public static func evaluate(_ raw: String) -> Verdict {
|
||||||
|
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !trimmed.isEmpty else { return .rejected(.empty) }
|
||||||
|
|
||||||
|
if isPhoneOrNumeric(trimmed) { return .rejected(.phoneOrNumeric) }
|
||||||
|
if isEmojiOrSymbolOnly(trimmed) { return .rejected(.emojiOrSymbolOnly) }
|
||||||
|
if isVerificationCode(trimmed) { return .rejected(.verificationCode) }
|
||||||
|
if trimmed.count < minimumLength { return .rejected(.tooShort) }
|
||||||
|
if isRepetitiveSpam(trimmed) { return .rejected(.repetitiveSpam) }
|
||||||
|
|
||||||
|
return .eligible(truncateSnapshot(trimmed))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wire / LLM snapshot cap (plan: 3000 grapheme clusters).
|
||||||
|
public static func truncateSnapshot(_ text: String) -> String {
|
||||||
|
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard trimmed.count > maxSnapshotLength else { return trimmed }
|
||||||
|
let end = trimmed.index(trimmed.startIndex, offsetBy: maxSnapshotLength)
|
||||||
|
return String(trimmed[..<end])
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Rules
|
||||||
|
|
||||||
|
/// R1: whole string looks like a phone / order number after stripping whitespace.
|
||||||
|
private static func isPhoneOrNumeric(_ text: String) -> Bool {
|
||||||
|
let compact = text.filter { !$0.isWhitespace }
|
||||||
|
guard !compact.isEmpty else { return false }
|
||||||
|
let allowed = CharacterSet(charactersIn: "0123456789-+()")
|
||||||
|
guard compact.unicodeScalars.allSatisfy({ allowed.contains($0) }) else { return false }
|
||||||
|
return compact.contains { $0.isNumber }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// R2: no letter, CJK, or digit — only emoji / punctuation / symbols.
|
||||||
|
private static func isEmojiOrSymbolOnly(_ text: String) -> Bool {
|
||||||
|
let compact = text.filter { !$0.isWhitespace }
|
||||||
|
guard !compact.isEmpty else { return false }
|
||||||
|
return !compact.contains { characterHasLetterOrNumber($0) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// R3: length 4…8, alphanumeric only, mixed letters + digits.
|
||||||
|
private static func isVerificationCode(_ text: String) -> Bool {
|
||||||
|
let compact = text.filter { !$0.isWhitespace }
|
||||||
|
guard (4...8).contains(compact.count) else { return false }
|
||||||
|
guard compact.allSatisfy({ $0.isLetter || $0.isNumber }) else { return false }
|
||||||
|
let hasLetter = compact.contains(where: \.isLetter)
|
||||||
|
let hasDigit = compact.contains(where: \.isNumber)
|
||||||
|
return hasLetter && hasDigit
|
||||||
|
}
|
||||||
|
|
||||||
|
/// R5: length ≥ 15, ≤2 distinct characters, one char ≥ 80% share.
|
||||||
|
private static func isRepetitiveSpam(_ text: String) -> Bool {
|
||||||
|
let compact = text.filter { !$0.isWhitespace }
|
||||||
|
guard compact.count >= minimumLength else { return false }
|
||||||
|
|
||||||
|
var counts: [Character: Int] = [:]
|
||||||
|
for ch in compact {
|
||||||
|
counts[ch, default: 0] += 1
|
||||||
|
}
|
||||||
|
guard counts.count <= 2 else { return false }
|
||||||
|
let maxShare = counts.values.max() ?? 0
|
||||||
|
return Double(maxShare) / Double(compact.count) >= 0.80
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func characterHasLetterOrNumber(_ character: Character) -> Bool {
|
||||||
|
if character.isLetter || character.isNumber { return true }
|
||||||
|
// CJK ideographs / kana counted as “letter-like” content for R2.
|
||||||
|
for scalar in character.unicodeScalars {
|
||||||
|
switch scalar.value {
|
||||||
|
case 0x4E00...0x9FFF, // CJK Unified
|
||||||
|
0x3400...0x4DBF, // CJK Ext A
|
||||||
|
0x3040...0x30FF, // Hiragana / Katakana
|
||||||
|
0xAC00...0xD7AF: // Hangul
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,8 +50,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
|||||||
case startRecording
|
case startRecording
|
||||||
case stopRecording
|
case stopRecording
|
||||||
case abort
|
case abort
|
||||||
|
/// Light warm-up: ASR locale/assets only — no mic capture.
|
||||||
|
case prewarm
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wire version that includes clipboard-command fields.
|
||||||
|
public static let currentProtocolVersion = 2
|
||||||
|
|
||||||
public let protocolVersion: Int
|
public let protocolVersion: Int
|
||||||
public let sessionId: UUID
|
public let sessionId: UUID
|
||||||
public let utteranceId: UUID
|
public let utteranceId: UUID
|
||||||
@@ -60,16 +65,25 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
|||||||
public let localeId: String
|
public let localeId: String
|
||||||
public let createdAt: TimeInterval
|
public let createdAt: TimeInterval
|
||||||
public let fieldContext: FlowFieldContext?
|
public let fieldContext: FlowFieldContext?
|
||||||
|
/// Dictation (default) vs clipboard instruction mode. Absent on legacy v1 → dictation.
|
||||||
|
public let utteranceMode: FlowUtteranceMode?
|
||||||
|
/// Frozen clipboard material; present on clipboard-command `startRecording`.
|
||||||
|
public let clipboardSnapshot: String?
|
||||||
|
/// Prior successful command output for continuous rewrite rounds.
|
||||||
|
public let previousOutput: String?
|
||||||
|
|
||||||
public init(
|
public init(
|
||||||
protocolVersion: Int = 1,
|
protocolVersion: Int = FlowCommand.currentProtocolVersion,
|
||||||
sessionId: UUID,
|
sessionId: UUID,
|
||||||
utteranceId: UUID,
|
utteranceId: UUID,
|
||||||
commandSeq: Int64,
|
commandSeq: Int64,
|
||||||
action: Action,
|
action: Action,
|
||||||
localeId: String,
|
localeId: String,
|
||||||
createdAt: TimeInterval = Date().timeIntervalSince1970,
|
createdAt: TimeInterval = Date().timeIntervalSince1970,
|
||||||
fieldContext: FlowFieldContext? = nil
|
fieldContext: FlowFieldContext? = nil,
|
||||||
|
utteranceMode: FlowUtteranceMode? = nil,
|
||||||
|
clipboardSnapshot: String? = nil,
|
||||||
|
previousOutput: String? = nil
|
||||||
) {
|
) {
|
||||||
self.protocolVersion = protocolVersion
|
self.protocolVersion = protocolVersion
|
||||||
self.sessionId = sessionId
|
self.sessionId = sessionId
|
||||||
@@ -79,6 +93,13 @@ public struct FlowCommand: Codable, Equatable, Sendable {
|
|||||||
self.localeId = localeId
|
self.localeId = localeId
|
||||||
self.createdAt = createdAt
|
self.createdAt = createdAt
|
||||||
self.fieldContext = fieldContext
|
self.fieldContext = fieldContext
|
||||||
|
self.utteranceMode = utteranceMode
|
||||||
|
self.clipboardSnapshot = clipboardSnapshot
|
||||||
|
self.previousOutput = previousOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
public var resolvedUtteranceMode: FlowUtteranceMode {
|
||||||
|
utteranceMode ?? .dictation
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,9 +127,11 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
|||||||
public let revision: Int64?
|
public let revision: Int64?
|
||||||
public let fieldFingerprint: String?
|
public let fieldFingerprint: String?
|
||||||
public let createdAt: TimeInterval
|
public let createdAt: TimeInterval
|
||||||
|
/// Echo of the command mode so the extension can skip raw fallback.
|
||||||
|
public let utteranceMode: FlowUtteranceMode?
|
||||||
|
|
||||||
public init(
|
public init(
|
||||||
protocolVersion: Int = 1,
|
protocolVersion: Int = FlowCommand.currentProtocolVersion,
|
||||||
sessionId: UUID,
|
sessionId: UUID,
|
||||||
utteranceId: UUID,
|
utteranceId: UUID,
|
||||||
commandSeq: Int64,
|
commandSeq: Int64,
|
||||||
@@ -120,7 +143,8 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
|||||||
hostGeneration: String? = nil,
|
hostGeneration: String? = nil,
|
||||||
revision: Int64? = nil,
|
revision: Int64? = nil,
|
||||||
fieldFingerprint: String? = nil,
|
fieldFingerprint: String? = nil,
|
||||||
createdAt: TimeInterval = Date().timeIntervalSince1970
|
createdAt: TimeInterval = Date().timeIntervalSince1970,
|
||||||
|
utteranceMode: FlowUtteranceMode? = nil
|
||||||
) {
|
) {
|
||||||
self.protocolVersion = protocolVersion
|
self.protocolVersion = protocolVersion
|
||||||
self.sessionId = sessionId
|
self.sessionId = sessionId
|
||||||
@@ -135,6 +159,16 @@ public struct FlowResult: Codable, Equatable, Sendable {
|
|||||||
self.revision = revision
|
self.revision = revision
|
||||||
self.fieldFingerprint = fieldFingerprint
|
self.fieldFingerprint = fieldFingerprint
|
||||||
self.createdAt = createdAt
|
self.createdAt = createdAt
|
||||||
|
self.utteranceMode = utteranceMode
|
||||||
|
}
|
||||||
|
|
||||||
|
public var resolvedUtteranceMode: FlowUtteranceMode {
|
||||||
|
utteranceMode ?? .dictation
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clipboard-command deliveries must never insert raw ASR into the field.
|
||||||
|
public var allowsRawFallback: Bool {
|
||||||
|
resolvedUtteranceMode != .clipboardCommand
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -130,6 +130,10 @@ public final class KeyboardState: ObservableObject {
|
|||||||
/// `true` while a cursor-drag pad is being pressed — drives the hint
|
/// `true` while a cursor-drag pad is being pressed — drives the hint
|
||||||
/// shown above the mic.
|
/// shown above the mic.
|
||||||
@Published public var cursorDragActive: Bool = false
|
@Published public var cursorDragActive: Bool = false
|
||||||
|
/// Opportunity-read: clipboard text is eligible for long-press command mode.
|
||||||
|
@Published public var clipboardCommandEligible: Bool = false
|
||||||
|
/// Active clipboard-command task (continuous rewrite window).
|
||||||
|
@Published public var clipboardCommandSessionActive: Bool = false
|
||||||
/// Whether translate-and-polish is armed for the current engine.
|
/// Whether translate-and-polish is armed for the current engine.
|
||||||
public var isTranslationEffective: Bool {
|
public var isTranslationEffective: Bool {
|
||||||
translationEnabled
|
translationEnabled
|
||||||
@@ -205,6 +209,9 @@ public final class KeyboardState: ObservableObject {
|
|||||||
public var beginRecording: () -> Void = {}
|
public var beginRecording: () -> Void = {}
|
||||||
public var endRecording: () -> Void = {}
|
public var endRecording: () -> Void = {}
|
||||||
public var tapMic: () -> Void = {}
|
public var tapMic: () -> Void = {}
|
||||||
|
public var beginClipboardCommand: () -> Void = {}
|
||||||
|
public var endClipboardCommand: () -> Void = {}
|
||||||
|
public var refreshClipboardEligibility: () -> Void = {}
|
||||||
public var openSettings: () -> Void = {}
|
public var openSettings: () -> Void = {}
|
||||||
public var startFlowSession: () -> Void = {}
|
public var startFlowSession: () -> Void = {}
|
||||||
public var setMode: (InputMode) -> Void = { _ in }
|
public var setMode: (InputMode) -> Void = { _ in }
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
// ClipboardCommandPromptComposerTests.swift
|
||||||
|
// OSGKeyboardTests
|
||||||
|
|
||||||
|
import XCTest
|
||||||
|
@testable import OSGKeyboardShared
|
||||||
|
|
||||||
|
final class ClipboardCommandPromptComposerTests: XCTestCase {
|
||||||
|
|
||||||
|
func testUserMessageIncludesMaterialInstructionAndPrevious() {
|
||||||
|
let input = ClipboardCommandPromptComposer.Input(
|
||||||
|
snapshot: "对方说周末见面",
|
||||||
|
instruction: "委婉拒绝",
|
||||||
|
previousOutput: "这周不太方便"
|
||||||
|
)
|
||||||
|
let user = ClipboardCommandPromptComposer.userMessage(input, language: .chinese)
|
||||||
|
XCTAssertTrue(user.contains("【材料】"))
|
||||||
|
XCTAssertTrue(user.contains("对方说周末见面"))
|
||||||
|
XCTAssertTrue(user.contains("【指令】"))
|
||||||
|
XCTAssertTrue(user.contains("委婉拒绝"))
|
||||||
|
XCTAssertTrue(user.contains("【上一版结果】"))
|
||||||
|
XCTAssertTrue(user.contains("这周不太方便"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSystemPromptDoesNotEmbedR6DictationBan() {
|
||||||
|
let system = ClipboardCommandPromptComposer.compose(
|
||||||
|
.init(snapshot: "材料", instruction: "总结"),
|
||||||
|
language: .chinese
|
||||||
|
)
|
||||||
|
XCTAssertTrue(system.contains("剪贴板写作助手"))
|
||||||
|
XCTAssertFalse(system.contains("不是向你提出的问题或命令"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testEligibilityTrackerKeepsSameChangeCountWindow() {
|
||||||
|
let first = ClipboardCommandEligibilityTracker.refresh(
|
||||||
|
changeCount: 3,
|
||||||
|
rawText: "周末有空一起吃个饭吗?我想聊下项目进度。",
|
||||||
|
previous: nil,
|
||||||
|
now: 1_000
|
||||||
|
)
|
||||||
|
XCTAssertNotNil(first)
|
||||||
|
|
||||||
|
let same = ClipboardCommandEligibilityTracker.refresh(
|
||||||
|
changeCount: 3,
|
||||||
|
rawText: "周末有空一起吃个饭吗?我想聊下项目进度。",
|
||||||
|
previous: first,
|
||||||
|
now: 1_010
|
||||||
|
)
|
||||||
|
XCTAssertEqual(same?.startedAt, first?.startedAt)
|
||||||
|
|
||||||
|
let expired = ClipboardCommandEligibilityTracker.refresh(
|
||||||
|
changeCount: 3,
|
||||||
|
rawText: "周末有空一起吃个饭吗?我想聊下项目进度。",
|
||||||
|
previous: first,
|
||||||
|
now: 1_040
|
||||||
|
)
|
||||||
|
XCTAssertNil(expired)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
// ClipboardMaterialFilterTests.swift
|
||||||
|
// OSGKeyboardTests
|
||||||
|
|
||||||
|
import XCTest
|
||||||
|
@testable import OSGKeyboardShared
|
||||||
|
|
||||||
|
final class ClipboardMaterialFilterTests: XCTestCase {
|
||||||
|
|
||||||
|
func testRejectsEmpty() {
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate(" "), .rejected(.empty))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsPhoneAndNumeric() {
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("13812345678"), .rejected(.phoneOrNumeric))
|
||||||
|
XCTAssertEqual(
|
||||||
|
ClipboardMaterialFilter.evaluate("+86 138-1234-5678"),
|
||||||
|
.rejected(.phoneOrNumeric)
|
||||||
|
)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("123-456"), .rejected(.phoneOrNumeric))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsEmojiOrSymbolOnly() {
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("😀😀😀"), .rejected(.emojiOrSymbolOnly))
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("!!!"), .rejected(.emojiOrSymbolOnly))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsVerificationCode() {
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("A8f2K1"), .rejected(.verificationCode))
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("x9Y2"), .rejected(.verificationCode))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsTooShort() {
|
||||||
|
// 周末吃饭吗 = 5 graphemes
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate("周末吃饭吗"), .rejected(.tooShort))
|
||||||
|
let fourteen = String(repeating: "啊", count: 14)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate(fourteen), .rejected(.tooShort))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsRepetitiveSpam() {
|
||||||
|
let spam = String(repeating: "啊", count: 15)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.evaluate(spam), .rejected(.repetitiveSpam))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAcceptsNaturalLanguage() {
|
||||||
|
let text = "周末有空一起吃个饭吗?我想聊下项目进度。"
|
||||||
|
switch ClipboardMaterialFilter.evaluate(text) {
|
||||||
|
case .eligible(let snapshot):
|
||||||
|
XCTAssertEqual(snapshot, text)
|
||||||
|
case .rejected(let reason):
|
||||||
|
XCTFail("expected eligible, got \(reason)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAllowsDigitsInsideNaturalSentence() {
|
||||||
|
let text = "明天 3 点见,我们在咖啡厅门口碰头再走。"
|
||||||
|
if case .rejected = ClipboardMaterialFilter.evaluate(text) {
|
||||||
|
XCTFail("sentence with digits should remain eligible")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testTruncateSnapshot() {
|
||||||
|
let long = String(repeating: "汉", count: 3_050)
|
||||||
|
let truncated = ClipboardMaterialFilter.truncateSnapshot(long)
|
||||||
|
XCTAssertEqual(truncated.count, ClipboardMaterialFilter.maxSnapshotLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testConstantsMatchPlan() {
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.minimumLength, 15)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.maxSnapshotLength, 3_000)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.longPressDuration, 0.45, accuracy: 0.001)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.minimumRecordingAfterHostConfirm, 0.70, accuracy: 0.001)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.eligibilityDuration, 30, accuracy: 0.001)
|
||||||
|
XCTAssertEqual(ClipboardMaterialFilter.sessionDuration, 30, accuracy: 0.001)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
# 剪贴板语音指令(Clipboard Voice Command)一页规划
|
||||||
|
|
||||||
|
> **文档状态**:产品与架构规划(**决策已冻结**;实现进行中)
|
||||||
|
> **适用范围**:iOS 主 App + 键盘扩展(复用现有 Flow / ASR / 润色管线)
|
||||||
|
> **分支**:`feature/clipboard-voice-command`
|
||||||
|
> **创建日期**:2026-08-07
|
||||||
|
> **修订日期**:2026-08-07(架构评审 + 状态机 + 过滤/手势 + 协议/Prompt 冻结)
|
||||||
|
> **关联能力**:Flow 会话、PiP 保活、Polish Style Packs、字段指纹插入
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 一句话定义
|
||||||
|
|
||||||
|
用户复制一段材料到剪贴板后,**长按麦克风**口述处理意图;系统以「剪贴板快照 = 材料、ASR = 指令、当前风格包 = 底色」生成结果,并直接写入当前输入框。短按始终回到普通听写。
|
||||||
|
|
||||||
|
覆盖场景:**社交回复**(委婉拒绝、温暖安慰)与 **文档处理**(精简、总结)共用一条管线。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 冻结决策
|
||||||
|
|
||||||
|
| 项 | 选择 |
|
||||||
|
|----|------|
|
||||||
|
| 入口 | **复用麦克风,长按为新增状态**:短按 = 现有听写 toggle;长按 = 剪贴板指令 |
|
||||||
|
| 长按录音 | 达标后 **立刻开录**;**松手结束本轮**并进入生成 |
|
||||||
|
| 生成中麦克风 | **F1:锁定**,等结果上屏后再长按连续改写(不排队、不取消重录) |
|
||||||
|
| 提示文案 | 有开场资格时改为:**「点击说话,长按处理剪贴板」** |
|
||||||
|
| 剪贴板读取 | **机会读**(键盘出现 / 回前台 / 触摸键盘)更新提示;长按开场再读一次并 **冻结快照**;不后台轮询 |
|
||||||
|
| 开场资格时钟 | 复制后 **30 秒内** 可进入 |
|
||||||
|
| 材料过滤 | R0–R6;过短 **有效长度 < 15**;见 §4 |
|
||||||
|
| 长按达标 | **0.45s**;松手结束;滑出再松仍送生成(见 §4.1) |
|
||||||
|
| 指令会话时钟 | 开场成功起 **30 秒**;每轮 **成功上屏** 刷新 30 秒;失败不刷新 |
|
||||||
|
| 与 Flow/PiP | **独立子状态**:指令 30s ≠ Flow 无活动超时;小会话结束不退出 Flow |
|
||||||
|
| 材料快照 | 开场瞬间冻结;会话中剪贴板变更不影响本会话 |
|
||||||
|
| 首轮写入 | 空框写入;非空追加 |
|
||||||
|
| 连续改写 | **干净 → 替换上轮产物**;**不干净 → 追加**(用户自删) |
|
||||||
|
| 替换实现 | **不依赖**系统撤销 API(扩展无法调用宿主 Undo);用自记文本 + `deleteBackward` 尽力替换 |
|
||||||
|
| 会话中短按 | **结束指令会话 → 听写** |
|
||||||
|
| 换输入框 | **结束指令会话**(拿不准是否换框时,按不干净处理:只追加不删) |
|
||||||
|
| 风格 | 共享基础润色契约;当前 Style Pack 作底色;**本轮指令优先** |
|
||||||
|
| 翻译开关 | 指令模式 MVP **忽略** |
|
||||||
|
| 预览 | 直接上屏,不强制确认 |
|
||||||
|
| 录音条 | **展示**指令 ASR 原文(仅 UI,不上屏到输入框) |
|
||||||
|
| 语音历史 | **只记成功产物**;不记指令 ASR;快照不落盘 |
|
||||||
|
| 密码 / 安全框 | **禁止**指令模式 |
|
||||||
|
| 扩展 jetsam | **指令会话结束**(内存态丢弃,可接受) |
|
||||||
|
| 失败 fallback | 指令模式 **禁止** raw ASR 插入输入框(与听写失败落原文相反) |
|
||||||
|
| 协议 | **扩展 FlowCommand(A1)**:`utteranceMode` + `clipboardSnapshot`;**start 即携带**;见 §11 |
|
||||||
|
| 快照上限 | wire 与送模型均为 **3000** 字(超出截断) |
|
||||||
|
| Prompt | **独立** `ClipboardCommandPromptComposer`;材料 / 指令 / 上一版;风格 **B1 短偏置**;见 §11 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 交互主路径
|
||||||
|
|
||||||
|
```text
|
||||||
|
复制文本
|
||||||
|
→ 机会读发现开场资格(≤30s + 通过材料过滤)
|
||||||
|
→ 提示:「点击说话,长按处理剪贴板」
|
||||||
|
→ 长按达标 → 立刻录音,再读并冻结 clipboardSnapshot,开启指令会话
|
||||||
|
→ 松手 → 停录;录音条曾展示指令 ASR
|
||||||
|
→ ASR(指令) + Snapshot(材料) + StylePack(底色) → LLM(生成中 mic 锁定)
|
||||||
|
→ 成功:空框写入 / 非空追加;记录本轮产物;刷新会话 30s;可写入语音历史(仅产物)
|
||||||
|
→ 失败:不插入;不写历史;会话保持但不续期
|
||||||
|
→(可选)出字后再长按连续改写:
|
||||||
|
干净则删上轮产物再写新版;否则追加
|
||||||
|
→ 短按 / 会话到期 / 换输入框 / 收起键盘 / jetsam → 结束会话,丢弃快照
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 开场过滤(可测规则,冻结)
|
||||||
|
|
||||||
|
命中任一条 → 保持听写提示,**不进入指令模式**。
|
||||||
|
|
||||||
|
**计数:** trim 首尾空白后用 Swift `String.count`(扩展字形簇),下称「有效长度」。
|
||||||
|
|
||||||
|
| # | 规则 | 可测定义 |
|
||||||
|
|---|------|----------|
|
||||||
|
| R0 | 空 / 非文本 | trim 后为空 |
|
||||||
|
| R1 | 整段像号码 | 去掉空白后匹配 `^[\d\-\+\(\)\s]+$` 且至少含 1 位数字(任意长度) |
|
||||||
|
| R2 | 纯 emoji/符号 | 去掉空白后 **无** 字母、汉字、数字 |
|
||||||
|
| R3 | 验证码形态 | 有效长度 4…8,整段仅 `[A-Za-z0-9]`,且 **同时含** 字母与数字 |
|
||||||
|
| R4 | 过短 | 有效长度 **< 15**(定死) |
|
||||||
|
| R5 | 单字符刷屏 | 去掉空白后长度 ≥ 15,不同字符种类 ≤ 2,且某一字符占比 ≥ 80% |
|
||||||
|
| R6 | 运行时拒绝 | 安全输入框、无 Full Access、读剪贴板失败(含系统粘贴权限拒绝) |
|
||||||
|
|
||||||
|
**不做(MVP):** 用模型判断「有无意义」;因含 URL/邮箱整段拒绝;因过长拒绝入口。
|
||||||
|
|
||||||
|
**送模型截断(不影响入口):** 快照超过 **3000** 字则截断(与 wire 上限相同),prompt 可注明已截断。
|
||||||
|
|
||||||
|
**例:**
|
||||||
|
|
||||||
|
| 文本 | 结果 |
|
||||||
|
|------|------|
|
||||||
|
| `13812345678`、`+86 138-1234-5678` | 拒(R1) |
|
||||||
|
| `😀😀😀`、`!!!` | 拒(R2) |
|
||||||
|
| `A8f2K1` | 拒(R3) |
|
||||||
|
| `周末吃饭吗`(<15) | 拒(R4) |
|
||||||
|
| `啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊` | 拒(R5) |
|
||||||
|
| `周末有空一起吃个饭吗?我想聊下项目进度。` | 过 |
|
||||||
|
|
||||||
|
首次读剪贴板触发系统粘贴权限且失败 → 无资格 + 弱提示。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4.1 长按阈值(冻结)
|
||||||
|
|
||||||
|
| 项 | 值 |
|
||||||
|
|----|-----|
|
||||||
|
| 长按达标 | **0.45s** |
|
||||||
|
| 未达 0.45s 松手 | 视为 **短按** → 听写 toggle |
|
||||||
|
| 达到 0.45s | 可选轻震 + **立刻**开指令录音 |
|
||||||
|
| 达标后滑出按钮再松手 | MVP:**仍结束本轮并送去生成**(不做滑出取消) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 写入与「干净」判定
|
||||||
|
|
||||||
|
**可替换(干净)须同时满足:** 同一指令会话、能判定仍在同一输入框、有上轮插入记录、光标仍在上轮产物末尾、上轮文本未被用户改动。
|
||||||
|
|
||||||
|
**推荐执行顺序:** LLM 成功拿到新文本 → 再删旧 → 再插新;删后插失败则尽力写回旧文本。不确定时 **只追加或不插,绝不误删用户原文**。
|
||||||
|
|
||||||
|
**说明:** 用户仍可自行使用系统撤销(摇一摇等)作为逃生口;产品自动替换不依赖该能力。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 失败态与边界(优先级)
|
||||||
|
|
||||||
|
1. **不丢用户原文**(替换不确定 → 追加或不插)
|
||||||
|
2. **不把指令 ASR 写入输入框**(ASR/LLM 失败均不插入;禁用听写式 raw fallback)
|
||||||
|
3. **不静默滥用剪贴板**(无资格不进指令;机会读 + 开场再读;不轮询)
|
||||||
|
4. **软失败可重试但不续期**(没听清 / 生成失败:不插入、会话保持、时钟不刷新)
|
||||||
|
5. **少打断**(弱提示,不弹模态)
|
||||||
|
|
||||||
|
| 场景 | 策略 |
|
||||||
|
|------|------|
|
||||||
|
| 开场资格过期 | **仅改回普通听写文案**;不必再弹「已过期」。此后长按不进指令 |
|
||||||
|
| 无有效语音 / ASR 空 | 不调用 LLM;不插入;**麦克风上方文本提示**;会话保持(不续期) |
|
||||||
|
| 指令不可解析 | 麦克风上方提示说明要怎么处理;不插入 |
|
||||||
|
| LLM 超时/拒绝/空结果 | 不插入;麦克风上方提示可重试;不刷新时钟 |
|
||||||
|
| 换输入框 / 收起键盘 / jetsam | 结束会话;已上屏保留 |
|
||||||
|
| 会话中剪贴板被覆盖 | 本会话仍用旧快照 |
|
||||||
|
| 密码框 | 禁止指令模式 |
|
||||||
|
| 生成中再长按 | 忽略(mic 锁定),等上屏后再改写 |
|
||||||
|
|
||||||
|
**隐私:** 快照仅内存;会话结束丢弃;不写历史明文;不把指令 ASR 写入历史。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 与现有架构的关系
|
||||||
|
|
||||||
|
| 可复用 | 必须新建(实现期) |
|
||||||
|
|--------|-------------------|
|
||||||
|
| Flow 命令/结果桥、PiP、ASR、插入器、字段指纹 | 长按手势层;机会读与 R0–R6 过滤 |
|
||||||
|
| `PolishingService`(含 `systemPrompt` 覆盖) | `ClipboardCommandPromptComposer`;finalize 按 `utteranceMode` 分流 |
|
||||||
|
| Flow 大会话保活 | 指令会话态(扩展);`FlowCommand` mode/snapshot/previousOutput;禁 raw 门禁 |
|
||||||
|
|
||||||
|
**两套会话(写死):**
|
||||||
|
|
||||||
|
- **Flow / PiP**:主 App 可响应录音(分钟级无活动等现有策略)
|
||||||
|
- **指令会话**:一次「处理剪贴板」任务(30s 刷新规则如上)
|
||||||
|
- 小会话结束 ≠ 退出 Flow;勿把指令 30s 接到 `FlowInactivityDuration`
|
||||||
|
|
||||||
|
**非目标(本期):** 后台轮询剪贴板、强制预览、独立回复大按钮、生成中排队/取消重录(F2/F3)、把指令模式混进现有 transcript-polish 路径、依赖宿主 Undo API。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 状态机(冻结)
|
||||||
|
|
||||||
|
状态机回答三件事:**现在在哪、什么事件会跳转、哪些操作允许**。分两层,勿混用。
|
||||||
|
|
||||||
|
### 8.1 层 A — 手势(手指)
|
||||||
|
|
||||||
|
| 状态 | 含义 |
|
||||||
|
|------|------|
|
||||||
|
| 空闲 | 未按下 |
|
||||||
|
| 按下待判定 | 已按下,尚未达到长按阈值(可能变成短按) |
|
||||||
|
| 指令按住录音 | 长按已达标,正在录指令;**松手 → 停录** |
|
||||||
|
| (听写录音) | 仍由现有 **短按 toggle** 进入/结束,不经本层长按路径 |
|
||||||
|
|
||||||
|
### 8.2 层 B — 剪贴板任务(生意)
|
||||||
|
|
||||||
|
| 状态 | 提示 / mic | 含义 |
|
||||||
|
|------|------------|------|
|
||||||
|
| **无资格** | 普通听写文案 | 材料不合格、已过期、安全框、无权限等 |
|
||||||
|
| **有资格** | 「点击说话,长按处理剪贴板」 | 机会读通过:合格材料且复制后 ≤30s |
|
||||||
|
| **指令录音中** | 录音条可展示指令 ASR | 长按已开场,快照已冻结 |
|
||||||
|
| **生成中** | mic **锁定**(F1) | 转写 + LLM;不接受新的短按/长按开录 |
|
||||||
|
| **可连续改写** | 可再长按;短按则退出 | 上一轮已结束(成功上屏或失败提示);会话 30s 未到期 |
|
||||||
|
| **已结束** | — | 清快照与上轮产物记录;再评估有资格/无资格 |
|
||||||
|
|
||||||
|
### 8.3 主转移
|
||||||
|
|
||||||
|
```text
|
||||||
|
无资格
|
||||||
|
│ 机会读:合格且 ≤30s
|
||||||
|
▼
|
||||||
|
有资格 ◄─────────────────────────────────────────┐
|
||||||
|
│ 长按达标(开场读并冻快照) │
|
||||||
|
▼ │
|
||||||
|
指令录音中 ──松手──► 生成中 │
|
||||||
|
│ │
|
||||||
|
┌───────────┼───────────┐ │
|
||||||
|
▼ ▼ ▼ │
|
||||||
|
成功上屏 失败(仅麦克风上方文本提示) │
|
||||||
|
│ │ │
|
||||||
|
└─────┬─────┘ │
|
||||||
|
▼ │
|
||||||
|
可连续改写 ──再长按───────────────────────┘
|
||||||
|
│ (回到「指令录音中」,同一快照)
|
||||||
|
│ 短按听写 / 会话30s到期 / 换输入框
|
||||||
|
│ / 收起键盘 / jetsam
|
||||||
|
▼
|
||||||
|
已结束 ──► 机会读 → 有资格 或 无资格
|
||||||
|
```
|
||||||
|
|
||||||
|
**资格过期:** 机会读发现超时 → **无资格**,提示改回普通文案即可,**不再**单独弹「剪贴板已过期」。
|
||||||
|
|
||||||
|
**生成失败:** 不插入、不写历史、不刷新会话 30s;落在 **可连续改写**(若会话未到期),麦克风上方文本提示;若墙钟已超过会话 30s → **已结束**。
|
||||||
|
|
||||||
|
**换框拿不准:** 不强制跳「已结束」时,按「不干净」只追加不删(见 §5);一旦能判定换框 → **已结束**。
|
||||||
|
|
||||||
|
### 8.4 事件 × 任务态(摘要)
|
||||||
|
|
||||||
|
| 当前任务态 | 短按 | 长按达标 | 松手 |
|
||||||
|
|------------|------|----------|------|
|
||||||
|
| 无资格 | 听写 toggle | 不进指令 | — |
|
||||||
|
| 有资格 | 听写 toggle | → 指令录音中 | — |
|
||||||
|
| 指令录音中 | — | — | → 生成中 |
|
||||||
|
| 生成中 | 忽略 | 忽略 | — |
|
||||||
|
| 可连续改写 | → 已结束,再听写 | → 指令录音中(同快照) | — |
|
||||||
|
| 已结束 | 听写 | 视重新评估后的资格 | — |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 成功标准(验收)
|
||||||
|
|
||||||
|
1. 复制合格文本后,经机会读,30s 内提示变为「点击说话,长按处理剪贴板」;过期后仅改回普通文案。
|
||||||
|
2. 长按立刻录音,松手后生成;口述「委婉拒绝 / 精简 / 总结」等,结果直接进入当前输入框。
|
||||||
|
3. 录音条可见指令 ASR;失败时输入框不被指令原文污染,麦克风上方有文本提示。
|
||||||
|
4. 出字后再长按改写:未手改则替换上轮产物;手改或不干净则追加。
|
||||||
|
5. 生成中无法再开下一轮录音(F1)。
|
||||||
|
6. 会话中短按恢复听写;换输入框结束指令会话。
|
||||||
|
7. 纯数字 / 纯 emoji / 验证码 / 有效长度<15 / 刷屏等不出现指令提示。
|
||||||
|
8. 语音历史仅出现成功产物,无指令 ASR、无剪贴板快照明文。
|
||||||
|
9. 长按 0.45s 开录;未达阈值松手走听写;滑出松手仍生成。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. 协议与 Prompt(冻结)
|
||||||
|
|
||||||
|
### 11.1 FlowCommand 扩展(A1)
|
||||||
|
|
||||||
|
在现有 `start/stop/abort` 上增加(`protocolVersion` bump,保持旧字段可解码):
|
||||||
|
|
||||||
|
| 字段 | 时机 | 含义 |
|
||||||
|
|------|------|------|
|
||||||
|
| `utteranceMode` | start(必填语义) | `dictation`(默认/缺省)\| `clipboardCommand` |
|
||||||
|
| `clipboardSnapshot` | **start 即带** | 开场冻结的材料;≤3000 字;仅 `clipboardCommand` |
|
||||||
|
| `fieldContext` | 可仍在 stop 时带 | 插入指纹 / 空框判定等(沿用现状) |
|
||||||
|
|
||||||
|
```text
|
||||||
|
长按达标 → 冻快照(≤3000)
|
||||||
|
→ startRecording { mode: clipboardCommand, clipboardSnapshot }
|
||||||
|
→ 松手 → stopRecording { fieldContext? }
|
||||||
|
→ host finalize:mode=clipboardCommand → 指令 Composer
|
||||||
|
失败 → error;rawText 可留作调试,ext 不得 insert raw
|
||||||
|
```
|
||||||
|
|
||||||
|
连续改写 = 同一指令会话内多个 utterance;每轮 start **重复携带同一份**内存快照。
|
||||||
|
「上一版产物」不进 FlowCommand,由扩展在成功上屏后记住,经 host 侧 prompt 用户区传入(见 11.2)——若上一版仅 ext 知道,则需在 stop/start 增加可选 `previousOutput`,或 finalize 前写入 App Group。
|
||||||
|
|
||||||
|
**推荐补字段(冻结):** start 或 stop 可选 `previousOutput: String?`(连续改写时由 ext 带上轮成功产物;首轮 nil)。与快照同属 utterance 自描述,避免旁路。
|
||||||
|
|
||||||
|
**FlowResult:** 回传 `utteranceMode`(或等效标记)。ext 对 `clipboardCommand`:**跳过**一切 raw ASR 上屏路径(含 `deliverRawFallbackIfAvailable`)。
|
||||||
|
|
||||||
|
### 11.2 Prompt 分流
|
||||||
|
|
||||||
|
| | 听写 | 剪贴板指令 |
|
||||||
|
|--|------|------------|
|
||||||
|
| Composer | `PolishPromptComposer`(含 R6) | **新建** `ClipboardCommandPromptComposer` |
|
||||||
|
| 用户文本角色 | ASR = 待整理正文 | ASR = **指令**;快照 = **材料** |
|
||||||
|
| 翻译开关 | 可生效 | **忽略**,强制非 translate |
|
||||||
|
| 失败 | 可 fallback raw ASR | **禁止** raw 上屏 |
|
||||||
|
| Style Pack | 完整听写人格装配 | **B1**:短语气偏置;指令优先 |
|
||||||
|
|
||||||
|
**System 契约(精神):** 按材料执行指令;只输出可直接发送的最终文本;无解释套话;不编造材料中没有的关键事实(除非指令要求语气发挥);Style Pack 为弱底色;口述指令覆盖风格。
|
||||||
|
|
||||||
|
**User 结构:**
|
||||||
|
|
||||||
|
```text
|
||||||
|
【材料】
|
||||||
|
{clipboardSnapshot}
|
||||||
|
|
||||||
|
【指令】
|
||||||
|
{asrInstruction}
|
||||||
|
|
||||||
|
【上一版结果】 ← 仅连续改写且有 previousOutput
|
||||||
|
{previousOutput}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 11.3 职责切分
|
||||||
|
|
||||||
|
| 职责 | 归属 |
|
||||||
|
|------|------|
|
||||||
|
| 机会读、R0–R6、资格 30s、hint、长按 0.45s | 扩展 |
|
||||||
|
| 指令会话 30s、快照内存、上轮产物、干净替换 | 扩展 |
|
||||||
|
| ASR、指令 LLM、结果 mode 标记 | 主 App |
|
||||||
|
| 坚持不插 raw、插入/替换执行 | 扩展 |
|
||||||
|
| Style Pack 短偏置读取 | 主 App(现有 store) |
|
||||||
|
|
||||||
|
主 App 对指令 utterance **尽量无会话态**(按包执行);指令会话态在扩展 → 与 jetsam=会话结束一致。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. 下一步(可进入实现)
|
||||||
|
|
||||||
|
1. 实现 Shared:过滤纯函数 + 单测;`FlowCommand`/`FlowResult` 字段与兼容解码。
|
||||||
|
2. 实现 `ClipboardCommandPromptComposer` + finalize 分流 + 禁 raw。
|
||||||
|
3. 扩展:机会读 hint、长按 0.45s、指令会话态、插入/替换。
|
||||||
|
4. 联调验收对照 §9。
|
||||||
+1
-1
@@ -52,7 +52,7 @@ settings:
|
|||||||
STRING_CATALOG_GENERATE_SYMBOLS: YES
|
STRING_CATALOG_GENERATE_SYMBOLS: YES
|
||||||
CLANG_CXX_LANGUAGE_STANDARD: c++17
|
CLANG_CXX_LANGUAGE_STANDARD: c++17
|
||||||
MARKETING_VERSION: "1.6.5"
|
MARKETING_VERSION: "1.6.5"
|
||||||
CURRENT_PROJECT_VERSION: "52"
|
CURRENT_PROJECT_VERSION: "53"
|
||||||
# 签名配置来自 Signing.local.xcconfig(gitignored,不会被覆盖)
|
# 签名配置来自 Signing.local.xcconfig(gitignored,不会被覆盖)
|
||||||
|
|
||||||
# 项目级签名 xcconfig,适用于所有 target
|
# 项目级签名 xcconfig,适用于所有 target
|
||||||
|
|||||||
Reference in New Issue
Block a user