feat(keyboard): show dictation progress as a fill on the microphone capsule

2026-10-20 20:05:21 +02:00
parent be56d2210c
commit 1ddbd28371
7 changed files with 296 additions and 1 deletions
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]

 ### Changed
+- **Dictation progress on the microphone**: while a dictation is being recognized and polished, the microphone capsule fills from left to right in the brand colour instead of showing a spinner. The fill steps forward when recognition finishes, slows as it nears the end, and completes only when the text is inserted; the polish estimate adapts to how long polishing usually takes on this device. / **麦克风上的听写进度**：听写识别与润色期间，麦克风胶囊改为从左到右填充品牌色，取代原来的转圈。识别完成时进度向前跳一段，越接近结束走得越慢，只有文字真正插入时才填满；润色耗时的预估会按这台设备的实际情况自动校准。
 - **Messaging shift to AI voice keyboard / voice double / agentic skills**: README and the product page now lead with “Say it. It’s typed. Speaks like you. Acts for you.” — the personalization story (learned style, three-tone replies) moves ahead of typing, clipboard skills are framed as agentic actions, and typing becomes the fallback capability. / **宣传定位转向 AI 语音键盘 / 嘴替 / Agentic 技能**：README 与产品页改用「开口即文字。会说你的话，会办你的事。」定位；个人化卖点（专属风格、三语气回复）提前，剪贴板技能表述升级为 Agentic 行动，打字退为兜底能力。
 - **Product page redesign**: rebuild the GitHub Pages home as a dark product stage with the new Icon Composer app icon, the three-device hero visual, and real on-device videos and UI cards replacing hand-drawn mockups; add 2.1 selling points (personal style, three-tone replies, monthly usage calendar, EventKit skills export, guided trials and credit rewards), a TestFlight link, and self-hosted inline SVG icons; bump page references to 2.1.0. / **产品页重设计**：GitHub Pages 主页改为深色产品舞台，启用 Icon Composer 新图标与三端同框主视觉，以实机视频与真实界面卡片取代手绘示意；补齐 2.1 卖点（专属风格、三语气个性回复、月度用量日历、EventKit 技能写入、引导体验与积分奖励），新增 TestFlight 入口，图标改为自托管内联 SVG；页面版本号更新至 2.1.0。
 - **Host-only analytics recording**: the keyboard extension no longer opens the analytics or keyboard-usage SQLite queues. The host app is the only process that records and uploads events, so hiding the keyboard cannot hold WAL locks and get killed as `0xdead10cc`. Keyboard-activated and per-keystroke usage events are no longer collected. / **仅主 App 记埋点**：键盘扩展不再打开统计与键盘用量 SQLite 队列；开库、记账与上传只在主 App 进行，收起键盘时不会因 WAL 锁被系统以 `0xdead10cc` 终止。不再采集「键盘激活」和逐次按键用量。
@@ -67,6 +67,7 @@ final class KeyboardFlowCoordinator {
    private var currentUtteranceId: UUID?
    private var currentStartDeadlineAt: TimeInterval?
    private var currentUtteranceRequest: FlowUtteranceRequest?
+    private let polishLatencyCalibration = DictationPolishLatencyCalibration()
    private var audioPrimeID: UUID?
    private var audioPrimeCancelTask: Task<Void, Never>?
    private var editHostConfirmed = false
@@ -471,6 +472,7 @@ final class KeyboardFlowCoordinator {
            if state.lastTranscript.isEmpty {
                state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
            }
+            beginDictationProgressIfNeeded()
            startFlowResultWatchdog()
            traceState("adoptHostBusy.processing", extra: "session=\(sessionId)")
        }
@@ -801,6 +803,7 @@ final class KeyboardFlowCoordinator {
            finishLocalCancel()
            return
        }
+        state.dictationProgress = nil
        let inserted = registerCancelledUtterance(utteranceID, kind: kind)
        if !inserted, isAwaitingFlowResult {
            state.phase = .processing
@@ -1101,6 +1104,7 @@ final class KeyboardFlowCoordinator {
            onAIRecognitionStarted(currentUtteranceId)
        } else if currentUtteranceRequest?.isEdit != true {
            state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
+            beginDictationProgressIfNeeded()
        }
        startFlowResultWatchdog()
        recomputeMicVoiceAvailability()
@@ -2152,6 +2156,8 @@ final class KeyboardFlowCoordinator {
                state.lastTranscript = partial
                if result.resolvedUtteranceMode == .aiQuestion {
                    onAITranscript(partial, result.utteranceId, result.status)
+                } else if result.status == .rawReady {
+                    markDictationPolishing(characterCount: partial.count)
                }
            }
        default:
@@ -2159,6 +2165,43 @@ final class KeyboardFlowCoordinator {
        }
    }

+    // MARK: - Dictation progress
+
+    /// Starts the microphone progress fill for a dictation wait. Keeps an
+    /// existing timeline for the same utterance so re-adoption does not rewind.
+    private func beginDictationProgressIfNeeded() {
+        guard let utteranceID = currentUtteranceId,
+              currentUtteranceRequest?.isAIQuestion != true,
+              currentUtteranceRequest?.isEdit != true,
+              state.dictationProgress?.utteranceID != utteranceID else { return }
+        state.dictationProgress = DictationProgressTimeline(
+            utteranceID: utteranceID,
+            transcribingStartedAt: Date().timeIntervalSince1970
+        )
+    }
+
+    private func markDictationPolishing(characterCount: Int) {
+        guard var timeline = state.dictationProgress,
+              timeline.polishingStartedAt == nil else { return }
+        timeline.markPolishing(
+            at: Date().timeIntervalSince1970,
+            characterCount: characterCount,
+            expectedSeconds: polishLatencyCalibration.expectedSeconds(
+                forCharacterCount: characterCount
+            )
+        )
+        state.dictationProgress = timeline
+    }
+
+    private func recordDictationPolishLatency() {
+        guard let timeline = state.dictationProgress,
+              let polishStart = timeline.polishingStartedAt else { return }
+        polishLatencyCalibration.record(
+            observedSeconds: Date().timeIntervalSince1970 - polishStart,
+            characterCount: timeline.polishCharacterCount
+        )
+    }
+
    private func startFlowResultWatchdog(
        purpose: ResultWatchdogPurpose = .delivery
    ) {
@@ -2212,6 +2255,7 @@ final class KeyboardFlowCoordinator {
                        self.completeAIResult(result)
                        return
                    }
+                    self.recordDictationPolishLatency()
                    self.textInserter.handleFlowTranscript(
                        TranscriptionDelivery(
                            text: text,
@@ -1048,6 +1048,9 @@ struct AIKeyboardView: View {
                    .regular.tint(microphoneTint).interactive(),
                    in: Capsule()
                )
+            if let timeline = activeDictationProgress {
+                dictationProgressFill(timeline)
+            }
            microphoneContent
        }
        .frame(width: primaryButtonWidth, height: primaryButtonHeight)
@@ -1096,11 +1099,41 @@ struct AIKeyboardView: View {
        }
    }

+    /// Dictation wait rendered as a fill; AI phases keep their own spinner.
+    private var activeDictationProgress: DictationProgressTimeline? {
+        guard state.phase == .processing, !state.aiSession.isBusy else { return nil }
+        return state.dictationProgress
+    }
+
+    /// Leading-edge fill in the brand colour. The real result swaps the
+    /// capsule to the solid accent success state, which reads as reaching 100 %.
+    private func dictationProgressFill(_ timeline: DictationProgressTimeline) -> some View {
+        TimelineView(.animation) { context in
+            let fraction = DictationProgressEstimator.fraction(
+                for: timeline,
+                at: context.date.timeIntervalSince1970
+            )
+            GeometryReader { proxy in
+                Rectangle()
+                    .fill(palette.accent)
+                    .frame(width: proxy.size.width * fraction)
+            }
+        }
+        .clipShape(Capsule())
+        .allowsHitTesting(false)
+        .accessibilityHidden(true)
+    }
+
    @ViewBuilder
    private var ordinaryMicrophoneContent: some View {
        switch state.phase {
        case .recording:
            waveform(color: OSGColor.recordingWaveform)
+        case .processing where activeDictationProgress != nil:
+            Image(systemName: "ellipsis")
+                .font(.system(size: 25, weight: .bold))
+                .foregroundStyle(OSGColor.fixedLightContent)
+                .symbolEffect(.variableColor.iterative, options: .repeating)
        case .requestingPermissions, .processing:
            ProgressView().tint(OSGColor.fixedLightContent)
        case .error:
@@ -1146,6 +1179,9 @@ struct AIKeyboardView: View {
        switch state.phase {
        case .recording:
            return palette.recordRed
+        case .processing where activeDictationProgress != nil:
+            // Unfilled track of the progress fill.
+            return palette.accent.opacity(0.35)
        case .requestingPermissions, .processing:
            return palette.recordRed.opacity(0.72)
        case .error, .denied:
@@ -0,0 +1,90 @@
+// DictationProgressTests.swift
+// OSGKeyboard · Keyboard Extension Tests
+//
+// Staged microphone progress fill: monotonic, never completes on its own,
+// steps forward at the ASR → polish milestone, and learns polish latency.
+
+@testable import OSGKeyboardShared
+import XCTest
+
+final class DictationProgressTests: XCTestCase {
+    private let start: TimeInterval = 1_000
+
+    private func timeline(polishingAfter seconds: TimeInterval? = nil,
+                          expected: TimeInterval = 3) -> DictationProgressTimeline {
+        var timeline = DictationProgressTimeline(utteranceID: UUID(), transcribingStartedAt: start)
+        if let seconds {
+            timeline.markPolishing(at: start + seconds, characterCount: 40, expectedSeconds: expected)
+        }
+        return timeline
+    }
+
+    func testTranscribingStaysBelowCeiling() {
+        let ceiling = DictationProgressEstimator.transcribingCeiling
+        XCTAssertLessThan(DictationProgressEstimator.fraction(for: timeline(), at: start + 3), ceiling)
+        // Long waits converge onto the ceiling (exp underflows to exactly it), never past it.
+        XCTAssertLessThanOrEqual(DictationProgressEstimator.fraction(for: timeline(), at: start + 60), ceiling)
+        XCTAssertEqual(DictationProgressEstimator.fraction(for: timeline(), at: start), 0)
+    }
+
+    func testFractionIsMonotonicAcrossMilestone() {
+        let subject = timeline(polishingAfter: 0.8)
+        var previous = -1.0
+        for step in 0...600 {
+            let value = DictationProgressEstimator.fraction(for: subject, at: start + Double(step) * 0.05)
+            XCTAssertGreaterThanOrEqual(value, previous)
+            previous = value
+        }
+    }
+
+    func testPolishingStepsPastFloorAndNeverCompletes() {
+        let subject = timeline(polishingAfter: 0.8)
+        let afterBlend = DictationProgressEstimator.fraction(for: subject, at: start + 0.8 + 0.3)
+        XCTAssertGreaterThanOrEqual(afterBlend, DictationProgressEstimator.polishingFloor)
+        let muchLater = DictationProgressEstimator.fraction(for: subject, at: start + 600)
+        XCTAssertLessThan(muchLater, 1)
+        XCTAssertLessThanOrEqual(muchLater, DictationProgressEstimator.polishingCeiling)
+    }
+
+    func testRepeatedRawReadyKeepsFirstMilestone() {
+        var subject = timeline(polishingAfter: 1)
+        subject.markPolishing(at: start + 5, characterCount: 999, expectedSeconds: 30)
+        XCTAssertEqual(subject.polishingStartedAt, start + 1)
+        XCTAssertEqual(subject.polishCharacterCount, 40)
+    }
+
+    func testCalibrationMovesTowardObservedLatencyWithinBounds() throws {
+        let suite = "DictationProgressTests.\(UUID().uuidString)"
+        let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
+        defer { defaults.removePersistentDomain(forName: suite) }
+        let calibration = DictationPolishLatencyCalibration(defaults: defaults)
+
+        let initial = calibration.expectedSeconds(forCharacterCount: 100)
+        calibration.record(observedSeconds: 8, characterCount: 100)
+        XCTAssertGreaterThan(calibration.expectedSeconds(forCharacterCount: 100), initial)
+
+        for _ in 0..<50 { calibration.record(observedSeconds: 0.01, characterCount: 0) }
+        XCTAssertEqual(
+            calibration.overheadSeconds,
+            DictationPolishLatencyCalibration.overheadRange.lowerBound,
+            accuracy: 0.001
+        )
+        calibration.record(observedSeconds: -1, characterCount: 0)
+        XCTAssertEqual(
+            calibration.overheadSeconds,
+            DictationPolishLatencyCalibration.overheadRange.lowerBound,
+            accuracy: 0.001
+        )
+    }
+
+    @MainActor
+    func testTimelineClearsWhenProcessingEnds() {
+        let state = KeyboardState()
+        state.phase = .processing
+        state.dictationProgress = timeline()
+        state.phase = .processing
+        XCTAssertNotNil(state.dictationProgress)
+        state.phase = .idle
+        XCTAssertNil(state.dictationProgress)
+    }
+}
@@ -0,0 +1,113 @@
+// DictationProgress.swift
+// OSGKeyboard · Shared
+//
+// Staged, time-estimated progress for the dictation wait after the user stops
+// recording. The host only reports two milestones to the keyboard (`rawReady`
+// when ASR finishes, `final` when polish finishes) and polish is a single
+// non-streaming request, so the fill between milestones is an estimate: it
+// eases toward a ceiling it never reaches, and only the real `final` result
+// completes the bar.
+
+import Foundation
+
+/// Timestamps of one dictation wait, owned by `KeyboardState`.
+public struct DictationProgressTimeline: Equatable, Sendable {
+    public let utteranceID: UUID
+    public let transcribingStartedAt: TimeInterval
+    public private(set) var polishingStartedAt: TimeInterval?
+    /// Raw transcript length when polish began; scales the expected duration.
+    public private(set) var polishCharacterCount: Int = 0
+    public private(set) var expectedPolishSeconds: TimeInterval = 0
+
+    public init(utteranceID: UUID, transcribingStartedAt: TimeInterval) {
+        self.utteranceID = utteranceID
+        self.transcribingStartedAt = transcribingStartedAt
+    }
+
+    /// Records the ASR → polish milestone once; later `rawReady` repeats are ignored.
+    public mutating func markPolishing(
+        at time: TimeInterval,
+        characterCount: Int,
+        expectedSeconds: TimeInterval
+    ) {
+        guard polishingStartedAt == nil else { return }
+        polishingStartedAt = time
+        polishCharacterCount = characterCount
+        expectedPolishSeconds = expectedSeconds
+    }
+}
+
+public enum DictationProgressEstimator {
+    /// Asymptote of the ASR stage; the bar never passes it before `rawReady`.
+    static let transcribingCeiling = 0.25
+    /// Time constant of the ASR stage curve (local drain is usually < 1 s).
+    static let transcribingTimeConstant: TimeInterval = 1.0
+    /// Where the polish stage starts, so `rawReady` reads as a visible step.
+    static let polishingFloor = 0.35
+    /// Asymptote of the polish stage; only the real result fills to 100 %.
+    static let polishingCeiling = 0.95
+    /// Duration of the ease from the ASR curve into the polish curve.
+    static let stageBlendSeconds: TimeInterval = 0.25
+
+    /// Fill fraction in `0...polishingCeiling` for `timeline` at `now`.
+    public static func fraction(
+        for timeline: DictationProgressTimeline,
+        at now: TimeInterval
+    ) -> Double {
+        guard let polishStart = timeline.polishingStartedAt else {
+            return transcribingFraction(elapsed: now - timeline.transcribingStartedAt)
+        }
+        let atMilestone = transcribingFraction(
+            elapsed: polishStart - timeline.transcribingStartedAt
+        )
+        let elapsed = max(0, now - polishStart)
+        // At `elapsed == expected` the polish curve covers ~86 % of its range.
+        let timeConstant = max(timeline.expectedPolishSeconds, 0.5) / 2
+        let polishing = polishingFloor
+            + (polishingCeiling - polishingFloor) * (1 - exp(-elapsed / timeConstant))
+        let blend = min(elapsed / stageBlendSeconds, 1)
+        return atMilestone + (polishing - atMilestone) * blend
+    }
+
+    static func transcribingFraction(elapsed: TimeInterval) -> Double {
+        transcribingCeiling * (1 - exp(-max(0, elapsed) / transcribingTimeConstant))
+    }
+}
+
+/// Learns this device's polish latency so the estimate tracks the user's
+/// provider and network. Stored in the extension's own defaults; it is a UI
+/// hint, not a setting, so it is neither synced nor shared with the host.
+public struct DictationPolishLatencyCalibration {
+    static let overheadKey = "keyboard.dictationProgress.polishOverheadSeconds"
+    static let defaultOverheadSeconds: TimeInterval = 1.5
+    static let secondsPerCharacter: TimeInterval = 0.012
+    static let overheadRange: ClosedRange<TimeInterval> = 0.5...15
+    /// Weight of the newest observation in the moving average.
+    static let smoothing = 0.3
+
+    private let defaults: UserDefaults
+
+    public init(defaults: UserDefaults = .standard) {
+        self.defaults = defaults
+    }
+
+    public func expectedSeconds(forCharacterCount count: Int) -> TimeInterval {
+        overheadSeconds + Double(max(0, count)) * Self.secondsPerCharacter
+    }
+
+    public func record(observedSeconds: TimeInterval, characterCount: Int) {
+        guard observedSeconds.isFinite, observedSeconds > 0 else { return }
+        let observedOverhead = observedSeconds
+            - Double(max(0, characterCount)) * Self.secondsPerCharacter
+        let blended = overheadSeconds * (1 - Self.smoothing) + observedOverhead * Self.smoothing
+        defaults.set(
+            min(max(blended, Self.overheadRange.lowerBound), Self.overheadRange.upperBound),
+            forKey: Self.overheadKey
+        )
+    }
+
+    var overheadSeconds: TimeInterval {
+        let stored = defaults.double(forKey: Self.overheadKey)
+        return stored > 0 ? stored : Self.defaultOverheadSeconds
+    }
+}
@@ -89,7 +89,17 @@ public final class KeyboardState: ObservableObject {
        public var id: String { rawValue }
    }

-    @Published public var phase: Phase = .idle
+    @Published public var phase: Phase = .idle {
+        didSet {
+            // A timeline only describes the current processing wait.
+            if phase != .processing, dictationProgress != nil {
+                dictationProgress = nil
+            }
+        }
+    }
+    /// Dictation wait shown as the microphone capsule's progress fill.
+    /// Nil outside dictation processing (AI, edit, and cancellation waits).
+    @Published public var dictationProgress: DictationProgressTimeline?
    /// Active chrome. Forced to `.voice` while recording / processing.
    @Published public var surface: Surface = .voice
    /// Surface the user was on when auto mode switched them to `.voice` to show
@@ -174,6 +174,7 @@
        "OSGKeyboardExtTests/CandidatePanelExpandTests",
        "OSGKeyboardExtTests/ClipboardSuggestionLifecycleTests",
        "OSGKeyboardExtTests/KeyboardStateTests",
+        "OSGKeyboardExtTests/DictationProgressTests",
        "OSGKeyboardExtTests/KeyboardUsageTypingTests",
        "OSGKeyboardExtTests/KeyboardSurfaceStateTests",
        "OSGKeyboardExtTests/KeyHitTestingTests",