rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(ai): expand AI flow and clipboard semantic coverage

Browse Source 
- Cover reply-variant decision paths, scene-detector abstention, and
  the merged reply center stance catalog.
- Lock in clipboard semantic analyzer thresholds, complaint-only
  suppression, and existing-skill routing.
- Add durable session, refresh, and managed-gateway reliability tests
  for the Apple account path.
- Verify personal-style prompt derivation, low-confidence ASR
  tendencies, and Polish style learning against real corpus evidence.
- Exercise the assistant keyboard and Polish styles UI flows in
  end-to-end UI tests.
This commit is contained in:
Rocky
2026-08-29 11:51:25 +08:00
parent 3c10d73d7f
commit b275b6b0d9
15 changed files with 1348 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OSGKeyboardTests/PolishStyleLearningServiceTests.swift
+360 -12
View File
@@ -215,6 +215,8 @@ final class PolishStyleLearningServiceTests: XCTestCase {
XCTAssertFalse(extractor.text.contains("这个 Prompt 后来已经被编辑"))
XCTAssertTrue(extractor.text.contains(String(source.prefix(100))))
XCTAssertTrue(extractor.text.contains(#""userEdited":true"#))
XCTAssertTrue(extractor.text.contains(#""residualBaseline""#))
XCTAssertTrue(extractor.text.contains(#""id":"builtin.chat""#))
XCTAssertTrue(extractor.text.contains("currentStyleContamination"))
XCTAssertTrue(extractor.text.contains("historicalStyleContamination"))
XCTAssertTrue(extractor.text.contains(#""asr":"#))
@@ -224,19 +226,33 @@ final class PolishStyleLearningServiceTests: XCTestCase {
XCTAssertTrue(extractor.prompt.contains("Evidence Extractor"))
XCTAssertTrue(extractor.prompt.contains("finalEdit >"))
XCTAssertTrue(extractor.prompt.contains("NOT the"))
XCTAssertTrue(extractor.prompt.contains("Deduplicate"))
XCTAssertTrue(extractor.prompt.contains("information order"))
XCTAssertTrue(extractor.prompt.contains("epistemic stance"))
XCTAssertTrue(extractor.prompt.contains("userEdited=false"))
XCTAssertTrue(extractor.prompt.contains("retention or migration"))
XCTAssertTrue(extractor.prompt.contains("must not erase"))
XCTAssertTrue(extractor.prompt.contains("asrObservedBefore"))
XCTAssertTrue(synthesizer.prompt.contains("Style Synthesizer"))
XCTAssertTrue(synthesizer.prompt.contains("ASR preserve mode"))
XCTAssertTrue(synthesizer.prompt.contains("AI reply active-transfer mode"))
XCTAssertTrue(synthesizer.prompt.contains("Legal Emoji"))
XCTAssertTrue(
synthesizer.prompt.contains("actively turn every supported candidate trait")
)
XCTAssertTrue(synthesizer.prompt.contains("Never invent migration"))
XCTAssertTrue(synthesizer.prompt.contains("Never replace non-empty candidate traits"))
XCTAssertTrue(synthesizer.text.contains(#""evidence":"#))
XCTAssertTrue(synthesizer.text.contains(#""learningMetadata":"#))
XCTAssertFalse(synthesizer.text.contains(replyMarker))
XCTAssertFalse(synthesizer.text.contains(selectedCandidateMarker))
XCTAssertFalse(synthesizer.text.contains(String(source.prefix(100))))
XCTAssertTrue(client.requests.allSatisfy { $0.timeout == 45 })
XCTAssertTrue(client.requests.allSatisfy { $0.options?.maxTokens == 4_096 })
let metadata = try XCTUnwrap(generated.learningMetadata)
XCTAssertEqual(metadata.schemaVersion, 2)
XCTAssertEqual(metadata.schemaVersion, 3)
XCTAssertEqual(metadata.evidenceStatus, "sufficient")
XCTAssertEqual(metadata.confidence, 0.86)
XCTAssertEqual(metadata.asrExampleCount, 1)
@@ -320,6 +336,29 @@ final class PolishStyleLearningServiceTests: XCTestCase {
}
}
func testExplicitTestBuildThresholdBypassAllowsEmptyCorpus() async throws {
let client = StyleLearningCapturingClient(
responses: [
Self.insufficientEvidenceResponse,
Self.emptyCorpusGeneratedStyleResponse
]
)
let service = PolishStyleLearningService(store: store, client: client)
let generated = try await service.generateStyle(
from: PolishStyleLearningCorpus(
examples: [],
effectiveCharacterCount: 0
),
outputLanguage: .chinese,
minimumEffectiveCharacterCount: 0
)
XCTAssertEqual(client.requests.count, 2)
XCTAssertEqual(generated.learningMetadata?.asrEffectiveCharacterCount, 0)
XCTAssertEqual(generated.learningMetadata?.evidenceStatus, "insufficient")
}
func testGeneratedStyleRejectsMissingRequiredSections() {
let raw = #"{"name":"Invalid","prompt":"Only one sentence.","allowsAddedEmoji":false}"#
@@ -346,7 +385,7 @@ final class PolishStyleLearningServiceTests: XCTestCase {
}
}
func testInsufficientEvidenceCannotCreateInventedTraits() async throws {
func testInsufficientEvidenceStillRunsSynthesizerWithoutHardcodedFallback() async throws {
let source = String(repeating: "保真语料", count: 625)
let corpus = PolishStyleLearningCorpus(
examples: [
@@ -359,11 +398,10 @@ final class PolishStyleLearningServiceTests: XCTestCase {
],
effectiveCharacterCount: 2_500
)
let inventedResponse = ##"{"name":"Invented","prompt":"Invented playful slang and secrets","allowsAddedEmoji":true}"##
let client = StyleLearningCapturingClient(
responses: [
Self.insufficientEvidenceResponse,
inventedResponse
Self.lowConfidenceInsufficientEvidenceResponse,
Self.insufficientGeneratedStyleResponse
]
)
let service = PolishStyleLearningService(store: store, client: client)
@@ -374,13 +412,34 @@ final class PolishStyleLearningServiceTests: XCTestCase {
)
XCTAssertEqual(client.requests.count, 2)
XCTAssertEqual(generated.name, "直接短句风格")
XCTAssertNotEqual(generated.name, "保守保真风格")
XCTAssertEqual(generated.learningMetadata?.evidenceStatus, "insufficient")
XCTAssertEqual(generated.learningMetadata?.confidence, 0.2)
XCTAssertFalse(generated.prompt.contains("Invented"))
XCTAssertFalse(generated.prompt.contains("slang"))
XCTAssertFalse(generated.allowsAddedEmoji)
XCTAssertTrue(generated.prompt.contains("短句"))
XCTAssertTrue(generated.prompt.contains("ASR preserve mode"))
XCTAssertTrue(generated.prompt.contains("AI reply active-transfer mode"))
XCTAssertTrue(client.requests[1].text.contains(#""status":"insufficient""#))
XCTAssertFalse(generated.allowsAddedEmoji)
}
func testNonemptyASRRepairsEmptyInsufficientEvidenceIntoCandidateTraits() async throws {
let client = StyleLearningCapturingClient(
responses: [
Self.insufficientEvidenceResponse,
Self.singleObservationInsufficientEvidenceResponse,
Self.insufficientGeneratedStyleResponse
]
)
let service = PolishStyleLearningService(store: store, client: client)
let generated = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTAssertEqual(client.requests.count, 3)
XCTAssertTrue(client.requests[1].prompt.contains("REPAIR ATTEMPT"))
XCTAssertTrue(generated.prompt.contains("短句"))
}
func testEvidenceSchemaRejectsFabricationAndProtocolOverrides() {
@@ -415,6 +474,40 @@ final class PolishStyleLearningServiceTests: XCTestCase {
)
}
func testInsufficientEvidenceMayKeepSupportedLowConfidenceObservations() throws {
let evidence = try PolishStyleLearningService.parseEvidence(
Self.lowConfidenceInsufficientEvidenceResponse
)
XCTAssertEqual(evidence.status, .insufficient)
XCTAssertEqual(evidence.asr.traits.first?.confidence, 0.2)
XCTAssertEqual(evidence.asr.evidence.first?.source, .asrRepeatedBefore)
XCTAssertTrue(evidence.reply.traits.isEmpty)
}
func testSingleRawASRObservationIsAcceptedAsLowConfidenceCandidate() throws {
let evidence = try PolishStyleLearningService.parseEvidence(
Self.singleObservationInsufficientEvidenceResponse
)
XCTAssertEqual(evidence.status, .insufficient)
XCTAssertEqual(evidence.asr.traits.first?.supportCount, 1)
XCTAssertEqual(evidence.asr.evidence.first?.source, .asrObservedBefore)
}
func testSufficientEvidenceRequiresMinimumOverallConfidence() {
let lowConfidence = Self.sufficientEvidenceResponse.replacingOccurrences(
of: #""confidence":0.86"#,
with: #""confidence":0.49"#
)
XCTAssertThrowsError(
try PolishStyleLearningService.parseEvidence(lowConfidence)
) { error in
XCTAssertEqual(error as? PolishStyleLearningError, .invalidResponse)
}
}
func testEvidenceSchemaEnforcesSourcePriorityAndSupportCounts() {
let weakCrossContext = Self.sufficientEvidenceResponse.replacingOccurrences(
of: #""source":"replyCrossContextSelection","summary":"跨场景偏好轻松语气","supportCount":2"#,
@@ -438,10 +531,10 @@ final class PolishStyleLearningServiceTests: XCTestCase {
)
}
func testGeneratedStyleRejectsTrailingProtocolContent() {
func testGeneratedStyleRejectsTrailingSecondJSONObject() {
XCTAssertThrowsError(
try PolishStyleLearningService.parseGeneratedStyle(
Self.generatedStyleResponse + "\nnot-json",
Self.generatedStyleResponse + "\n{}",
outputLanguage: .chinese
)
) { error in
@@ -449,6 +542,161 @@ final class PolishStyleLearningServiceTests: XCTestCase {
}
}
func testWrappedAndFencedJSONIsRecoveredWithoutRetry() async throws {
let client = StyleLearningCapturingClient(
responses: [
"\u{FEFF}Evidence follows:\n```json\n\(Self.sufficientEvidenceResponse)\n```\nDone.",
"```json\n\(Self.generatedStyleResponse)\n```"
]
)
let service = PolishStyleLearningService(store: store, client: client)
let generated = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTAssertEqual(generated.name, "我的说话风格")
XCTAssertEqual(client.requests.count, 2)
XCTAssertFalse(client.requests.contains { $0.prompt.contains("REPAIR ATTEMPT") })
}
func testInvalidEvidenceResponseRetriesOnceWithOriginalPayload() async throws {
let client = StyleLearningCapturingClient(
responses: [
"invalid evidence",
Self.sufficientEvidenceResponse,
Self.generatedStyleResponse
]
)
let service = PolishStyleLearningService(store: store, client: client)
_ = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTAssertEqual(client.requests.count, 3)
XCTAssertEqual(client.requests[0].text, client.requests[1].text)
XCTAssertTrue(client.requests[1].prompt.contains("REPAIR ATTEMPT"))
XCTAssertFalse(client.requests[2].prompt.contains("REPAIR ATTEMPT"))
}
func testInvalidSynthesisResponseRetriesOnceWithOriginalPayload() async throws {
let client = StyleLearningCapturingClient(
responses: [
Self.sufficientEvidenceResponse,
"invalid style",
Self.generatedStyleResponse
]
)
let service = PolishStyleLearningService(store: store, client: client)
_ = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTAssertEqual(client.requests.count, 3)
XCTAssertEqual(client.requests[1].text, client.requests[2].text)
XCTAssertTrue(client.requests[2].prompt.contains("REPAIR ATTEMPT"))
}
func testInvalidEvidenceResponseRetriesAtMostOnce() async {
let client = StyleLearningCapturingClient(
responses: ["invalid first response", "invalid repair response"]
)
let service = PolishStyleLearningService(store: store, client: client)
do {
_ = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTFail("Expected invalid response after one repair attempt")
} catch let error as PolishStyleLearningError {
XCTAssertEqual(error, .invalidResponse)
} catch {
XCTFail("Unexpected error: \(error)")
}
XCTAssertEqual(client.requests.count, 2)
XCTAssertTrue(client.requests[1].prompt.contains("REPAIR ATTEMPT"))
}
func testPromptTooLongSynthesisResponseIsNotRetried() async throws {
let oversizedPrompt = """
# 角色
\(String(repeating: "长", count: 6_000))
# 风格边界
ASR preserve mode。AI reply active-transfer mode。
# 示例
输入 → 输出
"""
let responseData = try JSONSerialization.data(withJSONObject: [
"name": "Too Long",
"prompt": oversizedPrompt,
"allowsAddedEmoji": false
])
let response = try XCTUnwrap(String(data: responseData, encoding: .utf8))
let client = StyleLearningCapturingClient(
responses: [Self.sufficientEvidenceResponse, response]
)
let service = PolishStyleLearningService(store: store, client: client)
do {
_ = try await service.generateStyle(
from: Self.readyCorpus(),
outputLanguage: .chinese
)
XCTFail("Expected prompt length rejection")
} catch let error as PolishStyleLearningError {
XCTAssertEqual(error, .promptTooLong(maximum: 6_000))
} catch {
XCTFail("Unexpected error: \(error)")
}
XCTAssertEqual(client.requests.count, 2)
}
func testFailureMessagesExposeSpecificActionableReasons() {
XCTAssertEqual(
PolishStyleLearningFailureMessage.localized(
for: PolishingService.PolishError.missingAPIKey,
language: .english
),
"The current AI service has no API key. Configure it in Settings and try again."
)
XCTAssertEqual(
PolishStyleLearningFailureMessage.localized(
for: LLMError.timeout,
language: .english
),
"The AI request timed out. Please try again."
)
XCTAssertEqual(
PolishStyleLearningFailureMessage.localized(
for: LLMError.http(status: 401),
language: .english
),
"API returned HTTP 401. Try again later or contact the provider."
)
XCTAssertEqual(
PolishStyleLearningFailureMessage.localized(
for: ManagedGatewayError.insufficientCredits,
language: .english
),
"Not enough credits. Open the Account tab in the main app to add credits."
)
XCTAssertEqual(
PolishStyleLearningFailureMessage.localized(
for: ManagedGatewayError.invalidGrant,
language: .chinese
),
"托管服务授权已失效，请打开主 App 重新连接账号。"
)
}
private static let sufficientEvidenceResponse = ##"""
{
"status":"sufficient",
@@ -486,6 +734,40 @@ final class PolishStyleLearningServiceTests: XCTestCase {
}
"""##
private static let lowConfidenceInsufficientEvidenceResponse = ##"""
{
"status":"insufficient",
"confidence":0.2,
"asr":{
"traits":[
{"name":"retention:短句倾向","description":"近似去重后仍观察到短句，但支持有限","confidence":0.2,"supportCount":2}
],
"evidence":[
{"source":"asrRepeatedBefore","summary":"两个不同场景的原声 before 使用短句","supportCount":2}
],
"contradictions":[]
},
"reply":{"traits":[],"evidence":[],"contradictions":[]}
}
"""##
private static let singleObservationInsufficientEvidenceResponse = ##"""
{
"status":"insufficient",
"confidence":0.18,
"asr":{
"traits":[
{"name":"retention:短句候选","description":"一次原始 ASR 观察显示用户倾向直接短句","confidence":0.18,"supportCount":1}
],
"evidence":[
{"source":"asrObservedBefore","summary":"原始 before 使用直接短句，样本仍少","supportCount":1}
],
"contradictions":[]
},
"reply":{"traits":[],"evidence":[],"contradictions":[]}
}
"""##
private static let generatedStyleResponse = ##"""
{
"name":"我的说话风格",
@@ -493,11 +775,44 @@ final class PolishStyleLearningServiceTests: XCTestCase {
"allowsAddedEmoji":true
}
"""##
private static let emptyCorpusGeneratedStyleResponse = ##"""
{
"name":"待补充语料",
"prompt":"# 角色\n当前没有可观察的个人语料，不声明个人表达特征。\n# 风格边界\nASR preserve mode：不推断未观察到的表达习惯。\nAI reply active-transfer mode：不迁移未经观察的回复偏好。\n# 示例\n输入：没有个人语料\n输出：等待用户提供语料。",
"allowsAddedEmoji":false
}
"""##
private static let insufficientGeneratedStyleResponse = ##"""
{
"name":"直接短句风格",
"prompt":"# 角色\n优先使用语料观察到的直接短句，先说结论，不扩写背景。\n# 风格边界\nASR preserve mode：保留短句节奏与直接表达；只在原文确有多个信息点时分句。\nAI reply active-transfer mode：当前没有回复偏好证据，不迁移未经支持的语气，但保持简短直接。\n# 示例\n输入：这个事情我觉得可以之后再确认一下\n输出：这个可以，之后再确认。",
"allowsAddedEmoji":false
}
"""##
private static func readyCorpus() -> PolishStyleLearningCorpus {
let source = String(repeating: "测试语料", count: 625)
return PolishStyleLearningCorpus(
examples: [
PolishStyleLearningExample(
prePolishText: source,
finalText: source + "。",
polishStyleID: "builtin.chat",
createdAt: Date()
)
],
effectiveCharacterCount: 2_500
)
}
}
private struct StyleLearningCapturedRequest {
let text: String
let prompt: String
let timeout: TimeInterval?
let options: LLMGenerationOptions?
}
private final class StyleLearningCapturingClient: LLMClient, @unchecked Sendable {
@@ -518,9 +833,42 @@ private final class StyleLearningCapturingClient: LLMClient, @unchecked Sendable
_ text: String,
systemPrompt: String,
timeout: TimeInterval?
) async throws -> String {
try await nextResponse(
text: text,
prompt: systemPrompt,
timeout: timeout,
options: nil
)
}
func polish(
_ text: String,
systemPrompt: String,
timeout: TimeInterval?,
options: LLMGenerationOptions
) async throws -> String {
try await nextResponse(
text: text,
prompt: systemPrompt,
timeout: timeout,
options: options
)
}
private func nextResponse(
text: String,
prompt: String,
timeout: TimeInterval?,
options: LLMGenerationOptions?
) async throws -> String {
requests.append(
StyleLearningCapturedRequest(text: text, prompt: systemPrompt)
StyleLearningCapturedRequest(
text: text,
prompt: prompt,
timeout: timeout,
options: options
)
)
guard !responses.isEmpty else { return "{}" }
let index = min(responseIndex, responses.count - 1)