// LLMClientTests.swift // OSGKeyboard · Tests // // Unit tests for the OpenAI-compatible LLM client using URLProtocol stub. import XCTest @testable import OSGKeyboard @testable import OSGKeyboardShared final class LLMClientTests: XCTestCase { override func setUpWithError() throws { // The Keychain is process-global in the simulator (one simulator, // one keychain DB), so an API key written by a previous test would // leak into the next one unless we wipe it here. We intentionally // swallow errors — `errSecItemNotFound` is fine. try? Keychain.deleteAPIKey() try? Keychain.deleteLegacyAPIKey() try? Keychain.deleteAPIKey(for: "qwen") StubURLProtocolStorage.config = nil StubURLProtocolStorage.delaySeconds = 0 StubURLProtocolStorage.lastRequest = nil } override func tearDownWithError() throws { try? Keychain.deleteAPIKey() try? Keychain.deleteLegacyAPIKey() try? Keychain.deleteAPIKey(for: "qwen") StubURLProtocolStorage.config = nil StubURLProtocolStorage.delaySeconds = 0 StubURLProtocolStorage.lastRequest = nil } // MARK: - ProviderConfig persistence func testProviderConfigPersistsAcrossInstances() { let suiteName = "group.com.osgkeyboard.shared.tests" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) let config1 = ProviderConfig(defaults: defaults) config1.baseURL = "https://example.com/v1" config1.apiKey = "test-key" config1.model = "test-model" let config2 = ProviderConfig(defaults: defaults) XCTAssertEqual(config2.baseURL, "https://example.com/v1") XCTAssertEqual(config2.apiKey, "test-key") XCTAssertEqual(config2.model, "test-model") XCTAssertTrue(config2.isConfigured) } /// Local engine path: with `engineMode = "local"`, `isConfigured` /// must return `true` even when the API key is empty — onboarding /// gates the "Next" button on this property, and the local path /// never needs a key. Regression: see commit `isConfigured` fix /// that exposed this gate. func testIsConfiguredTrueForLocalEngineWithoutAPIKey() { let suiteName = "group.com.osgkeyboard.shared.tests" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) let config = ProviderConfig(defaults: defaults) // No apiKey, no baseURL, no model — cloud would fail. XCTAssertFalse(config.isConfigured) // Switch to local engine: should flip to true regardless of // the missing cloud fields. config.engineMode = "local" XCTAssertTrue(config.isConfigured) // And back to cloud: should flip to false again. config.engineMode = "cloud" XCTAssertFalse(config.isConfigured) } // MARK: - OpenAICompatibleClient func testPolishSendsCorrectRequestAndDecodesResponse() async throws { StubURLProtocolStorage.config = (200, """ { "id": "chatcmpl-1", "choices": [ { "index": 0, "message": { "role": "assistant", "content": "Hello, world!" }, "finish_reason": "stop" } ] } """.data(using: .utf8)!) defer { StubURLProtocolStorage.config = nil } let cfg = URLSessionConfiguration.ephemeral cfg.protocolClasses = [StubURLProtocol.self] let session = URLSession(configuration: cfg) let client = OpenAICompatibleClient( baseURL: "https://example.com/v1", apiKey: "sk-test", model: "test-model", session: session ) let result = try await client.polish("hi", systemPrompt: "be brief") XCTAssertEqual(result, "Hello, world!") let req = StubURLProtocolStorage.lastRequest XCTAssertEqual(req?.httpMethod, "POST") XCTAssertTrue(req?.value(forHTTPHeaderField: "Authorization")?.hasPrefix("Bearer ") == true) } func testPolishThrowsOnHTTPError() async { StubURLProtocolStorage.config = (401, "Unauthorized".data(using: .utf8)!) defer { StubURLProtocolStorage.config = nil } let cfg = URLSessionConfiguration.ephemeral cfg.protocolClasses = [StubURLProtocol.self] let session = URLSession(configuration: cfg) let client = OpenAICompatibleClient( baseURL: "https://example.com/v1", apiKey: "sk-test", model: "m", session: session ) do { _ = try await client.polish("hi", systemPrompt: "p") XCTFail("expected error") } catch let LLMError.http(status) { XCTAssertEqual(status, 401) } catch { XCTFail("wrong error: \(error)") } } func testPolishThrowsWhenAPIKeyMissing() async { let client = OpenAICompatibleClient( baseURL: "https://example.com/v1", apiKey: "", model: "m" ) do { _ = try await client.polish("hi", systemPrompt: "p") XCTFail("expected error") } catch LLMError.noAPIKey { // ok } catch { XCTFail("wrong error: \(error)") } } // MARK: - P0-③ new coverage (catch-path + App Group cross-process) func testPolishThrowsOnHTTP429RateLimited() async { StubURLProtocolStorage.config = (429, "rate limited".data(using: .utf8)!) defer { StubURLProtocolStorage.config = nil } let cfg = URLSessionConfiguration.ephemeral cfg.protocolClasses = [StubURLProtocol.self] let session = URLSession(configuration: cfg) let client = OpenAICompatibleClient( baseURL: "https://example.com/v1", apiKey: "sk-test", model: "m", session: session ) do { _ = try await client.polish("hi", systemPrompt: "p") XCTFail("expected error") } catch LLMError.rateLimited { // ok } catch { XCTFail("wrong error: \(error)") } } func testPolishThrowsOnTransportTimeout() async { // Stub the transport so it never replies in time. The client has a // 15 s `requestTimeout` on the URLRequest; we arrange for the stub // to take 5 s (well under that) and instead *cancel* the in-flight // task ourselves before the stub wins the race. That's how the // KeyboardViewController triggers cancellation in real life (mode // switch mid-polish) and is the surface `LLMError.cancelled` was // added to cover. We also assert the client *throws* — i.e. the // old "stub returns 200 synchronously and we never see the error" // failure mode is gone. StubURLProtocolStorage.config = (200, Data()) StubURLProtocolStorage.delaySeconds = 5 defer { StubURLProtocolStorage.config = nil StubURLProtocolStorage.delaySeconds = 0 } let cfg = URLSessionConfiguration.ephemeral cfg.protocolClasses = [StubURLProtocol.self] let session = URLSession(configuration: cfg) let client = OpenAICompatibleClient( baseURL: "https://example.com/v1", apiKey: "sk-test", model: "m", session: session ) let task = Task { do { _ = try await client.polish("hi", systemPrompt: "p") return false // completed — unexpected } catch { throw error } } // Give the request a head start so it's already on the wire when // we cancel. try? await Task.sleep(nanoseconds: 50_000_000) // 50 ms task.cancel() var threw = false var caughtTransportish = false do { _ = try await task.value } catch is CancellationError { threw = true } catch let err as LLMError { threw = true // We accept any of: cancelled, transport, decoding — the URL // stack is platform-quirky about how it surfaces a cancelled // request from inside URLSession's protocol handler. switch err { case .cancelled, .transport, .decoding: caughtTransportish = true default: break } } catch { threw = true } XCTAssertTrue(threw, "expected client.polish to throw on cancelled transport") XCTAssertTrue(caughtTransportish, "expected .cancelled / .transport / .decoding — got something else") } /// Cross-process App Group contract: what `ProviderConfig` writes must /// be readable through `AppGroupStore` on the same suite. func testAppGroupCrossProcessAndOffModeShortCircuit() async { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } // Writer side: ProviderConfig (main App) writes API key + mode = off. let config = ProviderConfig(defaults: defaults) config.apiKey = "sk-test-1234" config.model = "gpt-4o-mini" config.baseURL = "https://example.com/v1" config.modeId = "off" // Reader side: AppGroupStore (keyboard extension) reads from the // same suite. let store = AppGroupStore(defaults: defaults) XCTAssertEqual(store.apiKey, "sk-test-1234", "API key did not survive the cross-process boundary") XCTAssertEqual(store.modeId, "off") XCTAssertEqual(store.model, "gpt-4o-mini") } func testAppGroupStoreNoAPIKeySurfacesAsLLMError() async { // Mirror what PolishingService does internally: construct a // client via AppGroupStore with an empty key, expect noAPIKey. // (PolishingService itself lives in the keyboard extension target // and isn't @testable-importable from this test target, so we // exercise the same path one layer down.) let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } let store = AppGroupStore(defaults: defaults) // apiKey stays empty by default — we never wrote one to the suite. let client = store.makeClient() do { _ = try await client.polish("hello", systemPrompt: "p") XCTFail("expected noAPIKey") } catch LLMError.noAPIKey { // ok } catch { XCTFail("wrong error: \(error)") } } // MARK: - TEST-2: cloud always polishes (legacy modeId ignored) /// Cloud engine must invoke the LLM even when a legacy `modeId == "off"` /// value is still present in the App Group suite. func testPolisherPolishesWhenCloudEvenIfModeOffLegacy() async throws { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } defaults.set("off", forKey: "config.modeId") defaults.set("cloud", forKey: "config.engineMode") defaults.set("https://example.com/v1", forKey: "config.baseURL") defaults.set("sk-test", forKey: "config.apiKey") defaults.set("gpt-4o-mini", forKey: "config.model") let counter = CallCounter() let countingClient = CountingLLMClient(counter: counter) { raw, _ in "POLISHED: \(raw)" } let store = AppGroupStore(defaults: defaults) let polisher = PolishingService( store: store, client: countingClient, timeout: 1 ) let result = try await polisher.polish(" hello world ") XCTAssertEqual(result, "POLISHED: hello world") let calls = await counter.value() XCTAssertEqual(calls, 1, "cloud engine must polish even with legacy modeId=off") } /// Local engine is ASR-only and never calls the cloud `LLMClient`. func testPolisherReturnsRawWhenEngineLocal() async throws { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } defaults.set("local", forKey: "config.engineMode") defaults.set("off", forKey: "config.modeId") let counter = CallCounter() let countingClient = CountingLLMClient(counter: counter) { _, _ in XCTFail("cloud LLMClient must not run under local engine") return "" } let store = AppGroupStore(defaults: defaults) let polisher = PolishingService( store: store, client: countingClient, timeout: 1 ) let result = try await polisher.polish(" hello ") XCTAssertEqual(result, "hello") let calls = await counter.value() XCTAssertEqual(calls, 0) } /// Local engine pins DeepSeek — cloud-provider URL/model in App Group /// must not leak into the LLM request (regression: Qwen URL + DeepSeek key → 401). func testResolveLLMEndpointUsesPresetWhenProviderPinned() { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } defaults.set("qwen", forKey: "config.providerId") defaults.set( "https://dashscope.aliyuncs.com/compatible-mode/v1", forKey: "config.baseURL" ) defaults.set("qwen-plus", forKey: "config.model") let store = AppGroupStore(defaults: defaults) let deepseekPreset = LLMProvider.provider(id: "deepseek") let pinned = PolishingService.resolveLLMEndpoint( store: store, preset: deepseekPreset, providerIdOverride: "deepseek" ) XCTAssertEqual(pinned.baseURL, deepseekPreset.defaultBaseURL) XCTAssertEqual(pinned.model, deepseekPreset.defaultModel) let qwenPreset = LLMProvider.provider(id: "qwen") let cloud = PolishingService.resolveLLMEndpoint( store: store, preset: qwenPreset, providerIdOverride: nil ) XCTAssertEqual( cloud.baseURL, "https://dashscope.aliyuncs.com/compatible-mode/v1", "cloud engine must keep user base URL" ) XCTAssertEqual(cloud.model, "qwen-plus", "cloud engine must keep user model") } func testTranslationChipVisibleWithoutTargetLocale() { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } defaults.set("cloud", forKey: "config.engineMode") defaults.set(TranslationLanguageCatalog.offLocaleId, forKey: "config.translationTargetLocaleId") let cloudStore = AppGroupStore(defaults: defaults) XCTAssertTrue(cloudStore.isTranslationChipVisible) XCTAssertFalse(cloudStore.isTranslationEffective) defaults.set("local", forKey: "config.engineMode") defaults.set(TranslationLanguageCatalog.offLocaleId, forKey: "config.translationTargetLocaleId") let localStore = AppGroupStore(defaults: defaults) XCTAssertTrue(localStore.isTranslationChipVisible) XCTAssertFalse(localStore.isTranslationEffective) } /// Local engine always runs the LLM step when translation is armed. func testPolisherTranslatesWhenLocalEngineTranslationEnabled() async throws { let suiteName = "group.com.osgkeyboard.shared.tests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! defaults.removePersistentDomain(forName: suiteName) defer { defaults.removePersistentDomain(forName: suiteName) } defaults.set("local", forKey: "config.engineMode") defaults.set("en", forKey: "config.translationTargetLocaleId") let counter = CallCounter() let countingClient = CountingLLMClient(counter: counter) { raw, prompt in XCTAssertEqual(raw, "你好") XCTAssertTrue(prompt.contains("English"), "translate prompt should target English") return "Hello" } let store = AppGroupStore(defaults: defaults) let polisher = PolishingService( store: store, client: countingClient, timeout: 1 ) let result = try await polisher.polish( " 你好 ", mode: .translate(targetLocaleId: "en"), providerIdOverride: "deepseek" ) XCTAssertEqual(result, "Hello") let calls = await counter.value() XCTAssertEqual(calls, 1) } func testTranslationPromptIncludesAppContextGuideline() { let prompt = TranslationPrompt.make( target: TranslationLanguageCatalog.resolve("en"), providerId: "openai", appContext: .code ) XCTAssertTrue(prompt.localizedCaseInsensitiveContains("preserve English identifiers")) } } // MARK: - Test helpers /// Thread-safe counter for proving a call site never invoked the LLM. private actor CallCounter { private(set) var n = 0 func bump() { n += 1 } func value() -> Int { n } } /// Minimal `LLMClient` that records each call and forwards to a user- /// supplied closure. Used by tests that need to prove a particular /// code path *did not* invoke the client. private struct CountingLLMClient: LLMClient { let counter: CallCounter let body: @Sendable (String, String) async throws -> String var requestTimeout: TimeInterval { 15 } func polish(_ text: String, systemPrompt: String, timeout: TimeInterval?) async throws -> String { await counter.bump() return try await body(text, systemPrompt) } } // MARK: - URLProtocol stub /// Per-test stub config holder. Tests set these via `StubURLProtocol.config =` /// before invoking the code under test, then reset to nil in cleanup. private enum StubURLProtocolStorage { nonisolated(unsafe) static var config: (statusCode: Int, body: Data)? nonisolated(unsafe) static var delaySeconds: Double = 0 nonisolated(unsafe) static var lastRequest: URLRequest? } private final class StubURLProtocol: URLProtocol, @unchecked Sendable { override class func canInit(with request: URLRequest) -> Bool { true } override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } override func startLoading() { let cfg = StubURLProtocolStorage.config ?? (statusCode: 200, body: Data()) let delay = StubURLProtocolStorage.delaySeconds StubURLProtocolStorage.lastRequest = request // Simulate a slow transport. We honour URLProtocol.stopLoading() so // cancellation doesn't leave the test hanging, and we yield to the // run loop so `URLSession.data(for:)` actually observes the delay // (a busy-wait would never let the cooperative scheduler time out). DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + delay) { [weak self] in guard let self else { return } guard self.client != nil else { return } let response = HTTPURLResponse( url: self.request.url!, statusCode: cfg.statusCode, httpVersion: "HTTP/1.1", headerFields: ["Content-Type": "application/json"] )! self.client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) self.client?.urlProtocol(self, didLoad: cfg.body) self.client?.urlProtocolDidFinishLoading(self) } } override func stopLoading() {} }