Expand onboarding and adaptive keyboard intelligence
Add resilient usage analytics, OOBE gateway flows, clipboard semantic ranking, purchase recovery, style learning, and managed current-information search.
This commit is contained in:
Rocky
@@ -0,0 +1,433 @@
|
||||
// ClipboardSemanticAnalyzer.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Fully local clipboard labeling. Deterministic Apple detectors produce
|
||||
// structural facts; project-trained NLModel classifiers add conservative
|
||||
// sentence-level intent labels. No clipboard text leaves the device here.
|
||||
|
||||
import Foundation
|
||||
import NaturalLanguage
|
||||
|
||||
public struct ClipboardLanguageLabel: Equatable, Sendable {
|
||||
public let identifier: String
|
||||
public let confidence: Double
|
||||
}
|
||||
|
||||
public struct ClipboardDateLabel: Equatable, Sendable {
|
||||
public let sourceText: String
|
||||
public let date: Date
|
||||
public let duration: TimeInterval
|
||||
public let timeZoneIdentifier: String?
|
||||
}
|
||||
|
||||
public struct ClipboardTextLabel: Equatable, Sendable {
|
||||
public let sourceText: String
|
||||
}
|
||||
|
||||
public enum ClipboardSentimentLabel: String, Equatable, Sendable {
|
||||
case positive
|
||||
case neutral
|
||||
case negative
|
||||
case unknown
|
||||
}
|
||||
|
||||
public struct ClipboardIntentLabel: Equatable, Sendable {
|
||||
public let confidence: Double
|
||||
public let threshold: Double
|
||||
public let isDetected: Bool
|
||||
public let isApprovedForAutomaticRouting: Bool
|
||||
}
|
||||
|
||||
public struct ClipboardSemanticAnalysis: Equatable, Sendable {
|
||||
public let language: ClipboardLanguageLabel?
|
||||
public let dates: [ClipboardDateLabel]
|
||||
public let addresses: [ClipboardTextLabel]
|
||||
public let phoneNumbers: [ClipboardTextLabel]
|
||||
public let urls: [URL]
|
||||
public let personNames: [ClipboardTextLabel]
|
||||
public let organizationNames: [ClipboardTextLabel]
|
||||
public let sentiment: ClipboardSentimentLabel
|
||||
public let sentimentConfidence: Double
|
||||
public let task: ClipboardIntentLabel
|
||||
public let question: ClipboardIntentLabel
|
||||
public let invitation: ClipboardIntentLabel
|
||||
public let complaint: ClipboardIntentLabel
|
||||
|
||||
public var hasDateOrTime: Bool { !dates.isEmpty }
|
||||
public var hasAddress: Bool { !addresses.isEmpty }
|
||||
public var hasPhoneNumber: Bool { !phoneNumbers.isEmpty }
|
||||
public var hasURL: Bool { !urls.isEmpty }
|
||||
public var hasPersonName: Bool { !personNames.isEmpty }
|
||||
public var hasOrganizationName: Bool { !organizationNames.isEmpty }
|
||||
}
|
||||
|
||||
public actor ClipboardSemanticAnalyzer {
|
||||
private struct Manifest: Decodable {
|
||||
let schemaVersion: Int
|
||||
let classifiers: [ManifestClassifier]
|
||||
}
|
||||
|
||||
private struct ManifestClassifier: Decodable {
|
||||
let id: String
|
||||
let modelFile: String
|
||||
let positiveLabel: String?
|
||||
let confidenceThreshold: Double?
|
||||
let acceptedForAutomaticRouting: Bool
|
||||
}
|
||||
|
||||
private struct ModelEntry {
|
||||
let configuration: ManifestClassifier
|
||||
let model: NLModel
|
||||
}
|
||||
|
||||
private enum IntentID: String, CaseIterable {
|
||||
case task
|
||||
case question
|
||||
case invitation
|
||||
case complaint
|
||||
}
|
||||
|
||||
private static let resourceDirectory = "ClipboardSemantics"
|
||||
private static let manifestName = "clipboard-semantic-models"
|
||||
private static let maximumSemanticSegments = 8
|
||||
private static let maximumSegmentCharacters = 500
|
||||
private static let minimumSentimentConfidence = 0.65
|
||||
private static let minimumSentimentMargin = 0.15
|
||||
|
||||
private let bundles: [Bundle]
|
||||
private var manifest: Manifest?
|
||||
private var models: [String: ModelEntry] = [:]
|
||||
private var didAttemptManifestLoad = false
|
||||
|
||||
public init(additionalBundles: [Bundle] = []) {
|
||||
var resolved = additionalBundles
|
||||
resolved.append(Bundle(for: BundleToken.self))
|
||||
resolved.append(.main)
|
||||
var seen = Set<String>()
|
||||
bundles = resolved.filter { seen.insert($0.bundlePath).inserted }
|
||||
}
|
||||
|
||||
public func analyze(_ sourceText: String) -> ClipboardSemanticAnalysis {
|
||||
let text = sourceText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty else {
|
||||
return emptyAnalysis()
|
||||
}
|
||||
|
||||
let language = languageLabel(for: text)
|
||||
let detectedData = detectStructuredData(in: text)
|
||||
let entities = detectNames(
|
||||
in: text,
|
||||
language: language.flatMap { NLLanguage(rawValue: $0.identifier) }
|
||||
)
|
||||
let segments = semanticSegments(in: text)
|
||||
let task = intentLabel(.task, segments: segments)
|
||||
let question = intentLabel(.question, segments: segments)
|
||||
let invitation = intentLabel(.invitation, segments: segments)
|
||||
let complaint = intentLabel(.complaint, segments: segments)
|
||||
let sentiment = sentimentLabel(segments: segments)
|
||||
|
||||
return ClipboardSemanticAnalysis(
|
||||
language: language,
|
||||
dates: detectedData.dates,
|
||||
addresses: detectedData.addresses,
|
||||
phoneNumbers: detectedData.phoneNumbers,
|
||||
urls: detectedData.urls,
|
||||
personNames: entities.people,
|
||||
organizationNames: entities.organizations,
|
||||
sentiment: sentiment.label,
|
||||
sentimentConfidence: sentiment.confidence,
|
||||
task: task,
|
||||
question: question,
|
||||
invitation: invitation,
|
||||
complaint: complaint
|
||||
)
|
||||
}
|
||||
|
||||
private func emptyAnalysis() -> ClipboardSemanticAnalysis {
|
||||
let emptyIntent = ClipboardIntentLabel(
|
||||
confidence: 0,
|
||||
threshold: 1,
|
||||
isDetected: false,
|
||||
isApprovedForAutomaticRouting: false
|
||||
)
|
||||
return ClipboardSemanticAnalysis(
|
||||
language: nil,
|
||||
dates: [],
|
||||
addresses: [],
|
||||
phoneNumbers: [],
|
||||
urls: [],
|
||||
personNames: [],
|
||||
organizationNames: [],
|
||||
sentiment: .unknown,
|
||||
sentimentConfidence: 0,
|
||||
task: emptyIntent,
|
||||
question: emptyIntent,
|
||||
invitation: emptyIntent,
|
||||
complaint: emptyIntent
|
||||
)
|
||||
}
|
||||
|
||||
private func languageLabel(for text: String) -> ClipboardLanguageLabel? {
|
||||
let recognizer = NLLanguageRecognizer()
|
||||
recognizer.processString(text)
|
||||
guard let dominant = recognizer.dominantLanguage else { return nil }
|
||||
let confidence = recognizer.languageHypotheses(withMaximum: 3)[dominant] ?? 0
|
||||
return ClipboardLanguageLabel(
|
||||
identifier: dominant.rawValue,
|
||||
confidence: rounded(confidence)
|
||||
)
|
||||
}
|
||||
|
||||
private func detectStructuredData(
|
||||
in text: String
|
||||
) -> (
|
||||
dates: [ClipboardDateLabel],
|
||||
addresses: [ClipboardTextLabel],
|
||||
phoneNumbers: [ClipboardTextLabel],
|
||||
urls: [URL]
|
||||
) {
|
||||
let checkingTypes: NSTextCheckingResult.CheckingType = [
|
||||
.date,
|
||||
.address,
|
||||
.phoneNumber,
|
||||
.link
|
||||
]
|
||||
guard let detector = try? NSDataDetector(types: checkingTypes.rawValue) else {
|
||||
return ([], [], [], [])
|
||||
}
|
||||
let range = NSRange(text.startIndex..., in: text)
|
||||
var dates: [ClipboardDateLabel] = []
|
||||
var addresses: [ClipboardTextLabel] = []
|
||||
var phoneNumbers: [ClipboardTextLabel] = []
|
||||
var urls: [URL] = []
|
||||
|
||||
for match in detector.matches(in: text, options: [], range: range) {
|
||||
guard let swiftRange = Range(match.range, in: text) else { continue }
|
||||
let source = String(text[swiftRange])
|
||||
switch match.resultType {
|
||||
case .date:
|
||||
if let date = match.date {
|
||||
dates.append(
|
||||
ClipboardDateLabel(
|
||||
sourceText: source,
|
||||
date: date,
|
||||
duration: match.duration,
|
||||
timeZoneIdentifier: match.timeZone?.identifier
|
||||
)
|
||||
)
|
||||
}
|
||||
case .address:
|
||||
addresses.append(ClipboardTextLabel(sourceText: source))
|
||||
case .phoneNumber:
|
||||
phoneNumbers.append(
|
||||
ClipboardTextLabel(sourceText: match.phoneNumber ?? source)
|
||||
)
|
||||
case .link:
|
||||
if let url = match.url {
|
||||
urls.append(url)
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return (
|
||||
dates,
|
||||
deduplicated(addresses),
|
||||
deduplicated(phoneNumbers),
|
||||
Array(Set(urls)).sorted { $0.absoluteString < $1.absoluteString }
|
||||
)
|
||||
}
|
||||
|
||||
private func detectNames(
|
||||
in text: String,
|
||||
language: NLLanguage?
|
||||
) -> (
|
||||
people: [ClipboardTextLabel],
|
||||
organizations: [ClipboardTextLabel]
|
||||
) {
|
||||
let tagger = NLTagger(tagSchemes: [.nameType])
|
||||
tagger.string = text
|
||||
if let language {
|
||||
tagger.setLanguage(language, range: text.startIndex..<text.endIndex)
|
||||
}
|
||||
var people: [ClipboardTextLabel] = []
|
||||
var organizations: [ClipboardTextLabel] = []
|
||||
tagger.enumerateTags(
|
||||
in: text.startIndex..<text.endIndex,
|
||||
unit: .word,
|
||||
scheme: .nameType,
|
||||
options: [.omitWhitespace, .omitPunctuation, .joinNames]
|
||||
) { tag, range in
|
||||
switch tag {
|
||||
case .personalName:
|
||||
people.append(ClipboardTextLabel(sourceText: String(text[range])))
|
||||
case .organizationName:
|
||||
organizations.append(ClipboardTextLabel(sourceText: String(text[range])))
|
||||
default:
|
||||
break
|
||||
}
|
||||
return true
|
||||
}
|
||||
return (deduplicated(people), deduplicated(organizations))
|
||||
}
|
||||
|
||||
private func semanticSegments(in text: String) -> [String] {
|
||||
if text.count <= Self.maximumSegmentCharacters {
|
||||
return [text]
|
||||
}
|
||||
let tokenizer = NLTokenizer(unit: .sentence)
|
||||
tokenizer.string = text
|
||||
var segments: [String] = []
|
||||
tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { range, _ in
|
||||
let segment = String(text[range])
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !segment.isEmpty {
|
||||
segments.append(String(segment.prefix(Self.maximumSegmentCharacters)))
|
||||
}
|
||||
return segments.count < Self.maximumSemanticSegments
|
||||
}
|
||||
if segments.isEmpty {
|
||||
return [String(text.prefix(Self.maximumSegmentCharacters))]
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
private func intentLabel(
|
||||
_ id: IntentID,
|
||||
segments: [String]
|
||||
) -> ClipboardIntentLabel {
|
||||
guard let entry = modelEntry(id: id.rawValue),
|
||||
let positiveLabel = entry.configuration.positiveLabel else {
|
||||
return ClipboardIntentLabel(
|
||||
confidence: 0,
|
||||
threshold: 1,
|
||||
isDetected: false,
|
||||
isApprovedForAutomaticRouting: false
|
||||
)
|
||||
}
|
||||
let threshold = entry.configuration.confidenceThreshold ?? 1
|
||||
let confidence = segments.map { segment in
|
||||
entry.model.predictedLabelHypotheses(
|
||||
for: segment,
|
||||
maximumCount: 2
|
||||
)[positiveLabel] ?? 0
|
||||
}.max() ?? 0
|
||||
let approved = entry.configuration.acceptedForAutomaticRouting
|
||||
return ClipboardIntentLabel(
|
||||
confidence: rounded(confidence),
|
||||
threshold: rounded(threshold),
|
||||
isDetected: approved && confidence >= threshold,
|
||||
isApprovedForAutomaticRouting: approved
|
||||
)
|
||||
}
|
||||
|
||||
private func sentimentLabel(
|
||||
segments: [String]
|
||||
) -> (label: ClipboardSentimentLabel, confidence: Double) {
|
||||
guard let entry = modelEntry(id: "sentiment") else {
|
||||
return (.unknown, 0)
|
||||
}
|
||||
var totals: [String: Double] = [:]
|
||||
for segment in segments {
|
||||
for (label, confidence) in entry.model.predictedLabelHypotheses(
|
||||
for: segment,
|
||||
maximumCount: 3
|
||||
) {
|
||||
totals[label, default: 0] += confidence
|
||||
}
|
||||
}
|
||||
let divisor = Double(max(segments.count, 1))
|
||||
let ranked = totals
|
||||
.map { (label: $0.key, confidence: $0.value / divisor) }
|
||||
.sorted { $0.confidence > $1.confidence }
|
||||
guard let winner = ranked.first else { return (.unknown, 0) }
|
||||
let runnerUp = ranked.dropFirst().first?.confidence ?? 0
|
||||
guard entry.configuration.acceptedForAutomaticRouting,
|
||||
winner.confidence >= Self.minimumSentimentConfidence,
|
||||
winner.confidence - runnerUp >= Self.minimumSentimentMargin,
|
||||
let label = ClipboardSentimentLabel(rawValue: winner.label)
|
||||
else {
|
||||
return (.unknown, rounded(winner.confidence))
|
||||
}
|
||||
return (label, rounded(winner.confidence))
|
||||
}
|
||||
|
||||
private func modelEntry(id: String) -> ModelEntry? {
|
||||
if let cached = models[id] {
|
||||
return cached
|
||||
}
|
||||
guard let configuration = loadedManifest()?
|
||||
.classifiers
|
||||
.first(where: { $0.id == id }),
|
||||
let modelURL = modelURL(fileName: configuration.modelFile),
|
||||
let model = try? NLModel(contentsOf: modelURL) else {
|
||||
return nil
|
||||
}
|
||||
let entry = ModelEntry(configuration: configuration, model: model)
|
||||
models[id] = entry
|
||||
return entry
|
||||
}
|
||||
|
||||
private func loadedManifest() -> Manifest? {
|
||||
if didAttemptManifestLoad {
|
||||
return manifest
|
||||
}
|
||||
didAttemptManifestLoad = true
|
||||
let decoder = JSONDecoder()
|
||||
for bundle in bundles {
|
||||
let url = bundle.url(
|
||||
forResource: Self.manifestName,
|
||||
withExtension: "json",
|
||||
subdirectory: Self.resourceDirectory
|
||||
) ?? bundle.url(
|
||||
forResource: Self.manifestName,
|
||||
withExtension: "json"
|
||||
)
|
||||
guard let url,
|
||||
let data = try? Data(contentsOf: url),
|
||||
let decoded = try? decoder.decode(Manifest.self, from: data),
|
||||
decoded.schemaVersion == 1 else {
|
||||
continue
|
||||
}
|
||||
manifest = decoded
|
||||
return decoded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private func modelURL(fileName: String) -> URL? {
|
||||
let sourceURL = URL(fileURLWithPath: fileName)
|
||||
let resource = sourceURL.deletingPathExtension().lastPathComponent
|
||||
for bundle in bundles {
|
||||
if let url = bundle.url(
|
||||
forResource: resource,
|
||||
withExtension: "mlmodelc",
|
||||
subdirectory: Self.resourceDirectory
|
||||
) ?? bundle.url(
|
||||
forResource: resource,
|
||||
withExtension: "mlmodelc"
|
||||
) {
|
||||
return url
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private func deduplicated(
|
||||
_ labels: [ClipboardTextLabel]
|
||||
) -> [ClipboardTextLabel] {
|
||||
var seen = Set<String>()
|
||||
return labels.filter {
|
||||
seen.insert($0.sourceText.folding(
|
||||
options: [.caseInsensitive, .diacriticInsensitive],
|
||||
locale: .current
|
||||
)).inserted
|
||||
}
|
||||
}
|
||||
|
||||
private func rounded(_ value: Double) -> Double {
|
||||
(value * 10_000).rounded() / 10_000
|
||||
}
|
||||
}
|
||||
|
||||
private final class BundleToken {}
|
||||
Reference in New Issue
Block a user