Correct product analytics cohorts and reporting
CI / verify (push) Has been cancelled
CI / publish (push) Has been cancelled

This commit is contained in:
Rocky
2026-08-21 17:50:35 +08:00
parent b25f5ae6e9
commit edd0d9feca
31 changed files with 1134 additions and 262 deletions
@@ -0,0 +1,26 @@
package com.osglab.account.features.admin.routes
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import java.time.Clock
import java.time.Instant
import java.time.ZoneOffset
class AdminStatsRangeTest : FunSpec({
val now = Instant.parse("2026-08-20T15:30:00Z")
val clock = Clock.fixed(now, ZoneOffset.UTC)
test("range presets cover exactly N UTC dates including the partial current date") {
parseAdminStatsRange("7d", clock) shouldBe
(Instant.parse("2026-08-14T00:00:00Z") to now)
parseAdminStatsRange("30d", clock) shouldBe
(Instant.parse("2026-07-22T00:00:00Z") to now)
parseAdminStatsRange("90d", clock) shouldBe
(Instant.parse("2026-05-23T00:00:00Z") to now)
}
test("unknown range is rejected") {
parseAdminStatsRange("31d", clock) shouldBe null
}
})
@@ -9,6 +9,8 @@ import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsGrowth
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsGuardrailRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsKeyboardUsageRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsMonetizationRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsLatencyRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsPurchaseFunnelRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsReferralRow
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsWindow
import com.osglab.account.features.admin.stats.repositories.AdminProductAnalyticsRepository
@@ -51,12 +53,14 @@ class AdminProductAnalyticsServiceTest : FunSpec({
result.retention.first().d7?.percent shouldBe 30.0
result.retention.first().d30 shouldBe null
result.growthFunnel.map { it.label } shouldBe listOf(
"首次启动",
"完成注册",
"已完成 24h 观察的新安装",
"24 小时内完成注册",
"24 小时内首次 AI 成功",
"D7 再次使用 AI",
"首次购买",
"24 小时内完成首购",
)
result.referralSignals.shared shouldBe 20
result.monetization.purchaseFunnel.last().count shouldBe 4
result.guardrails.latencyBuckets.single().successful shouldBe 7
captured.single().second.from shouldBe Instant.parse("2026-08-17T00:00:00Z")
captured.single().third.from shouldBe Instant.parse("2026-08-10T00:00:00Z")
}
@@ -141,7 +145,7 @@ private fun snapshot(): AdminProductAnalyticsSnapshot =
conversion30d = AdminAnalyticsCountRow(10, 50),
repeatPurchase = AdminAnalyticsCountRow(2, 10),
),
growthFunnel = AdminAnalyticsGrowthFunnelRow(100, 80, 60, 20, 10),
growthFunnel = AdminAnalyticsGrowthFunnelRow(100, 80, 60, 10),
retention = listOf(
AdminAnalyticsCohortRow(
cohortDate = LocalDate.parse("2026-08-01"),
@@ -176,4 +180,13 @@ private fun snapshot(): AdminProductAnalyticsSnapshot =
managedSuccess = AdminAnalyticsCountRow(95, 100),
creditBlockedUsers = 3,
),
latencyDistribution = listOf(
AdminAnalyticsLatencyRow("S1_TO_3", successful = 7, failed = 1),
),
purchaseFunnel = AdminAnalyticsPurchaseFunnelRow(
viewed = 12,
started = 8,
verified = 4,
cancelled = 2,
),
)
@@ -95,7 +95,10 @@ class AdminStatsRepositoryIntegrationTest : FunSpec({
populatedStats.overview.grantedCredits shouldBeExactly 100
populatedStats.grantedCreditsByDate.values.single() shouldBeExactly 100
factory.query { seedProductAnalytics() }
factory.query {
seedProductAnalytics()
seedAnalyticsCorrectness()
}
val populated = ExposedAdminProductAnalyticsRepository(factory).load(
range = AdminAnalyticsWindow(
from = Instant.parse("2026-08-10T00:00:00Z"),
@@ -111,19 +114,84 @@ class AdminStatsRepositoryIntegrationTest : FunSpec({
),
)
populated.currentWeeklyUsers shouldBeExactly 1
populated.newInstallations shouldBeExactly 1
populated.currentWeeklyUsers shouldBeExactly 4
populated.newInstallations shouldBeExactly 4
populated.activation24h shouldBe
com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow(2, 3)
populated.periodActiveUsers shouldBeExactly 4
populated.successfulAiRequests shouldBeExactly 5
populated.features.single().successes shouldBeExactly 5
populated.retention
.first { it.cohortDate.toString() == "2026-08-11" }
.d1 shouldBeExactly 1
populated.keyboardUsage.activeUsers shouldBeExactly 3
populated.keyboardUsage.keyboardUsers shouldBeExactly 3
populated.keyboardUsage.chineseCharacters shouldBeExactly 140
populated.keyboardUsage.englishCharacters shouldBeExactly 80
populated.keyboardUsage.inputSessions shouldBeExactly 6
populated.growthFunnel.opened shouldBeExactly 3
populated.growthFunnel.registered shouldBeExactly 1
populated.growthFunnel.activated shouldBeExactly 1
populated.growthFunnel.purchased shouldBeExactly 1
populated.referrals.bound shouldBeExactly 1
populated.referrals.activated shouldBeExactly 1
populated.referrals.rewarded shouldBeExactly 1
populated.purchaseFunnel.viewed shouldBeExactly 1
populated.purchaseFunnel.started shouldBeExactly 1
populated.purchaseFunnel.verified shouldBeExactly 1
populated.purchaseFunnel.cancelled shouldBeExactly 1
populated.latencyDistribution.sumOf { it.successful } shouldBeExactly 5
val sevenDayMatured = ExposedAdminProductAnalyticsRepository(factory).load(
range = analyticsWindow(
"2026-08-17T12:00:00Z",
"2026-08-18T12:00:00Z",
),
currentWeek = analyticsWindow(
"2026-08-17T12:00:00Z",
"2026-08-18T12:00:00Z",
),
previousWeek = analyticsWindow(
"2026-08-10T12:00:00Z",
"2026-08-11T12:00:00Z",
),
)
// Account 600...001 completes its seven-day observation window
// inside this report period, despite registering a week earlier.
sevenDayMatured.monetization.conversion7d shouldBe
com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow(1, 1)
populated.periodActiveUsers shouldBeExactly 1
populated.successfulAiRequests shouldBeExactly 2
populated.features.single().successes shouldBeExactly 2
populated.retention.single().d1 shouldBeExactly 1
populated.keyboardUsage.activeUsers shouldBeExactly 1
populated.keyboardUsage.keyboardUsers shouldBeExactly 1
populated.keyboardUsage.chineseCharacters shouldBeExactly 100
populated.keyboardUsage.englishCharacters shouldBeExactly 50
populated.keyboardUsage.inputSessions shouldBeExactly 4
val thirtyDayMatured = ExposedAdminProductAnalyticsRepository(factory).load(
range = analyticsWindow(
"2026-09-09T12:00:00Z",
"2026-09-10T12:00:00Z",
),
currentWeek = analyticsWindow(
"2026-09-09T12:00:00Z",
"2026-09-10T12:00:00Z",
),
previousWeek = analyticsWindow(
"2026-09-02T12:00:00Z",
"2026-09-03T12:00:00Z",
),
)
thirtyDayMatured.monetization.conversion30d shouldBe
com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow(1, 1)
val overview = ExposedAdminStatsRepository(factory).load(
AdminStatsRange(
from = Instant.parse("2026-08-10T00:00:00Z"),
until = Instant.parse("2026-08-17T00:00:00Z"),
),
)
overview.overview.totalUsers shouldBeExactly 3
overview.overview.activeUsers shouldBeExactly 2
overview.referralFunnel.bindings shouldBeExactly 1
overview.referralFunnel.activatedBindings shouldBeExactly 1
overview.referralFunnel.rewardedBindings shouldBeExactly 1
overview.referralRanking.single().invitedUsers shouldBeExactly 1
overview.referralRanking.single().rewardedUsers shouldBeExactly 1
overview.referralRanking.single().earnedCredits shouldBeExactly 25
}
} finally {
factory.close()
@@ -222,3 +290,233 @@ private fun seedProductAnalytics() {
""".trimIndent(),
)
}
private fun seedAnalyticsCorrectness() {
TransactionManager.current().exec(
"""
INSERT INTO accounts (id, apple_sub, created_at, updated_at) VALUES
(
'60000000-0000-0000-0000-000000000001', 'stats-apple-1',
'2026-08-11 00:05:00.000000', '2026-08-11 00:05:00.000000'
),
(
'60000000-0000-0000-0000-000000000002', 'stats-apple-2',
'2026-08-10 00:00:00.000000', '2026-08-10 00:00:00.000000'
),
(
'60000000-0000-0000-0000-000000000003', 'stats-apple-3',
'2026-08-10 00:00:00.000000', '2026-08-10 00:00:00.000000'
)
""".trimIndent(),
)
TransactionManager.current().exec(
"""
INSERT INTO product_analytics_installations (
installation_hash, account_id, created_at, updated_at
) VALUES
(
'${"b".repeat(64)}', '60000000-0000-0000-0000-000000000001',
'2026-08-11 00:00:00.000000', '2026-08-11 00:20:00.000000'
),
(
'${"c".repeat(64)}', '60000000-0000-0000-0000-000000000002',
'2026-08-11 00:00:00.000000', '2026-08-11 00:20:00.000000'
),
(
'${"d".repeat(64)}', NULL,
'2026-08-16 18:00:00.000000', '2026-08-16 18:10:00.000000'
),
(
'${"f".repeat(64)}', NULL,
'2026-08-10 23:50:00.000000', '2026-08-11 00:00:00.000000'
)
""".trimIndent(),
)
listOf(
eventValues("b", "101", "FIRST_OPEN", "2026-08-11 00:00:00", channel = "REFERRAL"),
eventValues(
"b",
"102",
"AI_FEATURE_SUCCEEDED",
"2026-08-11 00:10:00",
feature = "POLISH",
executionMode = "LOCAL",
durationBucket = "LT_1S",
),
eventValues("b", "103", "PURCHASE_VIEWED", "2026-08-11 00:12:00"),
eventValues("b", "104", "PURCHASE_STARTED", "2026-08-11 00:13:00"),
eventValues(
"b",
"105",
"PURCHASE_CANCELLED",
"2026-08-11 00:13:30",
failureCategory = "CANCELLED",
),
eventValues("d", "106", "FIRST_OPEN", "2026-08-16 18:00:00"),
eventValues(
"d",
"107",
"AI_FEATURE_SUCCEEDED",
"2026-08-16 18:10:00",
feature = "POLISH",
executionMode = "LOCAL",
durationBucket = "S3_TO_10",
),
eventValues(
"f",
"108",
"AI_FEATURE_SUCCEEDED",
"2026-08-10 23:50:00",
feature = "POLISH",
executionMode = "LOCAL",
durationBucket = "S1_TO_3",
),
eventValues("f", "109", "FIRST_OPEN", "2026-08-11 00:00:00"),
).forEach { values ->
TransactionManager.current().exec(
"""
INSERT INTO product_analytics_events (
installation_hash, client_event_id, event_name, occurred_at, surface,
acquisition_channel, feature, execution_mode, failure_category,
duration_bucket, app_version, os_version, payload_hash, received_at
) VALUES $values
""".trimIndent(),
)
}
TransactionManager.current().exec(
"""
INSERT INTO keyboard_usage_daily_summaries (
installation_hash, client_summary_id, summary_date,
chinese_character_count, english_character_count, other_character_count,
input_session_count, chinese_only_session_count, english_only_session_count,
mixed_language_session_count, other_only_session_count,
app_version, os_version, payload_hash, received_at
) VALUES
(
'${"b".repeat(64)}', '60000000-0000-0000-0000-000000000101', '2026-08-11',
20, 10, 0, 1, 0, 0, 1, 0,
'1.0', '18.6', '${"b".repeat(64)}', '2026-08-12 00:01:00.000000'
),
(
'${"c".repeat(64)}', '60000000-0000-0000-0000-000000000102', '2026-08-11',
20, 20, 0, 1, 0, 0, 1, 0,
'1.0', '18.6', '${"c".repeat(64)}', '2026-08-12 00:01:00.000000'
)
""".trimIndent(),
)
TransactionManager.current().exec(
"""
INSERT INTO referral_codes (id, owner_user_id, code, created_at)
VALUES (
'60000000-0000-0000-0000-000000000110',
'60000000-0000-0000-0000-000000000002',
'STATS-CODE',
'2026-08-11 00:06:00.000000'
)
""".trimIndent(),
)
TransactionManager.current().exec(
"""
INSERT INTO referral_bindings (
id, inviter_user_id, invitee_user_id, code_id, bound_at,
rewarded_at, reward_settlement_id, reward_status
) VALUES
(
'60000000-0000-0000-0000-000000000111',
'60000000-0000-0000-0000-000000000002',
'60000000-0000-0000-0000-000000000001',
'60000000-0000-0000-0000-000000000110',
'2026-08-11 00:07:00.000000',
'2026-08-11 00:20:00.000000',
'60000000-0000-0000-0000-000000000112',
'REWARDED'
),
(
'60000000-0000-0000-0000-000000000113',
'60000000-0000-0000-0000-000000000002',
'60000000-0000-0000-0000-000000000003',
'60000000-0000-0000-0000-000000000110',
'2026-08-09 00:07:00.000000',
'2026-08-11 00:20:00.000000',
'60000000-0000-0000-0000-000000000114',
'REWARDED'
)
""".trimIndent(),
)
TransactionManager.current().exec(
"""
INSERT INTO credit_ledger (
id, user_id, entry_type, amount_delta, balance_after,
idempotency_key, reference_id, created_at
) VALUES
(
'60000000-0000-0000-0000-000000000201',
'60000000-0000-0000-0000-000000000002',
'REFERRAL_INVITER', 25, 25, 'stats-referral-credit',
'60000000-0000-0000-0000-000000000111',
'2026-08-11 00:20:00.000000'
),
(
'60000000-0000-0000-0000-000000000202',
'60000000-0000-0000-0000-000000000001',
'STOREKIT_PURCHASE', 100, 100, 'stats-storekit-credit',
'60000000-0000-0000-0000-000000000203',
'2026-08-11 00:14:00.000000'
)
""".trimIndent(),
)
TransactionManager.current().exec(
"""
INSERT INTO storekit_credit_purchases (
id, transaction_id, original_transaction_id, user_id, app_account_token,
product_id, environment, credits_granted, ledger_entry_id,
signed_transaction_sha256, purchased_at, signed_at, created_at
) VALUES (
'60000000-0000-0000-0000-000000000203',
'stats-transaction', 'stats-original',
'60000000-0000-0000-0000-000000000001',
'60000000-0000-0000-0000-000000000001',
'com.osglab.credits.test', 'SANDBOX', 100,
'60000000-0000-0000-0000-000000000202',
'${"9".repeat(64)}',
'2026-08-11 00:14:00.000000',
'2026-08-11 00:14:00.000000',
'2026-08-11 00:14:00.000000'
)
""".trimIndent(),
)
}
private fun analyticsWindow(
from: String,
until: String,
): AdminAnalyticsWindow =
AdminAnalyticsWindow(
from = Instant.parse(from),
until = Instant.parse(until),
)
private fun eventValues(
hashCharacter: String,
eventSuffix: String,
eventName: String,
occurredAt: String,
channel: String? = null,
feature: String? = null,
executionMode: String? = null,
failureCategory: String? = null,
durationBucket: String? = null,
): String {
val quoted = { value: String? -> value?.let { "'$it'" } ?: "NULL" }
return """
(
'${hashCharacter.repeat(64)}',
'60000000-0000-0000-0000-000000000$eventSuffix',
'$eventName', '$occurredAt.000000', 'APP',
${quoted(channel)}, ${quoted(feature)}, ${quoted(executionMode)},
${quoted(failureCategory)}, ${quoted(durationBucket)},
'1.0', '18.6', '${eventSuffix.padStart(64, '0')}',
'$occurredAt.000001'
)
""".trimIndent()
}
@@ -52,6 +52,7 @@ class AdminStatsRepositoryTest : FunSpec({
referralFunnel = AdminReferralFunnelDto(
codesCreated = 5,
bindings = 7,
activatedBindings = 5,
rewardedBindings = 4,
pendingBindings = 2,
ineligibleBindings = 1,
@@ -111,6 +112,7 @@ class AdminStatsRepositoryTest : FunSpec({
referralFunnel = AdminReferralFunnelDto(
codesCreated = 0,
bindings = 0,
activatedBindings = 0,
rewardedBindings = 0,
pendingBindings = 3,
ineligibleBindings = 2,
@@ -182,7 +184,7 @@ class AdminStatsRepositoryTest : FunSpec({
registrationsByDate = emptyMap(),
grantedCreditsByDate = emptyMap(),
consumedCreditsByDate = emptyMap(),
referralFunnel = AdminReferralFunnelDto(0, 0, 0, 0, 0),
referralFunnel = AdminReferralFunnelDto(0, 0, 0, 0, 0, 0),
referralRanking = listOf(
AdminReferralRankDto("user-c", 2, 1, 20),
AdminReferralRankDto("user-a", 3, 1, 20),
@@ -108,8 +108,9 @@ class AnalyticsRepositoryIntegrationTest : FunSpec({
}
concurrentResults.sumOf(AnalyticsIngestResult::accepted) shouldBe 1
concurrentResults.sumOf(AnalyticsIngestResult::replayed) shouldBe 7
val concurrentInstallationId = requireNotNull(concurrentRequest.installationId)
val concurrentKeyboardUsage = KeyboardUsageBatchRequest(
installationId = concurrentRequest.installationId,
installationId = concurrentInstallationId,
summaries = listOf(
keyboardSummary().copy(
clientSummaryId = "50000000-0000-0000-0000-000000000099",
@@ -135,14 +136,14 @@ class AnalyticsRepositoryIntegrationTest : FunSpec({
keyboardSummaryCount(config) shouldBe 1
markInstallationUpdatedAt(
config,
concurrentRequest.installationId.sha256Hex(),
concurrentInstallationId.sha256Hex(),
now.minusSeconds(91L * 24 * 60 * 60),
)
repository.purgeAnonymousInstallations(
before = now.minusSeconds(90L * 24 * 60 * 60),
limit = 100,
) shouldBe 1
installationCount(config, concurrentRequest.installationId.sha256Hex()) shouldBe 0
installationCount(config, concurrentInstallationId.sha256Hex()) shouldBe 0
shouldThrow<ConflictException> {
service.ingest(
@@ -129,18 +129,7 @@ class AnalyticsRoutesTest {
}
private fun validBody(): String =
"""
{
"installationId":"$INSTALLATION_ID",
"events":[{
"clientEventId":"40000000-0000-0000-0000-000000000001",
"eventType":"SESSION_STARTED",
"occurredAt":"2026-08-20T01:00:00Z",
"surface":"APP",
"appVersion":"1.0"
}]
}
""".trimIndent()
checkNotNull(javaClass.getResource("/contracts/analytics-events-v1.json")).readText()
private fun validKeyboardUsageBody(): String =
"""
@@ -54,6 +54,47 @@ class AnalyticsServiceTest {
request.toString() shouldNotContain firstOpen().clientEventId
}
@Test
fun `legacy event installation IDs are accepted only when the whole batch agrees`(): Unit =
kotlinx.coroutines.runBlocking {
val repository = InMemoryAnalyticsRepository()
val service = service(repository)
val legacyEvents = listOf(
sessionStarted(uuid(70)).copy(installationId = installationId),
sessionStarted(uuid(71)).copy(installationId = installationId),
)
service.ingest(
accountId = null,
request = AnalyticsBatchRequest(events = legacyEvents),
) shouldBe AnalyticsIngestResult(accepted = 2, replayed = 0)
repository.lastBatch?.installationHash shouldBe installationId.sha256Hex()
listOf(
AnalyticsBatchRequest(events = listOf(sessionStarted(uuid(72)))),
AnalyticsBatchRequest(
events = listOf(
sessionStarted(uuid(73)).copy(installationId = installationId),
sessionStarted(uuid(74)).copy(
installationId = "10000000-0000-0000-0000-000000000002"
),
),
),
AnalyticsBatchRequest(
installationId = installationId,
events = listOf(
sessionStarted(uuid(75)).copy(
installationId = "10000000-0000-0000-0000-000000000002"
)
),
),
).forEach { invalid ->
shouldThrow<InvalidRequestException> {
service.ingest(null, invalid)
}.code shouldBe "invalid_request"
}
}
@Test
fun `authenticated ingestion links an anonymous installation and rejects another account`(): Unit =
kotlinx.coroutines.runBlocking {