Add privacy-safe product analytics
Establish an idempotent analytics pipeline and internal decision dashboard while keeping event metadata allowlisted and account deletion enforceable.
This commit is contained in:
@@ -23,7 +23,10 @@ import com.osglab.account.features.admin.services.AdminBootstrapService
|
||||
import com.osglab.account.features.admin.services.AdminOperatorService
|
||||
import com.osglab.account.features.admin.services.AdminSessionService
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminStatsRepository
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminProductAnalyticsRepository
|
||||
import com.osglab.account.features.admin.stats.repositories.ExposedAdminStatsRepository
|
||||
import com.osglab.account.features.admin.stats.repositories.ExposedAdminProductAnalyticsRepository
|
||||
import com.osglab.account.features.admin.stats.services.AdminProductAnalyticsService
|
||||
import com.osglab.account.features.admin.stats.services.AdminStatsService
|
||||
import com.osglab.account.features.admin.users.repositories.AdminUsersRepository
|
||||
import com.osglab.account.features.admin.users.repositories.ExposedAdminUsersRepository
|
||||
@@ -36,6 +39,12 @@ import com.osglab.account.features.account.AppleAccountReauthenticator
|
||||
import com.osglab.account.features.account.AppleRevocationOutboxProcessor
|
||||
import com.osglab.account.features.account.ExposedAccountRepository
|
||||
import com.osglab.account.features.account.accountRoutes
|
||||
import com.osglab.account.features.analytics.repositories.AnalyticsRepository
|
||||
import com.osglab.account.features.analytics.repositories.ExposedAnalyticsRepository
|
||||
import com.osglab.account.features.analytics.routes.analyticsRoutes
|
||||
import com.osglab.account.features.analytics.services.AnalyticsMaintenanceService
|
||||
import com.osglab.account.features.analytics.services.AnalyticsService
|
||||
import com.osglab.account.features.analytics.services.DefaultAnalyticsService
|
||||
import com.osglab.account.features.appleevents.AppleEventService
|
||||
import com.osglab.account.features.appleevents.AppleEventRepository
|
||||
import com.osglab.account.features.appleevents.AppleEventVerifier
|
||||
@@ -102,6 +111,7 @@ import com.osglab.account.features.integrity.UnavailableAppleDeviceCheckClient
|
||||
import com.osglab.account.features.integrity.createDeviceCheckClient
|
||||
import com.osglab.account.features.integrity.integrityRoutes
|
||||
import com.osglab.account.features.inviteweb.InviteWebConfig
|
||||
import com.osglab.account.features.inviteweb.InviteOpenRecorder
|
||||
import com.osglab.account.features.inviteweb.ReferralLookupPort
|
||||
import com.osglab.account.features.inviteweb.configureInviteWebRoutes
|
||||
import com.osglab.account.features.referrals.domain.ReferralException
|
||||
@@ -261,6 +271,13 @@ fun Application.module() {
|
||||
} catch (_: Exception) {
|
||||
// Durable outbox state is retried; never log sensitive token material.
|
||||
}
|
||||
try {
|
||||
koin.get<AnalyticsMaintenanceService>().purgeStaleAnonymousInstallations()
|
||||
} catch (exception: CancellationException) {
|
||||
throw exception
|
||||
} catch (_: Exception) {
|
||||
// Anonymous analytics cleanup is bounded and retried on the next cycle.
|
||||
}
|
||||
try {
|
||||
koin.get<GatewayReconciliationService>().reconcile()
|
||||
} catch (exception: CancellationException) {
|
||||
@@ -308,7 +325,8 @@ fun Application.module() {
|
||||
}
|
||||
rateLimit(PUBLIC_RATE_LIMIT) {
|
||||
appleEventRoutes(koin.get())
|
||||
configureInviteWebRoutes(koin.get(), koin.get())
|
||||
analyticsRoutes(koin.get())
|
||||
configureInviteWebRoutes(koin.get(), koin.get(), koin.get())
|
||||
integrityRoutes(koin.get())
|
||||
}
|
||||
if (appConfig.admin.enabled) {
|
||||
@@ -319,6 +337,7 @@ fun Application.module() {
|
||||
authService = koin.get(),
|
||||
sessionService = koin.get(),
|
||||
statsService = koin.get(),
|
||||
productAnalyticsService = koin.get(),
|
||||
usersService = koin.get(),
|
||||
grantService = koin.get(),
|
||||
operatorService = koin.get(),
|
||||
@@ -393,6 +412,8 @@ fun accountServerModule(config: AppConfig): Module = module {
|
||||
single { AdminAuditService(get()) }
|
||||
single<AdminStatsRepository> { ExposedAdminStatsRepository(get()) }
|
||||
single { AdminStatsService(get()) }
|
||||
single<AdminProductAnalyticsRepository> { ExposedAdminProductAnalyticsRepository(get()) }
|
||||
single { AdminProductAnalyticsService(get()) }
|
||||
single<AdminUsersRepository> { ExposedAdminUsersRepository(get()) }
|
||||
single { AdminUsersService(get()) }
|
||||
single { AdminGrantService(get()) }
|
||||
@@ -422,6 +443,14 @@ fun accountServerModule(config: AppConfig): Module = module {
|
||||
single<AuthRepository> { ExposedAuthRepository(get()) }
|
||||
single { SessionAccessAuthenticator(get(), get()) }
|
||||
single<AccountRepository> { ExposedAccountRepository(get(), get()) }
|
||||
single<AnalyticsRepository> { ExposedAnalyticsRepository(get()) }
|
||||
single<AnalyticsService> { DefaultAnalyticsService(get()) }
|
||||
single { AnalyticsMaintenanceService(get()) }
|
||||
single<InviteOpenRecorder> {
|
||||
InviteOpenRecorder {
|
||||
get<AnalyticsRepository>().recordInvitePageOpen(Instant.now())
|
||||
}
|
||||
}
|
||||
single<AppleEventRepository> { ExposedAppleEventRepository(get(), get(), config.antiAbuse) }
|
||||
single { AppleEventVerifier(config.apple, get()) }
|
||||
single { AppleEventService(get(), get()) }
|
||||
|
||||
@@ -18,6 +18,7 @@ import com.osglab.account.features.admin.services.AdminOperatorService
|
||||
import com.osglab.account.features.admin.services.AdminSessionService
|
||||
import com.osglab.account.features.admin.stats.models.AdminStatsDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminUsageAggregateDto
|
||||
import com.osglab.account.features.admin.stats.services.AdminProductAnalyticsService
|
||||
import com.osglab.account.features.admin.stats.services.AdminStatsService
|
||||
import com.osglab.account.features.admin.users.models.AdminUserDetailDto
|
||||
import com.osglab.account.features.admin.users.models.AdminUserLedgerEntryDto
|
||||
@@ -59,6 +60,7 @@ fun Route.adminApiRoutes(
|
||||
authService: AdminAuthService,
|
||||
sessionService: AdminSessionService,
|
||||
statsService: AdminStatsService,
|
||||
productAnalyticsService: AdminProductAnalyticsService,
|
||||
usersService: AdminUsersService,
|
||||
grantService: AdminGrantService,
|
||||
operatorService: AdminOperatorService,
|
||||
@@ -158,6 +160,16 @@ fun Route.adminApiRoutes(
|
||||
call.respond(stats.toReferralResponse())
|
||||
}
|
||||
|
||||
get("/analytics") {
|
||||
if (call.requirePrincipal(sessionService) == null) return@get
|
||||
val window = parseAdminStatsRange(call.request.queryParameters["range"], clock)
|
||||
?: run {
|
||||
call.respond(HttpStatusCode.BadRequest, AdminErrorResponse("VALIDATION_ERROR"))
|
||||
return@get
|
||||
}
|
||||
call.respond(productAnalyticsService.get(window.first, window.second))
|
||||
}
|
||||
|
||||
get("/users") {
|
||||
if (
|
||||
call.requireRole(
|
||||
@@ -500,6 +512,11 @@ private suspend fun AdminStatsService.getRange(
|
||||
range: String?,
|
||||
clock: Clock,
|
||||
): AdminStatsDto? {
|
||||
val window = parseAdminStatsRange(range, clock) ?: return null
|
||||
return get(window.first, window.second)
|
||||
}
|
||||
|
||||
private fun parseAdminStatsRange(range: String?, clock: Clock): Pair<java.time.Instant, java.time.Instant>? {
|
||||
val days = when (range) {
|
||||
null, "30d" -> 30L
|
||||
"7d" -> 7L
|
||||
@@ -507,7 +524,7 @@ private suspend fun AdminStatsService.getRange(
|
||||
else -> return null
|
||||
}
|
||||
val until = clock.instant()
|
||||
return get(until.minus(Duration.ofDays(days)), until)
|
||||
return until.minus(Duration.ofDays(days)) to until
|
||||
}
|
||||
|
||||
private suspend fun ApplicationCall.requirePrincipal(
|
||||
@@ -657,8 +674,6 @@ private fun AdminStatsDto.toReferralResponse(): AdminReferralResponse =
|
||||
AdminFunnelResponse("邀请码创建", referralFunnel.codesCreated),
|
||||
AdminFunnelResponse("成功绑定", referralFunnel.bindings),
|
||||
AdminFunnelResponse("有效使用并奖励", referralFunnel.rewardedBindings),
|
||||
AdminFunnelResponse("待资格确认", referralFunnel.pendingBindings),
|
||||
AdminFunnelResponse("不符合奖励条件", referralFunnel.ineligibleBindings),
|
||||
),
|
||||
ranking = referralRanking.map {
|
||||
AdminReferralRankResponse(
|
||||
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.osglab.account.features.admin.stats.models
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsRateDto(
|
||||
val numerator: Long,
|
||||
val denominator: Long,
|
||||
val percent: Double?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsChannelDto(
|
||||
val channel: String,
|
||||
val installations: Long,
|
||||
val activated: Long,
|
||||
val activationRate: AdminAnalyticsRateDto,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsCohortDto(
|
||||
val cohortDate: String,
|
||||
val size: Long,
|
||||
val d1: AdminAnalyticsRateDto?,
|
||||
val d7: AdminAnalyticsRateDto?,
|
||||
val d30: AdminAnalyticsRateDto?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsFeatureUsageDto(
|
||||
val feature: String,
|
||||
val executionMode: String,
|
||||
val users: Long,
|
||||
val successes: Long,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsFunnelStepDto(
|
||||
val label: String,
|
||||
val count: Long,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsPeriodDto(
|
||||
val from: String,
|
||||
val until: String,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsNorthStarDto(
|
||||
val weeklyAiActiveUsers: Long,
|
||||
val previousWeeklyAiActiveUsers: Long,
|
||||
val weekOverWeekPercent: Double?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsGrowthDto(
|
||||
val newInstallations: Long,
|
||||
val newAccounts: Long,
|
||||
val activation24h: AdminAnalyticsRateDto,
|
||||
val medianTimeToValueMinutes: Double?,
|
||||
val channels: List<AdminAnalyticsChannelDto>,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsActivityDto(
|
||||
val dau: Long,
|
||||
val wau: Long,
|
||||
val mau: Long,
|
||||
val stickinessPercent: Double?,
|
||||
val successfulAiRequests: Long,
|
||||
val successfulRequestsPerActiveUser: Double?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsConsumptionDto(
|
||||
val totalCredits: Long,
|
||||
val averageDailyCreditsPerActiveUser: Double?,
|
||||
val medianUserDailyCredits: Double?,
|
||||
val averageCreditsPerManagedRequest: Double?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsMonetizationDto(
|
||||
val payingUsers: Long,
|
||||
val purchases: Long,
|
||||
val creditsPurchased: Long,
|
||||
val conversion7d: AdminAnalyticsRateDto,
|
||||
val conversion30d: AdminAnalyticsRateDto,
|
||||
val repeatPurchaseRate: AdminAnalyticsRateDto,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminAnalyticsGuardrailsDto(
|
||||
val clientAiSuccessRate: AdminAnalyticsRateDto,
|
||||
val managedSuccessRate: AdminAnalyticsRateDto,
|
||||
val creditBlockedUsers: Long,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class AdminProductAnalyticsDto(
|
||||
val period: AdminAnalyticsPeriodDto,
|
||||
val northStar: AdminAnalyticsNorthStarDto,
|
||||
val growth: AdminAnalyticsGrowthDto,
|
||||
val activity: AdminAnalyticsActivityDto,
|
||||
val consumption: AdminAnalyticsConsumptionDto,
|
||||
val monetization: AdminAnalyticsMonetizationDto,
|
||||
val growthFunnel: List<AdminAnalyticsFunnelStepDto>,
|
||||
val retention: List<AdminAnalyticsCohortDto>,
|
||||
val aiFeatures: List<AdminAnalyticsFeatureUsageDto>,
|
||||
val referralFunnel: List<AdminAnalyticsFunnelStepDto>,
|
||||
val guardrails: AdminAnalyticsGuardrailsDto,
|
||||
)
|
||||
+829
@@ -0,0 +1,829 @@
|
||||
package com.osglab.account.features.admin.stats.repositories
|
||||
|
||||
import com.osglab.account.config.DatabaseFactory
|
||||
import org.jetbrains.exposed.v1.core.IColumnType
|
||||
import org.jetbrains.exposed.v1.javatime.JavaInstantColumnType
|
||||
import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager
|
||||
import java.math.BigDecimal
|
||||
import java.sql.ResultSet
|
||||
import java.time.Instant
|
||||
import java.time.LocalDate
|
||||
|
||||
data class AdminAnalyticsWindow(
|
||||
val from: Instant,
|
||||
val until: Instant,
|
||||
) {
|
||||
init {
|
||||
require(from < until)
|
||||
}
|
||||
}
|
||||
|
||||
data class AdminAnalyticsCountRow(
|
||||
val numerator: Long,
|
||||
val denominator: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsChannelRow(
|
||||
val channel: String,
|
||||
val installations: Long,
|
||||
val activated: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsCohortRow(
|
||||
val cohortDate: LocalDate,
|
||||
val size: Long,
|
||||
val d1: Long,
|
||||
val d7: Long,
|
||||
val d30: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsFeatureRow(
|
||||
val feature: String,
|
||||
val executionMode: String,
|
||||
val users: Long,
|
||||
val successes: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsConsumptionRow(
|
||||
val totalCredits: Long,
|
||||
val managedRequests: Long,
|
||||
val averageDailyCreditsPerActiveUser: Double?,
|
||||
val medianUserDailyCredits: Double?,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsMonetizationRow(
|
||||
val payingUsers: Long,
|
||||
val purchases: Long,
|
||||
val creditsPurchased: Long,
|
||||
val conversion7d: AdminAnalyticsCountRow,
|
||||
val conversion30d: AdminAnalyticsCountRow,
|
||||
val repeatPurchase: AdminAnalyticsCountRow,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsReferralRow(
|
||||
val shared: Long,
|
||||
val opened: Long,
|
||||
val bound: Long,
|
||||
val activated: Long,
|
||||
val rewarded: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsGuardrailRow(
|
||||
val clientSuccess: AdminAnalyticsCountRow,
|
||||
val managedSuccess: AdminAnalyticsCountRow,
|
||||
val creditBlockedUsers: Long,
|
||||
)
|
||||
|
||||
data class AdminAnalyticsGrowthFunnelRow(
|
||||
val opened: Long,
|
||||
val registered: Long,
|
||||
val activated: Long,
|
||||
val retainedD7: Long,
|
||||
val purchased: Long,
|
||||
)
|
||||
|
||||
data class AdminProductAnalyticsSnapshot(
|
||||
val currentWeeklyUsers: Long,
|
||||
val previousWeeklyUsers: Long,
|
||||
val newInstallations: Long,
|
||||
val newAccounts: Long,
|
||||
val activation24h: AdminAnalyticsCountRow,
|
||||
val medianTimeToValueMinutes: Double?,
|
||||
val channels: List<AdminAnalyticsChannelRow>,
|
||||
val dau: Long,
|
||||
val wau: Long,
|
||||
val mau: Long,
|
||||
val periodActiveUsers: Long,
|
||||
val successfulAiRequests: Long,
|
||||
val consumption: AdminAnalyticsConsumptionRow,
|
||||
val monetization: AdminAnalyticsMonetizationRow,
|
||||
val growthFunnel: AdminAnalyticsGrowthFunnelRow,
|
||||
val retention: List<AdminAnalyticsCohortRow>,
|
||||
val features: List<AdminAnalyticsFeatureRow>,
|
||||
val referrals: AdminAnalyticsReferralRow,
|
||||
val guardrails: AdminAnalyticsGuardrailRow,
|
||||
)
|
||||
|
||||
interface AdminProductAnalyticsRepository {
|
||||
suspend fun load(
|
||||
range: AdminAnalyticsWindow,
|
||||
currentWeek: AdminAnalyticsWindow,
|
||||
previousWeek: AdminAnalyticsWindow,
|
||||
): AdminProductAnalyticsSnapshot
|
||||
}
|
||||
|
||||
class ExposedAdminProductAnalyticsRepository(
|
||||
private val databaseFactory: DatabaseFactory,
|
||||
) : AdminProductAnalyticsRepository {
|
||||
override suspend fun load(
|
||||
range: AdminAnalyticsWindow,
|
||||
currentWeek: AdminAnalyticsWindow,
|
||||
previousWeek: AdminAnalyticsWindow,
|
||||
): AdminProductAnalyticsSnapshot = databaseFactory.query {
|
||||
val activation = loadActivation(range)
|
||||
AdminProductAnalyticsSnapshot(
|
||||
currentWeeklyUsers = loadValueActiveUsers(currentWeek),
|
||||
previousWeeklyUsers = loadValueActiveUsers(previousWeek),
|
||||
newInstallations = activation.denominator,
|
||||
newAccounts = loadNewAccounts(range),
|
||||
activation24h = AdminAnalyticsCountRow(activation.activated, activation.denominator),
|
||||
medianTimeToValueMinutes = activation.medianMinutes,
|
||||
channels = loadChannels(range),
|
||||
dau = loadValueActiveUsers(
|
||||
AdminAnalyticsWindow(range.until.minusSeconds(DAY_SECONDS), range.until),
|
||||
),
|
||||
wau = loadValueActiveUsers(
|
||||
AdminAnalyticsWindow(range.until.minusSeconds(7 * DAY_SECONDS), range.until),
|
||||
),
|
||||
mau = loadValueActiveUsers(
|
||||
AdminAnalyticsWindow(range.until.minusSeconds(30 * DAY_SECONDS), range.until),
|
||||
),
|
||||
periodActiveUsers = loadValueActiveUsers(range),
|
||||
successfulAiRequests = loadSuccessfulAiRequests(range),
|
||||
consumption = loadConsumption(range),
|
||||
monetization = loadMonetization(range),
|
||||
growthFunnel = loadGrowthFunnel(range),
|
||||
retention = loadRetention(range),
|
||||
features = loadFeatures(range),
|
||||
referrals = loadReferrals(range),
|
||||
guardrails = loadGuardrails(range),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadValueActiveUsers(window: AdminAnalyticsWindow): Long =
|
||||
querySingle(
|
||||
valueEventsCte() +
|
||||
"""
|
||||
SELECT COUNT(DISTINCT identity_key) AS aggregate_value
|
||||
FROM value_events
|
||||
WHERE occurred_at >= ? AND occurred_at < ?
|
||||
""",
|
||||
window.arguments(),
|
||||
) { it.exactLong("aggregate_value") }
|
||||
|
||||
private fun loadNewAccounts(range: AdminAnalyticsWindow): Long =
|
||||
querySingle(
|
||||
"""
|
||||
SELECT COUNT(*) AS aggregate_value
|
||||
FROM accounts
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
""",
|
||||
range.arguments(),
|
||||
) { it.exactLong("aggregate_value") }
|
||||
|
||||
private fun loadActivation(range: AdminAnalyticsWindow): ActivationRow =
|
||||
querySingle(
|
||||
"""
|
||||
WITH first_open AS (
|
||||
SELECT installation_hash, MIN(occurred_at) AS opened_at
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'FIRST_OPEN'
|
||||
AND occurred_at >= ? AND occurred_at < ?
|
||||
GROUP BY installation_hash
|
||||
),
|
||||
client_value AS (
|
||||
SELECT installation_hash, MIN(occurred_at) AS value_at
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
AND execution_mode IN ('LOCAL', 'BYOK')
|
||||
GROUP BY installation_hash
|
||||
),
|
||||
managed_value AS (
|
||||
SELECT i.installation_hash, MIN(u.created_at) AS value_at
|
||||
FROM product_analytics_installations i
|
||||
JOIN credit_usage_records u ON u.user_id = i.account_id
|
||||
GROUP BY i.installation_hash
|
||||
),
|
||||
first_value_by_install AS (
|
||||
SELECT installation_hash, MIN(value_at) AS value_at
|
||||
FROM (
|
||||
SELECT * FROM client_value
|
||||
UNION ALL
|
||||
SELECT * FROM managed_value
|
||||
) values_by_source
|
||||
GROUP BY installation_hash
|
||||
),
|
||||
activated AS (
|
||||
SELECT
|
||||
o.installation_hash,
|
||||
TIMESTAMPDIFF(SECOND, o.opened_at, v.value_at) AS seconds_to_value
|
||||
FROM first_open o
|
||||
JOIN first_value_by_install v ON v.installation_hash = o.installation_hash
|
||||
WHERE v.value_at >= o.opened_at
|
||||
AND v.value_at <= DATE_ADD(o.opened_at, INTERVAL 24 HOUR)
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
seconds_to_value,
|
||||
ROW_NUMBER() OVER (ORDER BY seconds_to_value) AS row_number_value,
|
||||
COUNT(*) OVER () AS total_rows
|
||||
FROM activated
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM first_open) AS denominator_value,
|
||||
(SELECT COUNT(*) FROM activated) AS activated_value,
|
||||
(
|
||||
SELECT AVG(seconds_to_value) / 60.0
|
||||
FROM ranked
|
||||
WHERE row_number_value IN (
|
||||
FLOOR((total_rows + 1) / 2),
|
||||
FLOOR((total_rows + 2) / 2)
|
||||
)
|
||||
) AS median_minutes
|
||||
""",
|
||||
range.arguments(),
|
||||
) {
|
||||
ActivationRow(
|
||||
denominator = it.exactLong("denominator_value"),
|
||||
activated = it.exactLong("activated_value"),
|
||||
medianMinutes = it.nullableDouble("median_minutes"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadChannels(range: AdminAnalyticsWindow): List<AdminAnalyticsChannelRow> =
|
||||
queryRows(
|
||||
"""
|
||||
WITH first_open AS (
|
||||
SELECT
|
||||
e.installation_hash,
|
||||
MIN(e.occurred_at) AS opened_at,
|
||||
COALESCE(
|
||||
MAX(
|
||||
CASE
|
||||
WHEN e.acquisition_channel <> 'UNKNOWN'
|
||||
THEN e.acquisition_channel
|
||||
END
|
||||
),
|
||||
'UNKNOWN'
|
||||
) AS channel
|
||||
FROM product_analytics_events e
|
||||
WHERE e.event_name = 'FIRST_OPEN'
|
||||
AND e.occurred_at >= ? AND e.occurred_at < ?
|
||||
GROUP BY e.installation_hash
|
||||
),
|
||||
client_value AS (
|
||||
SELECT installation_hash, MIN(occurred_at) AS value_at
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
AND execution_mode IN ('LOCAL', 'BYOK')
|
||||
GROUP BY installation_hash
|
||||
),
|
||||
managed_value AS (
|
||||
SELECT i.installation_hash, MIN(u.created_at) AS value_at
|
||||
FROM product_analytics_installations i
|
||||
JOIN credit_usage_records u ON u.user_id = i.account_id
|
||||
GROUP BY i.installation_hash
|
||||
),
|
||||
first_value_by_install AS (
|
||||
SELECT installation_hash, MIN(value_at) AS value_at
|
||||
FROM (
|
||||
SELECT * FROM client_value
|
||||
UNION ALL
|
||||
SELECT * FROM managed_value
|
||||
) values_by_source
|
||||
GROUP BY installation_hash
|
||||
)
|
||||
SELECT
|
||||
o.channel,
|
||||
COUNT(*) AS installations,
|
||||
SUM(
|
||||
CASE
|
||||
WHEN v.value_at >= o.opened_at
|
||||
AND v.value_at <= DATE_ADD(o.opened_at, INTERVAL 24 HOUR)
|
||||
THEN 1 ELSE 0
|
||||
END
|
||||
) AS activated
|
||||
FROM first_open o
|
||||
LEFT JOIN first_value_by_install v ON v.installation_hash = o.installation_hash
|
||||
GROUP BY o.channel
|
||||
ORDER BY o.channel
|
||||
""",
|
||||
range.arguments(),
|
||||
) {
|
||||
AdminAnalyticsChannelRow(
|
||||
channel = it.getString("channel"),
|
||||
installations = it.exactLong("installations"),
|
||||
activated = it.exactLong("activated"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadSuccessfulAiRequests(range: AdminAnalyticsWindow): Long =
|
||||
querySingle(
|
||||
"""
|
||||
SELECT
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM credit_usage_records
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
) + (
|
||||
SELECT COUNT(*)
|
||||
FROM product_analytics_events
|
||||
WHERE occurred_at >= ? AND occurred_at < ?
|
||||
AND event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
AND execution_mode IN ('LOCAL', 'BYOK')
|
||||
) AS aggregate_value
|
||||
""",
|
||||
range.arguments(repetitions = 2),
|
||||
) { it.exactLong("aggregate_value") }
|
||||
|
||||
private fun loadConsumption(range: AdminAnalyticsWindow): AdminAnalyticsConsumptionRow =
|
||||
querySingle(
|
||||
"""
|
||||
WITH user_days AS (
|
||||
SELECT
|
||||
user_id,
|
||||
DATE(created_at) AS usage_date,
|
||||
SUM(charged_credits) AS daily_credits
|
||||
FROM credit_usage_records
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
GROUP BY user_id, DATE(created_at)
|
||||
),
|
||||
day_totals AS (
|
||||
SELECT
|
||||
usage_date,
|
||||
SUM(daily_credits) AS credits,
|
||||
COUNT(*) AS users
|
||||
FROM user_days
|
||||
GROUP BY usage_date
|
||||
),
|
||||
ranked_user_days AS (
|
||||
SELECT
|
||||
daily_credits,
|
||||
ROW_NUMBER() OVER (ORDER BY daily_credits) AS row_number_value,
|
||||
COUNT(*) OVER () AS total_rows
|
||||
FROM user_days
|
||||
)
|
||||
SELECT
|
||||
COALESCE((SELECT SUM(daily_credits) FROM user_days), 0) AS total_credits,
|
||||
COALESCE((SELECT COUNT(*) FROM credit_usage_records
|
||||
WHERE created_at >= ? AND created_at < ?), 0) AS managed_requests,
|
||||
(SELECT AVG(credits / NULLIF(users, 0)) FROM day_totals)
|
||||
AS average_daily_per_user,
|
||||
(
|
||||
SELECT AVG(daily_credits)
|
||||
FROM ranked_user_days
|
||||
WHERE row_number_value IN (
|
||||
FLOOR((total_rows + 1) / 2),
|
||||
FLOOR((total_rows + 2) / 2)
|
||||
)
|
||||
) AS median_user_day
|
||||
""",
|
||||
range.arguments(repetitions = 2),
|
||||
) {
|
||||
AdminAnalyticsConsumptionRow(
|
||||
totalCredits = it.exactLong("total_credits"),
|
||||
managedRequests = it.exactLong("managed_requests"),
|
||||
averageDailyCreditsPerActiveUser = it.nullableDouble("average_daily_per_user"),
|
||||
medianUserDailyCredits = it.nullableDouble("median_user_day"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadMonetization(range: AdminAnalyticsWindow): AdminAnalyticsMonetizationRow {
|
||||
val sevenDayMaturity = range.until.minusSeconds(7 * DAY_SECONDS)
|
||||
val thirtyDayMaturity = range.until.minusSeconds(30 * DAY_SECONDS)
|
||||
return querySingle(
|
||||
"""
|
||||
WITH first_purchase AS (
|
||||
SELECT user_id, MIN(purchased_at) AS first_purchased_at, COUNT(*) AS lifetime_purchases
|
||||
FROM storekit_credit_purchases
|
||||
GROUP BY user_id
|
||||
),
|
||||
period_payers AS (
|
||||
SELECT user_id, COUNT(*) AS period_purchases
|
||||
FROM storekit_credit_purchases
|
||||
WHERE purchased_at >= ? AND purchased_at < ?
|
||||
GROUP BY user_id
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM period_payers) AS paying_users,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM storekit_credit_purchases
|
||||
WHERE purchased_at >= ? AND purchased_at < ?
|
||||
) AS purchases,
|
||||
(
|
||||
SELECT COALESCE(SUM(credits_granted), 0)
|
||||
FROM storekit_credit_purchases
|
||||
WHERE purchased_at >= ? AND purchased_at < ?
|
||||
) AS credits_purchased,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM accounts
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
) AS conversion_7_denominator,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM accounts a
|
||||
JOIN first_purchase p ON p.user_id = a.id
|
||||
WHERE a.created_at >= ? AND a.created_at < ?
|
||||
AND p.first_purchased_at <= DATE_ADD(a.created_at, INTERVAL 7 DAY)
|
||||
) AS conversion_7_numerator,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM accounts
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
) AS conversion_30_denominator,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM accounts a
|
||||
JOIN first_purchase p ON p.user_id = a.id
|
||||
WHERE a.created_at >= ? AND a.created_at < ?
|
||||
AND p.first_purchased_at <= DATE_ADD(a.created_at, INTERVAL 30 DAY)
|
||||
) AS conversion_30_numerator,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM period_payers pp
|
||||
JOIN first_purchase fp ON fp.user_id = pp.user_id
|
||||
WHERE fp.lifetime_purchases >= 2
|
||||
) AS repeat_numerator,
|
||||
(SELECT COUNT(*) FROM period_payers) AS repeat_denominator
|
||||
""",
|
||||
buildList {
|
||||
addAll(range.arguments(repetitions = 3))
|
||||
addAll(maturedWindowArguments(range.from, sevenDayMaturity))
|
||||
addAll(maturedWindowArguments(range.from, sevenDayMaturity))
|
||||
addAll(maturedWindowArguments(range.from, thirtyDayMaturity))
|
||||
addAll(maturedWindowArguments(range.from, thirtyDayMaturity))
|
||||
},
|
||||
) {
|
||||
AdminAnalyticsMonetizationRow(
|
||||
payingUsers = it.exactLong("paying_users"),
|
||||
purchases = it.exactLong("purchases"),
|
||||
creditsPurchased = it.exactLong("credits_purchased"),
|
||||
conversion7d = AdminAnalyticsCountRow(
|
||||
it.exactLong("conversion_7_numerator"),
|
||||
it.exactLong("conversion_7_denominator"),
|
||||
),
|
||||
conversion30d = AdminAnalyticsCountRow(
|
||||
it.exactLong("conversion_30_numerator"),
|
||||
it.exactLong("conversion_30_denominator"),
|
||||
),
|
||||
repeatPurchase = AdminAnalyticsCountRow(
|
||||
it.exactLong("repeat_numerator"),
|
||||
it.exactLong("repeat_denominator"),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun loadGrowthFunnel(range: AdminAnalyticsWindow): AdminAnalyticsGrowthFunnelRow =
|
||||
querySingle(
|
||||
"""
|
||||
WITH first_open AS (
|
||||
SELECT installation_hash, MIN(occurred_at) AS opened_at
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'FIRST_OPEN'
|
||||
AND occurred_at >= ? AND occurred_at < ?
|
||||
GROUP BY installation_hash
|
||||
),
|
||||
client_values AS (
|
||||
SELECT installation_hash, occurred_at
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
AND execution_mode IN ('LOCAL', 'BYOK')
|
||||
),
|
||||
managed_values AS (
|
||||
SELECT i.installation_hash, u.created_at AS occurred_at
|
||||
FROM product_analytics_installations i
|
||||
JOIN credit_usage_records u ON u.user_id = i.account_id
|
||||
),
|
||||
values_by_install AS (
|
||||
SELECT * FROM client_values
|
||||
UNION ALL
|
||||
SELECT * FROM managed_values
|
||||
),
|
||||
activated AS (
|
||||
SELECT
|
||||
o.installation_hash,
|
||||
o.opened_at,
|
||||
MIN(v.occurred_at) AS first_value_at
|
||||
FROM first_open o
|
||||
JOIN values_by_install v ON v.installation_hash = o.installation_hash
|
||||
WHERE v.occurred_at >= o.opened_at
|
||||
AND v.occurred_at <= DATE_ADD(o.opened_at, INTERVAL 24 HOUR)
|
||||
GROUP BY o.installation_hash, o.opened_at
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM first_open) AS opened,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM first_open o
|
||||
JOIN product_analytics_installations i
|
||||
ON i.installation_hash = o.installation_hash
|
||||
WHERE i.account_id IS NOT NULL
|
||||
) AS registered,
|
||||
(SELECT COUNT(*) FROM activated) AS activated,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM activated a
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM values_by_install v
|
||||
WHERE v.installation_hash = a.installation_hash
|
||||
AND DATE(v.occurred_at) = DATE_ADD(DATE(a.first_value_at), INTERVAL 7 DAY)
|
||||
)
|
||||
) AS retained_d7,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM first_open o
|
||||
JOIN product_analytics_installations i
|
||||
ON i.installation_hash = o.installation_hash
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM storekit_credit_purchases p
|
||||
WHERE p.user_id = i.account_id
|
||||
AND p.purchased_at >= o.opened_at
|
||||
AND p.purchased_at < ?
|
||||
)
|
||||
) AS purchased
|
||||
""",
|
||||
range.arguments() + listOf(INSTANT_COLUMN_TYPE to range.until),
|
||||
) {
|
||||
AdminAnalyticsGrowthFunnelRow(
|
||||
opened = it.exactLong("opened"),
|
||||
registered = it.exactLong("registered"),
|
||||
activated = it.exactLong("activated"),
|
||||
retainedD7 = it.exactLong("retained_d7"),
|
||||
purchased = it.exactLong("purchased"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadRetention(range: AdminAnalyticsWindow): List<AdminAnalyticsCohortRow> =
|
||||
queryRows(
|
||||
valueEventsCte() +
|
||||
"""
|
||||
, first_value_by_identity AS (
|
||||
SELECT identity_key, MIN(occurred_at) AS first_value_at
|
||||
FROM value_events
|
||||
GROUP BY identity_key
|
||||
)
|
||||
SELECT
|
||||
DATE(f.first_value_at) AS cohort_date,
|
||||
COUNT(DISTINCT f.identity_key) AS cohort_size,
|
||||
COUNT(
|
||||
DISTINCT CASE
|
||||
WHEN DATEDIFF(DATE(v.occurred_at), DATE(f.first_value_at)) = 1
|
||||
THEN f.identity_key
|
||||
END
|
||||
) AS retained_d1,
|
||||
COUNT(
|
||||
DISTINCT CASE
|
||||
WHEN DATEDIFF(DATE(v.occurred_at), DATE(f.first_value_at)) = 7
|
||||
THEN f.identity_key
|
||||
END
|
||||
) AS retained_d7,
|
||||
COUNT(
|
||||
DISTINCT CASE
|
||||
WHEN DATEDIFF(DATE(v.occurred_at), DATE(f.first_value_at)) = 30
|
||||
THEN f.identity_key
|
||||
END
|
||||
) AS retained_d30
|
||||
FROM first_value_by_identity f
|
||||
LEFT JOIN value_events v ON v.identity_key = f.identity_key
|
||||
WHERE f.first_value_at >= ? AND f.first_value_at < ?
|
||||
GROUP BY DATE(f.first_value_at)
|
||||
ORDER BY cohort_date DESC
|
||||
""",
|
||||
range.arguments(),
|
||||
) {
|
||||
AdminAnalyticsCohortRow(
|
||||
cohortDate = it.getObject("cohort_date", LocalDate::class.java),
|
||||
size = it.exactLong("cohort_size"),
|
||||
d1 = it.exactLong("retained_d1"),
|
||||
d7 = it.exactLong("retained_d7"),
|
||||
d30 = it.exactLong("retained_d30"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadFeatures(range: AdminAnalyticsWindow): List<AdminAnalyticsFeatureRow> =
|
||||
queryRows(
|
||||
"""
|
||||
SELECT
|
||||
e.feature,
|
||||
e.execution_mode,
|
||||
COUNT(
|
||||
DISTINCT COALESCE(
|
||||
CONCAT('a:', i.account_id),
|
||||
CONCAT('i:', e.installation_hash)
|
||||
)
|
||||
) AS users,
|
||||
COUNT(*) AS successes
|
||||
FROM product_analytics_events e
|
||||
JOIN product_analytics_installations i
|
||||
ON i.installation_hash = e.installation_hash
|
||||
WHERE e.occurred_at >= ? AND e.occurred_at < ?
|
||||
AND e.event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
GROUP BY e.feature, e.execution_mode
|
||||
ORDER BY successes DESC, e.feature, e.execution_mode
|
||||
""",
|
||||
range.arguments(),
|
||||
) {
|
||||
AdminAnalyticsFeatureRow(
|
||||
feature = it.getString("feature"),
|
||||
executionMode = it.getString("execution_mode"),
|
||||
users = it.exactLong("users"),
|
||||
successes = it.exactLong("successes"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadReferrals(range: AdminAnalyticsWindow): AdminAnalyticsReferralRow =
|
||||
querySingle(
|
||||
valueEventsCte() +
|
||||
"""
|
||||
SELECT
|
||||
(
|
||||
SELECT COUNT(DISTINCT installation_hash)
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'REFERRAL_SHARED'
|
||||
AND occurred_at >= ? AND occurred_at < ?
|
||||
) AS shared,
|
||||
(
|
||||
SELECT COUNT(DISTINCT installation_hash)
|
||||
FROM product_analytics_events
|
||||
WHERE event_name = 'INVITE_OPENED'
|
||||
AND occurred_at >= ? AND occurred_at < ?
|
||||
) + (
|
||||
SELECT COALESCE(SUM(counter_value), 0)
|
||||
FROM product_analytics_daily_counters
|
||||
WHERE counter_name = 'INVITE_PAGE_OPENED'
|
||||
AND counter_date >= DATE(?) AND counter_date < DATE(?)
|
||||
) AS opened,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM referral_bindings
|
||||
WHERE bound_at >= ? AND bound_at < ?
|
||||
) AS bound,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM referral_bindings r
|
||||
WHERE r.bound_at >= ? AND r.bound_at < ?
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM value_events v
|
||||
WHERE v.identity_key = CONCAT('a:', r.invitee_user_id)
|
||||
AND v.occurred_at >= r.bound_at
|
||||
AND v.occurred_at < ?
|
||||
)
|
||||
) AS activated,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM referral_bindings
|
||||
WHERE bound_at >= ? AND bound_at < ?
|
||||
AND reward_status = 'REWARDED'
|
||||
) AS rewarded
|
||||
""",
|
||||
buildList {
|
||||
addAll(range.arguments(repetitions = 5))
|
||||
add(INSTANT_COLUMN_TYPE to range.until)
|
||||
addAll(range.arguments())
|
||||
},
|
||||
) {
|
||||
AdminAnalyticsReferralRow(
|
||||
shared = it.exactLong("shared"),
|
||||
opened = it.exactLong("opened"),
|
||||
bound = it.exactLong("bound"),
|
||||
activated = it.exactLong("activated"),
|
||||
rewarded = it.exactLong("rewarded"),
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadGuardrails(range: AdminAnalyticsWindow): AdminAnalyticsGuardrailRow =
|
||||
querySingle(
|
||||
"""
|
||||
SELECT
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM product_analytics_events
|
||||
WHERE occurred_at >= ? AND occurred_at < ?
|
||||
AND event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
) AS client_success,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM product_analytics_events
|
||||
WHERE occurred_at >= ? AND occurred_at < ?
|
||||
AND event_name IN ('AI_FEATURE_SUCCEEDED', 'AI_FEATURE_FAILED')
|
||||
) AS client_terminal,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM provider_requests
|
||||
WHERE completed_at >= ? AND completed_at < ?
|
||||
AND status = 'SETTLED'
|
||||
) AS managed_success,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM provider_requests
|
||||
WHERE completed_at >= ? AND completed_at < ?
|
||||
AND status IN ('SETTLED', 'RELEASED', 'MANUAL_REVIEW')
|
||||
) AS managed_terminal,
|
||||
(
|
||||
SELECT COUNT(
|
||||
DISTINCT COALESCE(
|
||||
CONCAT('a:', i.account_id),
|
||||
CONCAT('i:', e.installation_hash)
|
||||
)
|
||||
)
|
||||
FROM product_analytics_events e
|
||||
JOIN product_analytics_installations i
|
||||
ON i.installation_hash = e.installation_hash
|
||||
WHERE e.occurred_at >= ? AND e.occurred_at < ?
|
||||
AND e.event_name = 'AI_FEATURE_FAILED'
|
||||
AND e.failure_category = 'INSUFFICIENT_CREDITS'
|
||||
) AS credit_blocked_users
|
||||
""",
|
||||
range.arguments(repetitions = 5),
|
||||
) {
|
||||
AdminAnalyticsGuardrailRow(
|
||||
clientSuccess = AdminAnalyticsCountRow(
|
||||
it.exactLong("client_success"),
|
||||
it.exactLong("client_terminal"),
|
||||
),
|
||||
managedSuccess = AdminAnalyticsCountRow(
|
||||
it.exactLong("managed_success"),
|
||||
it.exactLong("managed_terminal"),
|
||||
),
|
||||
creditBlockedUsers = it.exactLong("credit_blocked_users"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private data class ActivationRow(
|
||||
val denominator: Long,
|
||||
val activated: Long,
|
||||
val medianMinutes: Double?,
|
||||
)
|
||||
|
||||
private fun valueEventsCte(): String =
|
||||
"""
|
||||
WITH value_events AS (
|
||||
SELECT
|
||||
CONCAT('a:', user_id) AS identity_key,
|
||||
created_at AS occurred_at
|
||||
FROM credit_usage_records
|
||||
UNION ALL
|
||||
SELECT
|
||||
COALESCE(
|
||||
CONCAT('a:', i.account_id),
|
||||
CONCAT('i:', e.installation_hash)
|
||||
) AS identity_key,
|
||||
e.occurred_at
|
||||
FROM product_analytics_events e
|
||||
JOIN product_analytics_installations i
|
||||
ON i.installation_hash = e.installation_hash
|
||||
WHERE e.event_name = 'AI_FEATURE_SUCCEEDED'
|
||||
AND e.execution_mode IN ('LOCAL', 'BYOK')
|
||||
)
|
||||
""".trimIndent()
|
||||
|
||||
private fun AdminAnalyticsWindow.arguments(
|
||||
repetitions: Int = 1,
|
||||
): List<Pair<IColumnType<*>, Any?>> = buildList {
|
||||
repeat(repetitions) {
|
||||
add(INSTANT_COLUMN_TYPE to from)
|
||||
add(INSTANT_COLUMN_TYPE to until)
|
||||
}
|
||||
}
|
||||
|
||||
private fun maturedWindowArguments(
|
||||
from: Instant,
|
||||
maturityEnd: Instant,
|
||||
): List<Pair<IColumnType<*>, Any?>> =
|
||||
listOf(
|
||||
INSTANT_COLUMN_TYPE to from,
|
||||
INSTANT_COLUMN_TYPE to maxOf(from, maturityEnd),
|
||||
)
|
||||
|
||||
private fun <T> querySingle(
|
||||
sql: String,
|
||||
arguments: List<Pair<IColumnType<*>, Any?>>,
|
||||
transform: (ResultSet) -> T,
|
||||
): T = queryRows(sql, arguments, transform).single()
|
||||
|
||||
private fun <T> queryRows(
|
||||
sql: String,
|
||||
arguments: List<Pair<IColumnType<*>, Any?>>,
|
||||
transform: (ResultSet) -> T,
|
||||
): List<T> {
|
||||
val normalized = sql.trimIndent()
|
||||
// Exposed classifies statements beginning with WITH as updates. Wrapping the
|
||||
// CTE keeps prepared arguments and makes the statement unambiguously a query.
|
||||
val executable = if (normalized.startsWith("WITH ", ignoreCase = true)) {
|
||||
"SELECT * FROM (\n$normalized\n) AS analytics_result"
|
||||
} else {
|
||||
normalized
|
||||
}
|
||||
return TransactionManager.current().exec(executable, arguments) { result ->
|
||||
buildList {
|
||||
while (result.next()) add(transform(result))
|
||||
}
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
private fun ResultSet.exactLong(column: String): Long =
|
||||
requireNotNull(getBigDecimal(column)) { "Aggregate column $column must not be null" }
|
||||
.longValueExact()
|
||||
|
||||
private fun ResultSet.nullableDouble(column: String): Double? =
|
||||
getBigDecimal(column)?.toDouble()
|
||||
|
||||
private val INSTANT_COLUMN_TYPE = JavaInstantColumnType()
|
||||
private const val DAY_SECONDS = 86_400L
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
package com.osglab.account.features.admin.stats.services
|
||||
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsActivityDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsChannelDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsCohortDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsConsumptionDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsFeatureUsageDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsFunnelStepDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsGrowthDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsGuardrailsDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsMonetizationDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsNorthStarDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsPeriodDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminAnalyticsRateDto
|
||||
import com.osglab.account.features.admin.stats.models.AdminProductAnalyticsDto
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsWindow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminProductAnalyticsRepository
|
||||
import java.math.BigDecimal
|
||||
import java.math.RoundingMode
|
||||
import java.time.DayOfWeek
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
import java.time.temporal.TemporalAdjusters
|
||||
|
||||
class AdminProductAnalyticsService(
|
||||
private val repository: AdminProductAnalyticsRepository,
|
||||
) {
|
||||
suspend fun get(from: Instant, until: Instant): AdminProductAnalyticsDto {
|
||||
require(from < until)
|
||||
val currentWeek = currentWeekWindow(until)
|
||||
val elapsed = Duration.between(currentWeek.from, currentWeek.until)
|
||||
val previousWeek = AdminAnalyticsWindow(
|
||||
from = currentWeek.from.minus(Duration.ofDays(7)),
|
||||
until = currentWeek.from.minus(Duration.ofDays(7)).plus(elapsed),
|
||||
)
|
||||
val snapshot = repository.load(
|
||||
range = AdminAnalyticsWindow(from, until),
|
||||
currentWeek = currentWeek,
|
||||
previousWeek = previousWeek,
|
||||
)
|
||||
val referrals = snapshot.referrals
|
||||
val growth = snapshot.growthFunnel
|
||||
return AdminProductAnalyticsDto(
|
||||
period = AdminAnalyticsPeriodDto(from.toString(), until.toString()),
|
||||
northStar = AdminAnalyticsNorthStarDto(
|
||||
weeklyAiActiveUsers = snapshot.currentWeeklyUsers,
|
||||
previousWeeklyAiActiveUsers = snapshot.previousWeeklyUsers,
|
||||
weekOverWeekPercent = growthPercent(
|
||||
snapshot.currentWeeklyUsers,
|
||||
snapshot.previousWeeklyUsers,
|
||||
),
|
||||
),
|
||||
growth = AdminAnalyticsGrowthDto(
|
||||
newInstallations = snapshot.newInstallations,
|
||||
newAccounts = snapshot.newAccounts,
|
||||
activation24h = snapshot.activation24h.toRate(),
|
||||
medianTimeToValueMinutes = snapshot.medianTimeToValueMinutes?.rounded(),
|
||||
channels = snapshot.channels.map {
|
||||
AdminAnalyticsChannelDto(
|
||||
channel = it.channel,
|
||||
installations = it.installations,
|
||||
activated = it.activated,
|
||||
activationRate = AdminAnalyticsCountRow(
|
||||
it.activated,
|
||||
it.installations,
|
||||
).toRate(),
|
||||
)
|
||||
},
|
||||
),
|
||||
activity = AdminAnalyticsActivityDto(
|
||||
dau = snapshot.dau,
|
||||
wau = snapshot.wau,
|
||||
mau = snapshot.mau,
|
||||
stickinessPercent = percentage(snapshot.dau, snapshot.mau),
|
||||
successfulAiRequests = snapshot.successfulAiRequests,
|
||||
successfulRequestsPerActiveUser = ratio(
|
||||
snapshot.successfulAiRequests,
|
||||
snapshot.periodActiveUsers,
|
||||
),
|
||||
),
|
||||
consumption = AdminAnalyticsConsumptionDto(
|
||||
totalCredits = snapshot.consumption.totalCredits,
|
||||
averageDailyCreditsPerActiveUser =
|
||||
snapshot.consumption.averageDailyCreditsPerActiveUser?.rounded(),
|
||||
medianUserDailyCredits = snapshot.consumption.medianUserDailyCredits?.rounded(),
|
||||
averageCreditsPerManagedRequest = ratio(
|
||||
snapshot.consumption.totalCredits,
|
||||
snapshot.consumption.managedRequests,
|
||||
),
|
||||
),
|
||||
monetization = AdminAnalyticsMonetizationDto(
|
||||
payingUsers = snapshot.monetization.payingUsers,
|
||||
purchases = snapshot.monetization.purchases,
|
||||
creditsPurchased = snapshot.monetization.creditsPurchased,
|
||||
conversion7d = snapshot.monetization.conversion7d.toRate(),
|
||||
conversion30d = snapshot.monetization.conversion30d.toRate(),
|
||||
repeatPurchaseRate = snapshot.monetization.repeatPurchase.toRate(),
|
||||
),
|
||||
growthFunnel = listOf(
|
||||
AdminAnalyticsFunnelStepDto("首次启动", growth.opened),
|
||||
AdminAnalyticsFunnelStepDto("完成注册", growth.registered),
|
||||
AdminAnalyticsFunnelStepDto("24 小时内首次 AI 成功", growth.activated),
|
||||
AdminAnalyticsFunnelStepDto("D7 再次使用 AI", growth.retainedD7),
|
||||
AdminAnalyticsFunnelStepDto("首次购买", growth.purchased),
|
||||
),
|
||||
retention = snapshot.retention.map { cohort ->
|
||||
AdminAnalyticsCohortDto(
|
||||
cohortDate = cohort.cohortDate.toString(),
|
||||
size = cohort.size,
|
||||
d1 = cohort.takeIf { isMature(it.cohortDate.atStartOfDay().toInstant(ZoneOffset.UTC), 1, until) }
|
||||
?.let { AdminAnalyticsCountRow(it.d1, it.size).toRate() },
|
||||
d7 = cohort.takeIf { isMature(it.cohortDate.atStartOfDay().toInstant(ZoneOffset.UTC), 7, until) }
|
||||
?.let { AdminAnalyticsCountRow(it.d7, it.size).toRate() },
|
||||
d30 = cohort.takeIf {
|
||||
isMature(it.cohortDate.atStartOfDay().toInstant(ZoneOffset.UTC), 30, until)
|
||||
}?.let { AdminAnalyticsCountRow(it.d30, it.size).toRate() },
|
||||
)
|
||||
},
|
||||
aiFeatures = snapshot.features.map {
|
||||
AdminAnalyticsFeatureUsageDto(
|
||||
feature = it.feature,
|
||||
executionMode = it.executionMode,
|
||||
users = it.users,
|
||||
successes = it.successes,
|
||||
)
|
||||
},
|
||||
referralFunnel = listOf(
|
||||
AdminAnalyticsFunnelStepDto("发起分享", referrals.shared),
|
||||
AdminAnalyticsFunnelStepDto("打开邀请", referrals.opened),
|
||||
AdminAnalyticsFunnelStepDto("完成绑定", referrals.bound),
|
||||
AdminAnalyticsFunnelStepDto("首次 AI 成功", referrals.activated),
|
||||
AdminAnalyticsFunnelStepDto("完成奖励", referrals.rewarded),
|
||||
),
|
||||
guardrails = AdminAnalyticsGuardrailsDto(
|
||||
clientAiSuccessRate = snapshot.guardrails.clientSuccess.toRate(),
|
||||
managedSuccessRate = snapshot.guardrails.managedSuccess.toRate(),
|
||||
creditBlockedUsers = snapshot.guardrails.creditBlockedUsers,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun currentWeekWindow(until: Instant): AdminAnalyticsWindow {
|
||||
val referenceDate = until.minusNanos(1).atZone(ZoneOffset.UTC).toLocalDate()
|
||||
val weekStart = referenceDate
|
||||
.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY))
|
||||
.atStartOfDay(ZoneOffset.UTC)
|
||||
.toInstant()
|
||||
return AdminAnalyticsWindow(weekStart, until)
|
||||
}
|
||||
|
||||
private fun AdminAnalyticsCountRow.toRate(): AdminAnalyticsRateDto =
|
||||
AdminAnalyticsRateDto(
|
||||
numerator = numerator,
|
||||
denominator = denominator,
|
||||
percent = percentage(numerator, denominator),
|
||||
)
|
||||
|
||||
private fun percentage(numerator: Long, denominator: Long): Double? =
|
||||
if (denominator == 0L) null else
|
||||
numerator.toBigDecimal()
|
||||
.multiply(HUNDRED)
|
||||
.divide(denominator.toBigDecimal(), 1, RoundingMode.HALF_UP)
|
||||
.toDouble()
|
||||
|
||||
private fun ratio(numerator: Long, denominator: Long): Double? =
|
||||
if (denominator == 0L) null else
|
||||
numerator.toBigDecimal()
|
||||
.divide(denominator.toBigDecimal(), 2, RoundingMode.HALF_UP)
|
||||
.toDouble()
|
||||
|
||||
private fun growthPercent(current: Long, previous: Long): Double? =
|
||||
if (previous == 0L) null else
|
||||
(current - previous).toBigDecimal()
|
||||
.multiply(HUNDRED)
|
||||
.divide(previous.toBigDecimal(), 1, RoundingMode.HALF_UP)
|
||||
.toDouble()
|
||||
|
||||
private fun Double.rounded(scale: Int = 1): Double =
|
||||
toBigDecimal().setScale(scale, RoundingMode.HALF_UP).toDouble()
|
||||
|
||||
private fun isMature(cohortStart: Instant, offsetDays: Long, until: Instant): Boolean =
|
||||
!until.isBefore(cohortStart.plus(Duration.ofDays(offsetDays + 1)))
|
||||
|
||||
private val HUNDRED = BigDecimal.valueOf(100)
|
||||
@@ -0,0 +1,113 @@
|
||||
package com.osglab.account.features.analytics.domain
|
||||
|
||||
import com.osglab.account.common.errors.ApiException
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import java.time.Instant
|
||||
import java.util.UUID
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsEventType {
|
||||
FIRST_OPEN,
|
||||
SESSION_STARTED,
|
||||
KEYBOARD_ACTIVATED,
|
||||
AI_FEATURE_STARTED,
|
||||
AI_FEATURE_SUCCEEDED,
|
||||
AI_FEATURE_FAILED,
|
||||
PURCHASE_VIEWED,
|
||||
PURCHASE_STARTED,
|
||||
PURCHASE_CANCELLED,
|
||||
REFERRAL_SHARED,
|
||||
INVITE_OPENED,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsSurface {
|
||||
APP,
|
||||
KEYBOARD,
|
||||
INVITE_WEB,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsAcquisitionChannel {
|
||||
APP_STORE_ORGANIC,
|
||||
REFERRAL,
|
||||
SOCIAL_CONTENT,
|
||||
UNKNOWN,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsFeature {
|
||||
TRANSCRIPTION,
|
||||
POLISH,
|
||||
AI_ASSISTANT,
|
||||
AGENT,
|
||||
HOTWORD,
|
||||
OTHER,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsExecutionMode {
|
||||
MANAGED,
|
||||
LOCAL,
|
||||
BYOK,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsFailureCategory {
|
||||
NETWORK,
|
||||
PROVIDER,
|
||||
TIMEOUT,
|
||||
CANCELLED,
|
||||
INSUFFICIENT_CREDITS,
|
||||
VALIDATION,
|
||||
UNKNOWN,
|
||||
}
|
||||
|
||||
@Serializable
|
||||
enum class AnalyticsDurationBucket {
|
||||
LT_1S,
|
||||
S1_TO_3,
|
||||
S3_TO_10,
|
||||
S10_TO_30,
|
||||
GTE_30S,
|
||||
}
|
||||
|
||||
data class AnalyticsEvent(
|
||||
val clientEventId: UUID,
|
||||
val eventType: AnalyticsEventType,
|
||||
val occurredAt: Instant,
|
||||
val surface: AnalyticsSurface,
|
||||
val acquisitionChannel: AnalyticsAcquisitionChannel?,
|
||||
val feature: AnalyticsFeature?,
|
||||
val executionMode: AnalyticsExecutionMode?,
|
||||
val failureCategory: AnalyticsFailureCategory?,
|
||||
val durationBucket: AnalyticsDurationBucket?,
|
||||
val appVersion: String?,
|
||||
val osVersion: String?,
|
||||
val payloadHash: String,
|
||||
) {
|
||||
override fun toString(): String = "AnalyticsEvent([REDACTED])"
|
||||
}
|
||||
|
||||
data class AnalyticsBatch(
|
||||
val installationHash: String,
|
||||
val accountId: UUID?,
|
||||
val events: List<AnalyticsEvent>,
|
||||
val receivedAt: Instant,
|
||||
) {
|
||||
override fun toString(): String =
|
||||
"AnalyticsBatch(installationHash=[REDACTED], accountId=[REDACTED], events=${events.size})"
|
||||
}
|
||||
|
||||
data class AnalyticsIngestResult(
|
||||
val accepted: Int,
|
||||
val replayed: Int,
|
||||
)
|
||||
|
||||
class AnalyticsEventTimeException :
|
||||
ApiException(
|
||||
status = HttpStatusCode.UnprocessableEntity,
|
||||
code = "event_time_invalid",
|
||||
message = "An event timestamp is outside the accepted range",
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.osglab.account.features.analytics.models
|
||||
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsAcquisitionChannel
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsDurationBucket
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventType
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsExecutionMode
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFailureCategory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFeature
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsSurface
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class AnalyticsBatchRequest(
|
||||
val installationId: String,
|
||||
val events: List<AnalyticsEventRequest>,
|
||||
) {
|
||||
override fun toString(): String =
|
||||
"AnalyticsBatchRequest(installationId=[REDACTED], events=[REDACTED size=${events.size}])"
|
||||
}
|
||||
|
||||
@Serializable
|
||||
data class AnalyticsEventRequest(
|
||||
val clientEventId: String,
|
||||
val eventType: AnalyticsEventType,
|
||||
val occurredAt: String,
|
||||
val surface: AnalyticsSurface,
|
||||
val acquisitionChannel: AnalyticsAcquisitionChannel? = null,
|
||||
val feature: AnalyticsFeature? = null,
|
||||
val executionMode: AnalyticsExecutionMode? = null,
|
||||
val failureCategory: AnalyticsFailureCategory? = null,
|
||||
val durationBucket: AnalyticsDurationBucket? = null,
|
||||
val appVersion: String? = null,
|
||||
val osVersion: String? = null,
|
||||
) {
|
||||
override fun toString(): String = "AnalyticsEventRequest([REDACTED])"
|
||||
}
|
||||
|
||||
@Serializable
|
||||
data class AnalyticsIngestResponse(
|
||||
val accepted: Int,
|
||||
val replayed: Int,
|
||||
) {
|
||||
companion object {
|
||||
fun fromDomain(result: AnalyticsIngestResult): AnalyticsIngestResponse =
|
||||
AnalyticsIngestResponse(result.accepted, result.replayed)
|
||||
}
|
||||
}
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
package com.osglab.account.features.analytics.repositories
|
||||
|
||||
import com.osglab.account.common.errors.ConflictException
|
||||
import com.osglab.account.config.DatabaseFactory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsAcquisitionChannel
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsBatch
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsDurationBucket
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEvent
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventType
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsExecutionMode
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFailureCategory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFeature
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsSurface
|
||||
import org.jetbrains.exposed.v1.core.Table
|
||||
import org.jetbrains.exposed.v1.core.and
|
||||
import org.jetbrains.exposed.v1.core.eq
|
||||
import org.jetbrains.exposed.v1.core.inList
|
||||
import org.jetbrains.exposed.v1.core.isNull
|
||||
import org.jetbrains.exposed.v1.core.less
|
||||
import org.jetbrains.exposed.v1.core.plus
|
||||
import org.jetbrains.exposed.v1.javatime.date
|
||||
import org.jetbrains.exposed.v1.javatime.timestamp
|
||||
import org.jetbrains.exposed.v1.jdbc.insert
|
||||
import org.jetbrains.exposed.v1.jdbc.insertIgnore
|
||||
import org.jetbrains.exposed.v1.jdbc.deleteWhere
|
||||
import org.jetbrains.exposed.v1.jdbc.selectAll
|
||||
import org.jetbrains.exposed.v1.jdbc.update
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
interface AnalyticsRepository {
|
||||
suspend fun ingest(batch: AnalyticsBatch): AnalyticsIngestResult
|
||||
suspend fun recordInvitePageOpen(occurredAt: Instant)
|
||||
suspend fun purgeAnonymousInstallations(before: Instant, limit: Int): Int
|
||||
}
|
||||
|
||||
class ExposedAnalyticsRepository(
|
||||
private val databaseFactory: DatabaseFactory,
|
||||
) : AnalyticsRepository {
|
||||
override suspend fun purgeAnonymousInstallations(before: Instant, limit: Int): Int {
|
||||
require(limit in 1..10_000)
|
||||
return databaseFactory.query {
|
||||
val hashes = AnalyticsInstallations
|
||||
.selectAll()
|
||||
.where {
|
||||
AnalyticsInstallations.accountId.isNull() and
|
||||
(AnalyticsInstallations.updatedAt less before)
|
||||
}
|
||||
.limit(limit)
|
||||
.map { it[AnalyticsInstallations.installationHash] }
|
||||
if (hashes.isEmpty()) {
|
||||
0
|
||||
} else {
|
||||
AnalyticsInstallations.deleteWhere {
|
||||
AnalyticsInstallations.installationHash inList hashes
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun recordInvitePageOpen(occurredAt: Instant) {
|
||||
databaseFactory.query {
|
||||
val date = occurredAt.atZone(ZoneOffset.UTC).toLocalDate()
|
||||
AnalyticsDailyCounters.insertIgnore {
|
||||
it[counterDate] = date
|
||||
it[counterName] = INVITE_PAGE_OPENED
|
||||
it[counterValue] = 0
|
||||
it[updatedAt] = occurredAt
|
||||
}
|
||||
AnalyticsDailyCounters.update({
|
||||
(AnalyticsDailyCounters.counterDate eq date) and
|
||||
(AnalyticsDailyCounters.counterName eq INVITE_PAGE_OPENED)
|
||||
}) {
|
||||
it[counterValue] = AnalyticsDailyCounters.counterValue + 1
|
||||
it[updatedAt] = occurredAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun ingest(batch: AnalyticsBatch): AnalyticsIngestResult =
|
||||
databaseFactory.query {
|
||||
AnalyticsInstallations.insertIgnore {
|
||||
it[installationHash] = batch.installationHash
|
||||
it[accountId] = batch.accountId?.toString()
|
||||
it[createdAt] = batch.receivedAt
|
||||
it[updatedAt] = batch.receivedAt
|
||||
}
|
||||
|
||||
val installation = AnalyticsInstallations
|
||||
.selectAll()
|
||||
.where { AnalyticsInstallations.installationHash eq batch.installationHash }
|
||||
.forUpdate()
|
||||
.single()
|
||||
val linkedAccount = installation[AnalyticsInstallations.accountId]
|
||||
when {
|
||||
batch.accountId == null -> Unit
|
||||
linkedAccount == null -> AnalyticsInstallations.update({
|
||||
AnalyticsInstallations.installationHash eq batch.installationHash
|
||||
}) {
|
||||
it[accountId] = batch.accountId.toString()
|
||||
it[updatedAt] = batch.receivedAt
|
||||
}
|
||||
linkedAccount != batch.accountId.toString() ->
|
||||
throw ConflictException("Installation is linked to another account")
|
||||
}
|
||||
AnalyticsInstallations.update({
|
||||
AnalyticsInstallations.installationHash eq batch.installationHash
|
||||
}) {
|
||||
it[updatedAt] = batch.receivedAt
|
||||
}
|
||||
|
||||
var accepted = 0
|
||||
var replayed = 0
|
||||
batch.events.forEach { event ->
|
||||
val existingPayloadHash = AnalyticsEvents
|
||||
.selectAll()
|
||||
.where {
|
||||
(AnalyticsEvents.installationHash eq batch.installationHash) and
|
||||
(AnalyticsEvents.clientEventId eq event.clientEventId.toString())
|
||||
}
|
||||
.singleOrNull()
|
||||
?.get(AnalyticsEvents.payloadHash)
|
||||
when {
|
||||
existingPayloadHash == null -> {
|
||||
insertEvent(batch, event)
|
||||
accepted += 1
|
||||
}
|
||||
existingPayloadHash == event.payloadHash -> replayed += 1
|
||||
else -> throw ConflictException("Client event ID was reused with another payload")
|
||||
}
|
||||
}
|
||||
AnalyticsIngestResult(accepted = accepted, replayed = replayed)
|
||||
}
|
||||
|
||||
private fun insertEvent(batch: AnalyticsBatch, event: AnalyticsEvent) {
|
||||
AnalyticsEvents.insert {
|
||||
it[installationHash] = batch.installationHash
|
||||
it[clientEventId] = event.clientEventId.toString()
|
||||
it[eventType] = event.eventType
|
||||
it[occurredAt] = event.occurredAt
|
||||
it[surface] = event.surface
|
||||
it[acquisitionChannel] = event.acquisitionChannel
|
||||
it[feature] = event.feature
|
||||
it[executionMode] = event.executionMode
|
||||
it[failureCategory] = event.failureCategory
|
||||
it[durationBucket] = event.durationBucket
|
||||
it[appVersion] = event.appVersion
|
||||
it[osVersion] = event.osVersion
|
||||
it[payloadHash] = event.payloadHash
|
||||
it[receivedAt] = batch.receivedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private object AnalyticsInstallations : Table("product_analytics_installations") {
|
||||
val installationHash = char("installation_hash", 64)
|
||||
val accountId = varchar("account_id", 36).nullable()
|
||||
val createdAt = timestamp("created_at")
|
||||
val updatedAt = timestamp("updated_at")
|
||||
|
||||
override val primaryKey = PrimaryKey(installationHash)
|
||||
}
|
||||
|
||||
private object AnalyticsEvents : Table("product_analytics_events") {
|
||||
val installationHash = char("installation_hash", 64)
|
||||
val clientEventId = char("client_event_id", 36)
|
||||
val eventType = enumerationByName<AnalyticsEventType>("event_name", 32)
|
||||
val occurredAt = timestamp("occurred_at")
|
||||
val surface = enumerationByName<AnalyticsSurface>("surface", 16)
|
||||
val acquisitionChannel =
|
||||
enumerationByName<AnalyticsAcquisitionChannel>("acquisition_channel", 32).nullable()
|
||||
val feature = enumerationByName<AnalyticsFeature>("feature", 32).nullable()
|
||||
val executionMode =
|
||||
enumerationByName<AnalyticsExecutionMode>("execution_mode", 16).nullable()
|
||||
val failureCategory =
|
||||
enumerationByName<AnalyticsFailureCategory>("failure_category", 32).nullable()
|
||||
val durationBucket =
|
||||
enumerationByName<AnalyticsDurationBucket>("duration_bucket", 16).nullable()
|
||||
val appVersion = varchar("app_version", 32).nullable()
|
||||
val osVersion = varchar("os_version", 32).nullable()
|
||||
val payloadHash = char("payload_hash", 64)
|
||||
val receivedAt = timestamp("received_at")
|
||||
|
||||
override val primaryKey = PrimaryKey(installationHash, clientEventId)
|
||||
}
|
||||
|
||||
private object AnalyticsDailyCounters : Table("product_analytics_daily_counters") {
|
||||
val counterDate = date("counter_date")
|
||||
val counterName = varchar("counter_name", 32)
|
||||
val counterValue = long("counter_value")
|
||||
val updatedAt = timestamp("updated_at")
|
||||
|
||||
override val primaryKey = PrimaryKey(counterDate, counterName)
|
||||
}
|
||||
|
||||
private const val INVITE_PAGE_OPENED = "INVITE_PAGE_OPENED"
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.osglab.account.features.analytics.routes
|
||||
|
||||
import com.osglab.account.common.security.AccountPrincipal
|
||||
import com.osglab.account.common.security.SESSION_AUTH_NAME
|
||||
import com.osglab.account.common.errors.InvalidRequestException
|
||||
import com.osglab.account.features.analytics.models.AnalyticsBatchRequest
|
||||
import com.osglab.account.features.analytics.models.AnalyticsIngestResponse
|
||||
import com.osglab.account.features.analytics.services.AnalyticsService
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.http.HttpHeaders
|
||||
import io.ktor.server.auth.authenticate
|
||||
import io.ktor.server.auth.principal
|
||||
import io.ktor.server.request.header
|
||||
import io.ktor.server.request.receiveText
|
||||
import io.ktor.server.response.respond
|
||||
import io.ktor.server.routing.Route
|
||||
import io.ktor.server.routing.post
|
||||
import kotlinx.serialization.SerializationException
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
fun Route.analyticsRoutes(service: AnalyticsService) {
|
||||
authenticate(SESSION_AUTH_NAME, optional = true) {
|
||||
post("/v1/analytics/events") {
|
||||
val declaredLength = call.request.header(HttpHeaders.ContentLength)?.toLongOrNull()
|
||||
if (declaredLength != null && declaredLength > MAX_ANALYTICS_BODY_BYTES) {
|
||||
throw InvalidRequestException("Analytics request body is too large")
|
||||
}
|
||||
val body = call.receiveText()
|
||||
if (body.toByteArray(Charsets.UTF_8).size > MAX_ANALYTICS_BODY_BYTES) {
|
||||
throw InvalidRequestException("Analytics request body is too large")
|
||||
}
|
||||
val request = try {
|
||||
ANALYTICS_JSON.decodeFromString<AnalyticsBatchRequest>(body)
|
||||
} catch (_: SerializationException) {
|
||||
throw InvalidRequestException("Analytics request body is invalid")
|
||||
}
|
||||
val result = service.ingest(
|
||||
accountId = call.principal<AccountPrincipal>()?.userId,
|
||||
request = request,
|
||||
)
|
||||
call.respond(HttpStatusCode.OK, AnalyticsIngestResponse.fromDomain(result))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val ANALYTICS_JSON = Json {
|
||||
ignoreUnknownKeys = false
|
||||
explicitNulls = false
|
||||
}
|
||||
private const val MAX_ANALYTICS_BODY_BYTES = 64 * 1024
|
||||
@@ -0,0 +1,247 @@
|
||||
package com.osglab.account.features.analytics.services
|
||||
|
||||
import com.osglab.account.common.errors.InvalidRequestException
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsAcquisitionChannel
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsBatch
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEvent
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventTimeException
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventType
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFailureCategory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsSurface
|
||||
import com.osglab.account.features.analytics.models.AnalyticsBatchRequest
|
||||
import com.osglab.account.features.analytics.models.AnalyticsEventRequest
|
||||
import com.osglab.account.features.analytics.repositories.AnalyticsRepository
|
||||
import java.security.MessageDigest
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.format.DateTimeParseException
|
||||
import java.util.UUID
|
||||
|
||||
interface AnalyticsService {
|
||||
suspend fun ingest(
|
||||
accountId: UUID?,
|
||||
request: AnalyticsBatchRequest,
|
||||
): AnalyticsIngestResult
|
||||
}
|
||||
|
||||
class DefaultAnalyticsService(
|
||||
private val repository: AnalyticsRepository,
|
||||
private val clock: Clock = Clock.systemUTC(),
|
||||
) : AnalyticsService {
|
||||
override suspend fun ingest(
|
||||
accountId: UUID?,
|
||||
request: AnalyticsBatchRequest,
|
||||
): AnalyticsIngestResult {
|
||||
if (request.events.size !in MIN_BATCH_SIZE..MAX_BATCH_SIZE) {
|
||||
throw InvalidRequestException("events must contain between 1 and 50 items")
|
||||
}
|
||||
val installationId = parseUuid(request.installationId, "installationId")
|
||||
val now = clock.instant()
|
||||
val events = request.events.map { validateAndMap(it, now) }
|
||||
return repository.ingest(
|
||||
AnalyticsBatch(
|
||||
installationHash = installationId.toString().sha256Hex(),
|
||||
accountId = accountId,
|
||||
events = events,
|
||||
receivedAt = now,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private fun validateAndMap(request: AnalyticsEventRequest, now: Instant): AnalyticsEvent {
|
||||
val clientEventId = parseUuid(request.clientEventId, "clientEventId")
|
||||
val occurredAt = parseOccurredAt(request.occurredAt)
|
||||
if (
|
||||
occurredAt.isBefore(now.minus(MAX_EVENT_AGE)) ||
|
||||
occurredAt.isAfter(now.plus(MAX_FUTURE_SKEW))
|
||||
) {
|
||||
throw AnalyticsEventTimeException()
|
||||
}
|
||||
validateReleaseIdentifier(request.appVersion, "appVersion")
|
||||
validateReleaseIdentifier(request.osVersion, "osVersion")
|
||||
validateEventShape(request)
|
||||
|
||||
return AnalyticsEvent(
|
||||
clientEventId = clientEventId,
|
||||
eventType = request.eventType,
|
||||
occurredAt = occurredAt,
|
||||
surface = request.surface,
|
||||
acquisitionChannel = request.acquisitionChannel,
|
||||
feature = request.feature,
|
||||
executionMode = request.executionMode,
|
||||
failureCategory = request.failureCategory,
|
||||
durationBucket = request.durationBucket,
|
||||
appVersion = request.appVersion,
|
||||
osVersion = request.osVersion,
|
||||
payloadHash = payloadHash(request, clientEventId, occurredAt),
|
||||
)
|
||||
}
|
||||
|
||||
private fun validateEventShape(event: AnalyticsEventRequest) {
|
||||
val valid = when (event.eventType) {
|
||||
AnalyticsEventType.FIRST_OPEN ->
|
||||
event.surface == AnalyticsSurface.APP &&
|
||||
event.acquisitionChannel != null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.SESSION_STARTED ->
|
||||
event.surface in setOf(AnalyticsSurface.APP, AnalyticsSurface.KEYBOARD) &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.KEYBOARD_ACTIVATED ->
|
||||
event.surface == AnalyticsSurface.KEYBOARD &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.AI_FEATURE_STARTED ->
|
||||
event.feature != null &&
|
||||
event.executionMode != null &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.AI_FEATURE_SUCCEEDED ->
|
||||
event.feature != null &&
|
||||
event.executionMode != null &&
|
||||
event.durationBucket != null &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.failureCategory == null
|
||||
|
||||
AnalyticsEventType.AI_FEATURE_FAILED ->
|
||||
event.feature != null &&
|
||||
event.executionMode != null &&
|
||||
event.failureCategory != null &&
|
||||
event.acquisitionChannel == null
|
||||
|
||||
AnalyticsEventType.PURCHASE_VIEWED,
|
||||
AnalyticsEventType.PURCHASE_STARTED,
|
||||
->
|
||||
event.surface == AnalyticsSurface.APP &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.PURCHASE_CANCELLED ->
|
||||
event.surface == AnalyticsSurface.APP &&
|
||||
event.failureCategory == AnalyticsFailureCategory.CANCELLED &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.REFERRAL_SHARED ->
|
||||
event.surface == AnalyticsSurface.APP &&
|
||||
event.acquisitionChannel == null &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
|
||||
AnalyticsEventType.INVITE_OPENED ->
|
||||
event.surface == AnalyticsSurface.INVITE_WEB &&
|
||||
event.acquisitionChannel == AnalyticsAcquisitionChannel.REFERRAL &&
|
||||
event.feature == null &&
|
||||
event.executionMode == null &&
|
||||
event.failureCategory == null &&
|
||||
event.durationBucket == null
|
||||
}
|
||||
if (!valid) {
|
||||
throw InvalidRequestException("Event fields do not match the event type")
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseOccurredAt(value: String): Instant {
|
||||
if (!value.endsWith('Z')) {
|
||||
throw InvalidRequestException("occurredAt must be a UTC ISO-8601 timestamp")
|
||||
}
|
||||
return try {
|
||||
Instant.parse(value)
|
||||
} catch (_: DateTimeParseException) {
|
||||
throw InvalidRequestException("occurredAt must be a UTC ISO-8601 timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
private fun validateReleaseIdentifier(value: String?, field: String) {
|
||||
if (value != null && !RELEASE_IDENTIFIER.matches(value)) {
|
||||
throw InvalidRequestException("$field must be a 1 to 32 character release identifier")
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseUuid(value: String, field: String): UUID {
|
||||
if (!UUID_PATTERN.matches(value)) {
|
||||
throw InvalidRequestException("$field must be a UUID")
|
||||
}
|
||||
return try {
|
||||
UUID.fromString(value)
|
||||
} catch (_: IllegalArgumentException) {
|
||||
throw InvalidRequestException("$field must be a UUID")
|
||||
}
|
||||
}
|
||||
|
||||
private fun payloadHash(
|
||||
event: AnalyticsEventRequest,
|
||||
clientEventId: UUID,
|
||||
occurredAt: Instant,
|
||||
): String = listOf(
|
||||
clientEventId.toString(),
|
||||
event.eventType.name,
|
||||
occurredAt.toString(),
|
||||
event.surface.name,
|
||||
event.acquisitionChannel?.name,
|
||||
event.feature?.name,
|
||||
event.executionMode?.name,
|
||||
event.failureCategory?.name,
|
||||
event.durationBucket?.name,
|
||||
event.appVersion,
|
||||
event.osVersion,
|
||||
).joinToString(separator = "\u0000") { it ?: "" }.sha256Hex()
|
||||
|
||||
private fun String.sha256Hex(): String =
|
||||
MessageDigest.getInstance("SHA-256")
|
||||
.digest(toByteArray(Charsets.UTF_8))
|
||||
.joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and 0xff) }
|
||||
|
||||
private companion object {
|
||||
const val MIN_BATCH_SIZE = 1
|
||||
const val MAX_BATCH_SIZE = 50
|
||||
val MAX_EVENT_AGE: Duration = Duration.ofDays(35)
|
||||
val MAX_FUTURE_SKEW: Duration = Duration.ofMinutes(5)
|
||||
val UUID_PATTERN =
|
||||
Regex("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
val RELEASE_IDENTIFIER = Regex("[A-Za-z0-9._+\\-]{1,32}")
|
||||
}
|
||||
}
|
||||
|
||||
class AnalyticsMaintenanceService(
|
||||
private val repository: AnalyticsRepository,
|
||||
private val clock: Clock = Clock.systemUTC(),
|
||||
private val anonymousRetention: Duration = Duration.ofDays(90),
|
||||
) {
|
||||
init {
|
||||
require(!anonymousRetention.isNegative && !anonymousRetention.isZero)
|
||||
}
|
||||
|
||||
suspend fun purgeStaleAnonymousInstallations(): Int =
|
||||
repository.purgeAnonymousInstallations(
|
||||
before = clock.instant().minus(anonymousRetention),
|
||||
limit = PURGE_BATCH_SIZE,
|
||||
)
|
||||
|
||||
private companion object {
|
||||
const val PURGE_BATCH_SIZE = 1_000
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,10 @@ fun interface ReferralLookupPort {
|
||||
suspend fun isValid(code: String): Boolean
|
||||
}
|
||||
|
||||
fun interface InviteOpenRecorder {
|
||||
suspend fun record()
|
||||
}
|
||||
|
||||
data class InviteWebConfig(
|
||||
val appStoreUrl: String,
|
||||
val appleAppId: String,
|
||||
@@ -75,6 +79,7 @@ sealed interface InvitePageResult {
|
||||
class InvitePageService(
|
||||
private val referralLookup: ReferralLookupPort,
|
||||
private val config: InviteWebConfig,
|
||||
private val inviteOpenRecorder: InviteOpenRecorder = InviteOpenRecorder {},
|
||||
) {
|
||||
private val appStoreUrl = validateHttpsUrl(config.appStoreUrl, "APP_STORE_URL", allowQuery = true)
|
||||
.toASCIIString()
|
||||
@@ -101,6 +106,13 @@ class InvitePageService(
|
||||
return InvitePageResult.TemporarilyUnavailable
|
||||
}
|
||||
if (!valid) return InvitePageResult.Invalid
|
||||
try {
|
||||
inviteOpenRecorder.record()
|
||||
} catch (exception: CancellationException) {
|
||||
throw exception
|
||||
} catch (_: Exception) {
|
||||
// Analytics must never make a valid invitation unavailable.
|
||||
}
|
||||
|
||||
val nonce = createNonce()
|
||||
val universalLink = "$universalLinkBaseUrl/$validCode".escapeHtml()
|
||||
@@ -129,6 +141,7 @@ fun Route.configureInviteWebRoutes() {
|
||||
}
|
||||
configureInviteWebRoutes(
|
||||
referralLookup = koin.get<ReferralLookupPort>(),
|
||||
inviteOpenRecorder = koin.get<InviteOpenRecorder>(),
|
||||
config = InviteWebConfig(
|
||||
appStoreUrl = appConfig.appStoreUrl,
|
||||
appleAppId = "$teamId.${appConfig.apple.clientId}",
|
||||
@@ -140,8 +153,9 @@ fun Route.configureInviteWebRoutes() {
|
||||
fun Route.configureInviteWebRoutes(
|
||||
referralLookup: ReferralLookupPort,
|
||||
config: InviteWebConfig,
|
||||
inviteOpenRecorder: InviteOpenRecorder = InviteOpenRecorder {},
|
||||
) {
|
||||
val service = InvitePageService(referralLookup, config)
|
||||
val service = InvitePageService(referralLookup, config, inviteOpenRecorder)
|
||||
|
||||
get("/i/{code}") {
|
||||
when (val result = service.render(call.parameters["code"])) {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
CREATE TABLE product_analytics_installations (
|
||||
installation_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
account_id VARCHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
created_at DATETIME(6) NOT NULL,
|
||||
updated_at DATETIME(6) NOT NULL,
|
||||
PRIMARY KEY (installation_hash),
|
||||
INDEX ix_product_analytics_installations_account (account_id),
|
||||
CONSTRAINT fk_product_analytics_installations_account
|
||||
FOREIGN KEY (account_id) REFERENCES accounts (id) ON DELETE CASCADE
|
||||
) ENGINE = InnoDB;
|
||||
|
||||
CREATE TABLE product_analytics_events (
|
||||
installation_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
client_event_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
event_name VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
occurred_at DATETIME(6) NOT NULL,
|
||||
surface VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
acquisition_channel VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
feature VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
execution_mode VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
failure_category VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
duration_bucket VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
app_version VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
os_version VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL,
|
||||
payload_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
received_at DATETIME(6) NOT NULL,
|
||||
PRIMARY KEY (installation_hash, client_event_id),
|
||||
INDEX ix_product_analytics_events_type_occurred (event_name, occurred_at),
|
||||
INDEX ix_product_analytics_events_occurred (occurred_at),
|
||||
CONSTRAINT fk_product_analytics_events_installation
|
||||
FOREIGN KEY (installation_hash)
|
||||
REFERENCES product_analytics_installations (installation_hash)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT chk_product_analytics_events_type CHECK (
|
||||
event_name IN (
|
||||
'FIRST_OPEN',
|
||||
'SESSION_STARTED',
|
||||
'KEYBOARD_ACTIVATED',
|
||||
'AI_FEATURE_STARTED',
|
||||
'AI_FEATURE_SUCCEEDED',
|
||||
'AI_FEATURE_FAILED',
|
||||
'PURCHASE_VIEWED',
|
||||
'PURCHASE_STARTED',
|
||||
'PURCHASE_CANCELLED',
|
||||
'REFERRAL_SHARED',
|
||||
'INVITE_OPENED'
|
||||
)
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_surface CHECK (
|
||||
surface IN ('APP', 'KEYBOARD', 'INVITE_WEB')
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_acquisition CHECK (
|
||||
acquisition_channel IS NULL
|
||||
OR acquisition_channel IN (
|
||||
'APP_STORE_ORGANIC',
|
||||
'REFERRAL',
|
||||
'SOCIAL_CONTENT',
|
||||
'UNKNOWN'
|
||||
)
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_feature CHECK (
|
||||
feature IS NULL
|
||||
OR feature IN (
|
||||
'TRANSCRIPTION',
|
||||
'POLISH',
|
||||
'AI_ASSISTANT',
|
||||
'AGENT',
|
||||
'HOTWORD',
|
||||
'OTHER'
|
||||
)
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_execution CHECK (
|
||||
execution_mode IS NULL OR execution_mode IN ('MANAGED', 'LOCAL', 'BYOK')
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_failure CHECK (
|
||||
failure_category IS NULL
|
||||
OR failure_category IN (
|
||||
'NETWORK',
|
||||
'PROVIDER',
|
||||
'TIMEOUT',
|
||||
'CANCELLED',
|
||||
'INSUFFICIENT_CREDITS',
|
||||
'VALIDATION',
|
||||
'UNKNOWN'
|
||||
)
|
||||
),
|
||||
CONSTRAINT chk_product_analytics_events_duration CHECK (
|
||||
duration_bucket IS NULL
|
||||
OR duration_bucket IN ('LT_1S', 'S1_TO_3', 'S3_TO_10', 'S10_TO_30', 'GTE_30S')
|
||||
)
|
||||
) ENGINE = InnoDB;
|
||||
|
||||
-- First-party invitation page views are counted without cookies, IP addresses,
|
||||
-- user agents, installation IDs, or referral codes.
|
||||
CREATE TABLE product_analytics_daily_counters (
|
||||
counter_date DATE NOT NULL,
|
||||
counter_name VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
counter_value BIGINT UNSIGNED NOT NULL DEFAULT 0,
|
||||
updated_at DATETIME(6) NOT NULL,
|
||||
PRIMARY KEY (counter_date, counter_name),
|
||||
CONSTRAINT chk_product_analytics_daily_counter_name
|
||||
CHECK (counter_name IN ('INVITE_PAGE_OPENED'))
|
||||
) ENGINE = InnoDB;
|
||||
@@ -123,6 +123,29 @@ class DeploymentConsistencyTest : FunSpec({
|
||||
}
|
||||
}
|
||||
|
||||
test("product analytics contract stays allowlisted and account deletions cascade") {
|
||||
val migration = root.read(
|
||||
"src/main/resources/db/migration/V16__product_analytics_events.sql",
|
||||
)
|
||||
migration shouldContain "product_analytics_installations"
|
||||
migration shouldContain "product_analytics_events"
|
||||
migration shouldContain "REFERENCES accounts (id) ON DELETE CASCADE"
|
||||
migration shouldContain "PRIMARY KEY (installation_hash, client_event_id)"
|
||||
|
||||
val openApi = root.read("docs/openapi.yaml")
|
||||
val eventSchema = openApi
|
||||
.substringAfter(" ProductAnalyticsEvent:")
|
||||
.substringBefore(" AdminSessionState:")
|
||||
eventSchema shouldContain "additionalProperties: false"
|
||||
eventSchema shouldContain "AI_FEATURE_SUCCEEDED"
|
||||
eventSchema shouldContain "INSUFFICIENT_CREDITS"
|
||||
eventSchema shouldNotContain "prompt"
|
||||
eventSchema shouldNotContain "transcript"
|
||||
eventSchema shouldNotContain "audio"
|
||||
eventSchema shouldNotContain "modelOutput"
|
||||
openApi shouldContain "schema: { \$ref: \"#/components/schemas/AdminProductAnalytics\" }"
|
||||
}
|
||||
|
||||
test("production Compose reuses private MySQL and hardens the application container") {
|
||||
val compose = root.read("compose.yaml")
|
||||
|
||||
@@ -228,6 +251,7 @@ private val EXPECTED_PUBLIC_PATHS = setOf(
|
||||
"/v1/auth/logout",
|
||||
"/v1/account",
|
||||
"/v1/apple/events",
|
||||
"/v1/analytics/events",
|
||||
"/v1/credits/balance",
|
||||
"/v1/credits/ledger",
|
||||
"/v1/credits/rates",
|
||||
@@ -255,6 +279,7 @@ private val EXPECTED_PUBLIC_PATHS = setOf(
|
||||
"/v1/admin/auth/logout",
|
||||
"/v1/admin/overview",
|
||||
"/v1/admin/referrals",
|
||||
"/v1/admin/analytics",
|
||||
"/v1/admin/users",
|
||||
"/v1/admin/users/{userId}",
|
||||
"/v1/admin/users/{userId}/ledger",
|
||||
|
||||
@@ -51,7 +51,7 @@ class SmokeDeploymentTest : FunSpec({
|
||||
|
||||
test("runtime grants cover every migrated table without mutable history privileges") {
|
||||
val grants = root.read("deploy/smoke/runtime-grants.sql")
|
||||
val migrationTables = (1..13)
|
||||
val migrationTables = (1..16)
|
||||
.flatMap { version ->
|
||||
val migration = Files.list(root.resolve("src/main/resources/db/migration")).use { paths ->
|
||||
paths.filter { it.fileName.toString().startsWith("V${version}__") }
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.osglab.account.features.admin.services.AdminOperatorService
|
||||
import com.osglab.account.features.admin.services.AdminOperatorErrorCode
|
||||
import com.osglab.account.features.admin.services.AdminOperatorException
|
||||
import com.osglab.account.features.admin.services.AdminSessionService
|
||||
import com.osglab.account.features.admin.stats.services.AdminProductAnalyticsService
|
||||
import com.osglab.account.features.admin.stats.services.AdminStatsService
|
||||
import com.osglab.account.features.admin.users.models.AdminUserLedgerEntryDto
|
||||
import com.osglab.account.features.admin.users.models.AdminUserLedgerPageDto
|
||||
@@ -315,6 +316,7 @@ private fun io.ktor.server.application.Application.installAdminTestRoutes(
|
||||
authService = authService,
|
||||
sessionService = sessionService,
|
||||
statsService = mockk<AdminStatsService>(relaxed = true),
|
||||
productAnalyticsService = mockk<AdminProductAnalyticsService>(relaxed = true),
|
||||
usersService = usersService,
|
||||
grantService = grantService,
|
||||
operatorService = operatorService,
|
||||
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package com.osglab.account.features.admin.stats
|
||||
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsChannelRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCohortRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsConsumptionRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsFeatureRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsGrowthFunnelRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsGuardrailRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsMonetizationRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsReferralRow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsWindow
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminProductAnalyticsRepository
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminProductAnalyticsSnapshot
|
||||
import com.osglab.account.features.admin.stats.services.AdminProductAnalyticsService
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.matchers.shouldBe
|
||||
import java.time.Instant
|
||||
import java.time.LocalDate
|
||||
|
||||
class AdminProductAnalyticsServiceTest : FunSpec({
|
||||
test("maps product metrics with explicit rates and mature retention windows") {
|
||||
val captured = mutableListOf<Triple<AdminAnalyticsWindow, AdminAnalyticsWindow, AdminAnalyticsWindow>>()
|
||||
val repository = object : AdminProductAnalyticsRepository {
|
||||
override suspend fun load(
|
||||
range: AdminAnalyticsWindow,
|
||||
currentWeek: AdminAnalyticsWindow,
|
||||
previousWeek: AdminAnalyticsWindow,
|
||||
): AdminProductAnalyticsSnapshot {
|
||||
captured += Triple(range, currentWeek, previousWeek)
|
||||
return snapshot()
|
||||
}
|
||||
}
|
||||
val service = AdminProductAnalyticsService(repository)
|
||||
val until = Instant.parse("2026-08-20T09:00:00Z")
|
||||
|
||||
val result = service.get(Instant.parse("2026-07-21T09:00:00Z"), until)
|
||||
|
||||
result.northStar.weeklyAiActiveUsers shouldBe 120
|
||||
result.northStar.weekOverWeekPercent shouldBe 20.0
|
||||
result.growth.activation24h.percent shouldBe 60.0
|
||||
result.activity.stickinessPercent shouldBe 20.0
|
||||
result.activity.successfulRequestsPerActiveUser shouldBe 5.0
|
||||
result.consumption.averageCreditsPerManagedRequest shouldBe 2.5
|
||||
result.retention.first().d1?.percent shouldBe 50.0
|
||||
result.retention.first().d7?.percent shouldBe 30.0
|
||||
result.retention.first().d30 shouldBe null
|
||||
result.growthFunnel.map { it.label } shouldBe listOf(
|
||||
"首次启动",
|
||||
"完成注册",
|
||||
"24 小时内首次 AI 成功",
|
||||
"D7 再次使用 AI",
|
||||
"首次购买",
|
||||
)
|
||||
captured.single().second.from shouldBe Instant.parse("2026-08-17T00:00:00Z")
|
||||
captured.single().third.from shouldBe Instant.parse("2026-08-10T00:00:00Z")
|
||||
}
|
||||
|
||||
test("zero denominators remain unavailable instead of reporting false zero rates") {
|
||||
val empty = snapshot().copy(
|
||||
previousWeeklyUsers = 0,
|
||||
mau = 0,
|
||||
periodActiveUsers = 0,
|
||||
activation24h = AdminAnalyticsCountRow(0, 0),
|
||||
consumption = AdminAnalyticsConsumptionRow(0, 0, null, null),
|
||||
)
|
||||
val service = AdminProductAnalyticsService(
|
||||
object : AdminProductAnalyticsRepository {
|
||||
override suspend fun load(
|
||||
range: AdminAnalyticsWindow,
|
||||
currentWeek: AdminAnalyticsWindow,
|
||||
previousWeek: AdminAnalyticsWindow,
|
||||
) = empty
|
||||
},
|
||||
)
|
||||
|
||||
val result = service.get(
|
||||
Instant.parse("2026-08-19T00:00:00Z"),
|
||||
Instant.parse("2026-08-20T00:00:00Z"),
|
||||
)
|
||||
|
||||
result.northStar.weekOverWeekPercent shouldBe null
|
||||
result.growth.activation24h.percent shouldBe null
|
||||
result.activity.stickinessPercent shouldBe null
|
||||
result.activity.successfulRequestsPerActiveUser shouldBe null
|
||||
result.consumption.averageCreditsPerManagedRequest shouldBe null
|
||||
}
|
||||
})
|
||||
|
||||
private fun snapshot(): AdminProductAnalyticsSnapshot =
|
||||
AdminProductAnalyticsSnapshot(
|
||||
currentWeeklyUsers = 120,
|
||||
previousWeeklyUsers = 100,
|
||||
newInstallations = 100,
|
||||
newAccounts = 80,
|
||||
activation24h = AdminAnalyticsCountRow(60, 100),
|
||||
medianTimeToValueMinutes = 7.6,
|
||||
channels = listOf(
|
||||
AdminAnalyticsChannelRow("APP_STORE_ORGANIC", 100, 60),
|
||||
),
|
||||
dau = 30,
|
||||
wau = 120,
|
||||
mau = 150,
|
||||
periodActiveUsers = 100,
|
||||
successfulAiRequests = 500,
|
||||
consumption = AdminAnalyticsConsumptionRow(
|
||||
totalCredits = 1_000,
|
||||
managedRequests = 400,
|
||||
averageDailyCreditsPerActiveUser = 12.25,
|
||||
medianUserDailyCredits = 8.0,
|
||||
),
|
||||
monetization = AdminAnalyticsMonetizationRow(
|
||||
payingUsers = 10,
|
||||
purchases = 12,
|
||||
creditsPurchased = 8_000,
|
||||
conversion7d = AdminAnalyticsCountRow(8, 70),
|
||||
conversion30d = AdminAnalyticsCountRow(10, 50),
|
||||
repeatPurchase = AdminAnalyticsCountRow(2, 10),
|
||||
),
|
||||
growthFunnel = AdminAnalyticsGrowthFunnelRow(100, 80, 60, 20, 10),
|
||||
retention = listOf(
|
||||
AdminAnalyticsCohortRow(
|
||||
cohortDate = LocalDate.parse("2026-08-01"),
|
||||
size = 20,
|
||||
d1 = 10,
|
||||
d7 = 6,
|
||||
d30 = 2,
|
||||
),
|
||||
),
|
||||
features = listOf(
|
||||
AdminAnalyticsFeatureRow("POLISH", "MANAGED", 30, 100),
|
||||
),
|
||||
referrals = AdminAnalyticsReferralRow(20, 15, 10, 8, 5),
|
||||
guardrails = AdminAnalyticsGuardrailRow(
|
||||
clientSuccess = AdminAnalyticsCountRow(90, 100),
|
||||
managedSuccess = AdminAnalyticsCountRow(95, 100),
|
||||
creditBlockedUsers = 3,
|
||||
),
|
||||
)
|
||||
+96
@@ -3,6 +3,8 @@ package com.osglab.account.features.admin.stats
|
||||
import com.osglab.account.config.DatabaseConfig
|
||||
import com.osglab.account.config.DatabaseFactory
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminStatsRange
|
||||
import com.osglab.account.features.admin.stats.repositories.AdminAnalyticsWindow
|
||||
import com.osglab.account.features.admin.stats.repositories.ExposedAdminProductAnalyticsRepository
|
||||
import com.osglab.account.features.admin.stats.repositories.ExposedAdminStatsRepository
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.matchers.collections.shouldBeEmpty
|
||||
@@ -11,6 +13,7 @@ import io.kotest.matchers.shouldBe
|
||||
import org.opentest4j.TestAbortedException
|
||||
import org.testcontainers.DockerClientFactory
|
||||
import org.testcontainers.containers.MySQLContainer
|
||||
import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager
|
||||
import java.time.Instant
|
||||
|
||||
class AdminStatsRepositoryIntegrationTest : FunSpec({
|
||||
@@ -48,6 +51,20 @@ class AdminStatsRepositoryIntegrationTest : FunSpec({
|
||||
until = Instant.parse("2026-08-17T12:00:00Z"),
|
||||
),
|
||||
)
|
||||
val analytics = ExposedAdminProductAnalyticsRepository(factory).load(
|
||||
range = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-10T12:00:00Z"),
|
||||
until = Instant.parse("2026-08-17T12:00:00Z"),
|
||||
),
|
||||
currentWeek = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-17T00:00:00Z"),
|
||||
until = Instant.parse("2026-08-17T12:00:00Z"),
|
||||
),
|
||||
previousWeek = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-10T00:00:00Z"),
|
||||
until = Instant.parse("2026-08-10T12:00:00Z"),
|
||||
),
|
||||
)
|
||||
|
||||
// A dedicated container starts empty, so every scalar and grouped aggregate is explicit.
|
||||
if (mysql != null) {
|
||||
@@ -59,6 +76,36 @@ class AdminStatsRepositoryIntegrationTest : FunSpec({
|
||||
snapshot.registrationsByDate shouldBe emptyMap()
|
||||
snapshot.referralRanking.shouldBeEmpty()
|
||||
snapshot.usage.shouldBeEmpty()
|
||||
analytics.currentWeeklyUsers shouldBeExactly 0
|
||||
analytics.newInstallations shouldBeExactly 0
|
||||
analytics.retention.shouldBeEmpty()
|
||||
analytics.features.shouldBeEmpty()
|
||||
analytics.consumption.totalCredits shouldBeExactly 0
|
||||
|
||||
factory.query { seedProductAnalytics() }
|
||||
val populated = ExposedAdminProductAnalyticsRepository(factory).load(
|
||||
range = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-10T00:00:00Z"),
|
||||
until = Instant.parse("2026-08-17T00:00:00Z"),
|
||||
),
|
||||
currentWeek = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-10T00:00:00Z"),
|
||||
until = Instant.parse("2026-08-17T00:00:00Z"),
|
||||
),
|
||||
previousWeek = AdminAnalyticsWindow(
|
||||
from = Instant.parse("2026-08-03T00:00:00Z"),
|
||||
until = Instant.parse("2026-08-10T00:00:00Z"),
|
||||
),
|
||||
)
|
||||
|
||||
populated.currentWeeklyUsers shouldBeExactly 1
|
||||
populated.newInstallations shouldBeExactly 1
|
||||
populated.activation24h shouldBe
|
||||
com.osglab.account.features.admin.stats.repositories.AdminAnalyticsCountRow(1, 1)
|
||||
populated.periodActiveUsers shouldBeExactly 1
|
||||
populated.successfulAiRequests shouldBeExactly 2
|
||||
populated.features.single().successes shouldBeExactly 2
|
||||
populated.retention.single().d1 shouldBeExactly 1
|
||||
}
|
||||
} finally {
|
||||
factory.close()
|
||||
@@ -69,3 +116,52 @@ class AdminStatsRepositoryIntegrationTest : FunSpec({
|
||||
|
||||
private class StatsMySqlContainer(image: String) :
|
||||
MySQLContainer<StatsMySqlContainer>(image)
|
||||
|
||||
private fun seedProductAnalytics() {
|
||||
TransactionManager.current().exec(
|
||||
"""
|
||||
INSERT INTO product_analytics_installations (
|
||||
installation_hash, account_id, created_at, updated_at
|
||||
) VALUES (
|
||||
'${"a".repeat(64)}', NULL,
|
||||
'2026-08-11 00:00:00.000000', '2026-08-12 00:10:00.000000'
|
||||
)
|
||||
""".trimIndent(),
|
||||
)
|
||||
listOf(
|
||||
"""
|
||||
(
|
||||
'${"a".repeat(64)}', '40000000-0000-0000-0000-000000000001',
|
||||
'FIRST_OPEN', '2026-08-11 00:00:00.000000', 'APP',
|
||||
'APP_STORE_ORGANIC', NULL, NULL, NULL, NULL,
|
||||
'1.0', '18.6', '${"1".repeat(64)}', '2026-08-11 00:00:01.000000'
|
||||
)
|
||||
""".trimIndent(),
|
||||
"""
|
||||
(
|
||||
'${"a".repeat(64)}', '40000000-0000-0000-0000-000000000002',
|
||||
'AI_FEATURE_SUCCEEDED', '2026-08-11 00:10:00.000000', 'KEYBOARD',
|
||||
NULL, 'POLISH', 'LOCAL', NULL, 'S1_TO_3',
|
||||
'1.0', '18.6', '${"2".repeat(64)}', '2026-08-11 00:10:01.000000'
|
||||
)
|
||||
""".trimIndent(),
|
||||
"""
|
||||
(
|
||||
'${"a".repeat(64)}', '40000000-0000-0000-0000-000000000003',
|
||||
'AI_FEATURE_SUCCEEDED', '2026-08-12 00:10:00.000000', 'KEYBOARD',
|
||||
NULL, 'POLISH', 'LOCAL', NULL, 'S1_TO_3',
|
||||
'1.0', '18.6', '${"3".repeat(64)}', '2026-08-12 00:10:01.000000'
|
||||
)
|
||||
""".trimIndent(),
|
||||
).forEach { values ->
|
||||
TransactionManager.current().exec(
|
||||
"""
|
||||
INSERT INTO product_analytics_events (
|
||||
installation_hash, client_event_id, event_name, occurred_at, surface,
|
||||
acquisition_channel, feature, execution_mode, failure_category,
|
||||
duration_bucket, app_version, os_version, payload_hash, received_at
|
||||
) VALUES $values
|
||||
""".trimIndent(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
package com.osglab.account.features.analytics
|
||||
|
||||
import com.osglab.account.common.errors.ConflictException
|
||||
import com.osglab.account.config.DatabaseConfig
|
||||
import com.osglab.account.config.DatabaseFactory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventType
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsSurface
|
||||
import com.osglab.account.features.analytics.models.AnalyticsBatchRequest
|
||||
import com.osglab.account.features.analytics.models.AnalyticsEventRequest
|
||||
import com.osglab.account.features.analytics.repositories.ExposedAnalyticsRepository
|
||||
import com.osglab.account.features.analytics.services.DefaultAnalyticsService
|
||||
import io.kotest.assertions.throwables.shouldThrow
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.matchers.shouldBe
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import java.security.MessageDigest
|
||||
import java.sql.DriverManager
|
||||
import java.time.Clock
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
import java.util.UUID
|
||||
import org.opentest4j.TestAbortedException
|
||||
import org.testcontainers.DockerClientFactory
|
||||
import org.testcontainers.containers.MySQLContainer
|
||||
|
||||
class AnalyticsRepositoryIntegrationTest : FunSpec({
|
||||
test("V16 repository links accounts replays atomically and cascades account deletion") {
|
||||
withAnalyticsDatabase { config, databaseFactory ->
|
||||
val now = Instant.parse("2026-08-20T01:00:00Z")
|
||||
val accountId = UUID.fromString("20000000-0000-0000-0000-000000000001")
|
||||
val otherAccountId = UUID.fromString("20000000-0000-0000-0000-000000000002")
|
||||
val installationId = "10000000-0000-0000-0000-000000000001"
|
||||
insertAccounts(config, listOf(accountId, otherAccountId), now)
|
||||
val repository = ExposedAnalyticsRepository(databaseFactory)
|
||||
val service = DefaultAnalyticsService(
|
||||
repository = repository,
|
||||
clock = Clock.fixed(now, ZoneOffset.UTC),
|
||||
)
|
||||
val original = event(
|
||||
id = "30000000-0000-0000-0000-000000000001",
|
||||
now = now,
|
||||
surface = AnalyticsSurface.APP,
|
||||
)
|
||||
val request = AnalyticsBatchRequest(installationId, listOf(original))
|
||||
|
||||
service.ingest(null, request) shouldBe AnalyticsIngestResult(1, 0)
|
||||
service.ingest(accountId, request) shouldBe AnalyticsIngestResult(0, 1)
|
||||
service.ingest(accountId, request) shouldBe AnalyticsIngestResult(0, 1)
|
||||
shouldThrow<ConflictException> {
|
||||
service.ingest(otherAccountId, request)
|
||||
}
|
||||
|
||||
installationCount(config, installationId) shouldBe 0
|
||||
installationCount(config, installationId.sha256Hex()) shouldBe 1
|
||||
linkedAccount(config, installationId.sha256Hex()) shouldBe accountId.toString()
|
||||
eventCount(config) shouldBe 1
|
||||
repository.recordInvitePageOpen(now)
|
||||
repository.recordInvitePageOpen(now.plusSeconds(30))
|
||||
scalarInt(
|
||||
config,
|
||||
"SELECT counter_value FROM product_analytics_daily_counters " +
|
||||
"WHERE counter_name = 'INVITE_PAGE_OPENED'",
|
||||
) shouldBe 2
|
||||
|
||||
val concurrentRequest = AnalyticsBatchRequest(
|
||||
installationId = "10000000-0000-0000-0000-000000000099",
|
||||
events = listOf(
|
||||
event(
|
||||
id = "30000000-0000-0000-0000-000000000099",
|
||||
now = now,
|
||||
surface = AnalyticsSurface.APP,
|
||||
),
|
||||
),
|
||||
)
|
||||
val concurrentResults = coroutineScope {
|
||||
List(8) {
|
||||
async { service.ingest(null, concurrentRequest) }
|
||||
}.awaitAll()
|
||||
}
|
||||
concurrentResults.sumOf(AnalyticsIngestResult::accepted) shouldBe 1
|
||||
concurrentResults.sumOf(AnalyticsIngestResult::replayed) shouldBe 7
|
||||
markInstallationUpdatedAt(
|
||||
config,
|
||||
concurrentRequest.installationId.sha256Hex(),
|
||||
now.minusSeconds(91L * 24 * 60 * 60),
|
||||
)
|
||||
repository.purgeAnonymousInstallations(
|
||||
before = now.minusSeconds(90L * 24 * 60 * 60),
|
||||
limit = 100,
|
||||
) shouldBe 1
|
||||
installationCount(config, concurrentRequest.installationId.sha256Hex()) shouldBe 0
|
||||
|
||||
shouldThrow<ConflictException> {
|
||||
service.ingest(
|
||||
accountId,
|
||||
AnalyticsBatchRequest(
|
||||
installationId = installationId,
|
||||
events = listOf(
|
||||
event(
|
||||
id = "30000000-0000-0000-0000-000000000002",
|
||||
now = now,
|
||||
surface = AnalyticsSurface.APP,
|
||||
),
|
||||
original.copy(surface = AnalyticsSurface.KEYBOARD),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
eventCount(config) shouldBe 1
|
||||
|
||||
deleteAccount(config, accountId)
|
||||
installationCount(config, installationId.sha256Hex()) shouldBe 0
|
||||
eventCount(config) shouldBe 0
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
private fun event(
|
||||
id: String,
|
||||
now: Instant,
|
||||
surface: AnalyticsSurface,
|
||||
) = AnalyticsEventRequest(
|
||||
clientEventId = id,
|
||||
eventType = AnalyticsEventType.SESSION_STARTED,
|
||||
occurredAt = now.toString(),
|
||||
surface = surface,
|
||||
)
|
||||
|
||||
private suspend fun withAnalyticsDatabase(
|
||||
block: suspend (DatabaseConfig, DatabaseFactory) -> Unit,
|
||||
) {
|
||||
val externalJdbcUrl = System.getenv("TEST_MYSQL_JDBC_URL")?.takeIf(String::isNotBlank)
|
||||
if (externalJdbcUrl == null && !DockerClientFactory.instance().isDockerAvailable) {
|
||||
throw TestAbortedException("Docker is unavailable; MySQL integration test skipped")
|
||||
}
|
||||
val mysql = if (externalJdbcUrl == null) {
|
||||
AnalyticsMySqlContainer("mysql:8.4")
|
||||
.withDatabaseName("osg_analytics_test")
|
||||
.withUsername("test")
|
||||
.withPassword("test")
|
||||
.also(AnalyticsMySqlContainer::start)
|
||||
} else {
|
||||
null
|
||||
}
|
||||
val config = DatabaseConfig(
|
||||
jdbcUrl = externalJdbcUrl ?: requireNotNull(mysql).jdbcUrl,
|
||||
username = System.getenv("TEST_MYSQL_USER")?.takeIf(String::isNotBlank)
|
||||
?: mysql?.username
|
||||
?: "root",
|
||||
password = System.getenv("TEST_MYSQL_PASSWORD") ?: mysql?.password ?: "",
|
||||
maximumPoolSize = 4,
|
||||
)
|
||||
val databaseFactory = DatabaseFactory(config)
|
||||
try {
|
||||
databaseFactory.database
|
||||
block(config, databaseFactory)
|
||||
} finally {
|
||||
databaseFactory.close()
|
||||
mysql?.stop()
|
||||
}
|
||||
}
|
||||
|
||||
private fun insertAccounts(
|
||||
config: DatabaseConfig,
|
||||
accountIds: List<UUID>,
|
||||
now: Instant,
|
||||
) {
|
||||
DriverManager.getConnection(config.jdbcUrl, config.username, config.password).use { connection ->
|
||||
connection.prepareStatement(
|
||||
"""
|
||||
INSERT INTO accounts (id, apple_sub, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""".trimIndent()
|
||||
).use { statement ->
|
||||
accountIds.forEach { accountId ->
|
||||
statement.setString(1, accountId.toString())
|
||||
statement.setString(2, "analytics-test-$accountId")
|
||||
statement.setTimestamp(3, java.sql.Timestamp.from(now))
|
||||
statement.setTimestamp(4, java.sql.Timestamp.from(now))
|
||||
statement.addBatch()
|
||||
}
|
||||
statement.executeBatch()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun installationCount(config: DatabaseConfig, hash: String): Int =
|
||||
scalarInt(
|
||||
config,
|
||||
"SELECT COUNT(*) FROM product_analytics_installations WHERE installation_hash = ?",
|
||||
hash,
|
||||
)
|
||||
|
||||
private fun eventCount(config: DatabaseConfig): Int =
|
||||
scalarInt(config, "SELECT COUNT(*) FROM product_analytics_events")
|
||||
|
||||
private fun linkedAccount(config: DatabaseConfig, hash: String): String? =
|
||||
DriverManager.getConnection(config.jdbcUrl, config.username, config.password).use { connection ->
|
||||
connection.prepareStatement(
|
||||
"SELECT account_id FROM product_analytics_installations WHERE installation_hash = ?"
|
||||
).use { statement ->
|
||||
statement.setString(1, hash)
|
||||
statement.executeQuery().use { result ->
|
||||
result.next()
|
||||
result.getString(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun scalarInt(config: DatabaseConfig, sql: String, argument: String? = null): Int =
|
||||
DriverManager.getConnection(config.jdbcUrl, config.username, config.password).use { connection ->
|
||||
connection.prepareStatement(sql).use { statement ->
|
||||
argument?.let { statement.setString(1, it) }
|
||||
statement.executeQuery().use { result ->
|
||||
result.next()
|
||||
result.getInt(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun deleteAccount(config: DatabaseConfig, accountId: UUID) {
|
||||
DriverManager.getConnection(config.jdbcUrl, config.username, config.password).use { connection ->
|
||||
connection.prepareStatement("DELETE FROM accounts WHERE id = ?").use { statement ->
|
||||
statement.setString(1, accountId.toString())
|
||||
statement.executeUpdate()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun markInstallationUpdatedAt(
|
||||
config: DatabaseConfig,
|
||||
installationHash: String,
|
||||
updatedAt: Instant,
|
||||
) {
|
||||
DriverManager.getConnection(config.jdbcUrl, config.username, config.password).use { connection ->
|
||||
connection.prepareStatement(
|
||||
"UPDATE product_analytics_installations SET updated_at = ? WHERE installation_hash = ?"
|
||||
).use { statement ->
|
||||
statement.setTimestamp(1, java.sql.Timestamp.from(updatedAt))
|
||||
statement.setString(2, installationHash)
|
||||
statement.executeUpdate() shouldBe 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun String.sha256Hex(): String =
|
||||
MessageDigest.getInstance("SHA-256")
|
||||
.digest(toByteArray(Charsets.UTF_8))
|
||||
.joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and 0xff) }
|
||||
|
||||
private class AnalyticsMySqlContainer(image: String) :
|
||||
MySQLContainer<AnalyticsMySqlContainer>(image)
|
||||
@@ -0,0 +1,153 @@
|
||||
package com.osglab.account.features.analytics
|
||||
|
||||
import com.osglab.account.common.api.installApiStatusPages
|
||||
import com.osglab.account.common.errors.ConflictException
|
||||
import com.osglab.account.common.security.AccountPrincipal
|
||||
import com.osglab.account.common.security.installSessionAuthentication
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventTimeException
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.models.AnalyticsBatchRequest
|
||||
import com.osglab.account.features.analytics.routes.analyticsRoutes
|
||||
import com.osglab.account.features.analytics.services.AnalyticsService
|
||||
import io.kotest.matchers.shouldBe
|
||||
import io.kotest.matchers.string.shouldContain
|
||||
import io.kotest.matchers.string.shouldNotContain
|
||||
import io.ktor.client.request.bearerAuth
|
||||
import io.ktor.client.request.post
|
||||
import io.ktor.client.request.setBody
|
||||
import io.ktor.client.statement.bodyAsText
|
||||
import io.ktor.http.ContentType
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.http.contentType
|
||||
import io.ktor.serialization.kotlinx.json.json
|
||||
import io.ktor.server.application.install
|
||||
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
|
||||
import io.ktor.server.routing.routing
|
||||
import io.ktor.server.testing.testApplication
|
||||
import java.util.UUID
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlin.test.Test
|
||||
|
||||
class AnalyticsRoutesTest {
|
||||
private val accountId = UUID.fromString("20000000-0000-0000-0000-000000000001")
|
||||
private val sessionId = UUID.fromString("30000000-0000-0000-0000-000000000001")
|
||||
|
||||
@Test
|
||||
fun `route accepts anonymous and authenticated batches without returning identity`() =
|
||||
testApplication {
|
||||
val service = RecordingAnalyticsService()
|
||||
application {
|
||||
install(ContentNegotiation) { json(Json { explicitNulls = false }) }
|
||||
installApiStatusPages()
|
||||
installSessionAuthentication { token ->
|
||||
token.takeIf { it == "valid-token" }?.let {
|
||||
AccountPrincipal(accountId, sessionId)
|
||||
}
|
||||
}
|
||||
routing { analyticsRoutes(service) }
|
||||
}
|
||||
|
||||
val anonymous = client.post("/v1/analytics/events") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody())
|
||||
}
|
||||
val authenticated = client.post("/v1/analytics/events") {
|
||||
bearerAuth("valid-token")
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody())
|
||||
}
|
||||
val invalidBearer = client.post("/v1/analytics/events") {
|
||||
bearerAuth("invalid-token")
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody())
|
||||
}
|
||||
|
||||
anonymous.status shouldBe HttpStatusCode.OK
|
||||
anonymous.bodyAsText() shouldBe """{"accepted":1,"replayed":0}"""
|
||||
authenticated.status shouldBe HttpStatusCode.OK
|
||||
authenticated.bodyAsText() shouldNotContain accountId.toString()
|
||||
authenticated.bodyAsText() shouldNotContain "installationId"
|
||||
invalidBearer.status shouldBe HttpStatusCode.Unauthorized
|
||||
service.accountIds shouldBe listOf(null, accountId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `route returns stable validation conflict and event-time errors`() = testApplication {
|
||||
val service = ErrorAnalyticsService()
|
||||
application {
|
||||
install(ContentNegotiation) { json(Json { explicitNulls = false }) }
|
||||
installApiStatusPages()
|
||||
installSessionAuthentication { null }
|
||||
routing { analyticsRoutes(service) }
|
||||
}
|
||||
|
||||
val invalidEnum = client.post("/v1/analytics/events") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody().replace("SESSION_STARTED", "ARBITRARY_EVENT"))
|
||||
}
|
||||
val unknownField = client.post("/v1/analytics/events") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody().replace("\"appVersion\"", "\"userText\":\"forbidden\",\"appVersion\""))
|
||||
}
|
||||
val conflict = client.post("/v1/analytics/events") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody().replace(INSTALLATION_ID, CONFLICT_INSTALLATION_ID))
|
||||
}
|
||||
val invalidTime = client.post("/v1/analytics/events") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(validBody().replace(INSTALLATION_ID, INVALID_TIME_INSTALLATION_ID))
|
||||
}
|
||||
|
||||
invalidEnum.status shouldBe HttpStatusCode.BadRequest
|
||||
invalidEnum.bodyAsText() shouldContain """"code":"invalid_request""""
|
||||
unknownField.status shouldBe HttpStatusCode.BadRequest
|
||||
unknownField.bodyAsText() shouldContain """"code":"invalid_request""""
|
||||
conflict.status shouldBe HttpStatusCode.Conflict
|
||||
conflict.bodyAsText() shouldContain """"code":"conflict""""
|
||||
invalidTime.status shouldBe HttpStatusCode.UnprocessableEntity
|
||||
invalidTime.bodyAsText() shouldContain """"code":"event_time_invalid""""
|
||||
}
|
||||
|
||||
private fun validBody(): String =
|
||||
"""
|
||||
{
|
||||
"installationId":"$INSTALLATION_ID",
|
||||
"events":[{
|
||||
"clientEventId":"40000000-0000-0000-0000-000000000001",
|
||||
"eventType":"SESSION_STARTED",
|
||||
"occurredAt":"2026-08-20T01:00:00Z",
|
||||
"surface":"APP",
|
||||
"appVersion":"1.0"
|
||||
}]
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private companion object {
|
||||
const val INSTALLATION_ID = "10000000-0000-0000-0000-000000000001"
|
||||
const val CONFLICT_INSTALLATION_ID = "10000000-0000-0000-0000-000000000002"
|
||||
const val INVALID_TIME_INSTALLATION_ID = "10000000-0000-0000-0000-000000000003"
|
||||
}
|
||||
}
|
||||
|
||||
private class RecordingAnalyticsService : AnalyticsService {
|
||||
val accountIds = mutableListOf<UUID?>()
|
||||
|
||||
override suspend fun ingest(
|
||||
accountId: UUID?,
|
||||
request: AnalyticsBatchRequest,
|
||||
): AnalyticsIngestResult {
|
||||
accountIds += accountId
|
||||
return AnalyticsIngestResult(accepted = request.events.size, replayed = 0)
|
||||
}
|
||||
}
|
||||
|
||||
private class ErrorAnalyticsService : AnalyticsService {
|
||||
override suspend fun ingest(
|
||||
accountId: UUID?,
|
||||
request: AnalyticsBatchRequest,
|
||||
): AnalyticsIngestResult = when (request.installationId) {
|
||||
"10000000-0000-0000-0000-000000000002" -> throw ConflictException("Conflict")
|
||||
"10000000-0000-0000-0000-000000000003" -> throw AnalyticsEventTimeException()
|
||||
else -> AnalyticsIngestResult(request.events.size, 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
package com.osglab.account.features.analytics
|
||||
|
||||
import com.osglab.account.common.errors.ConflictException
|
||||
import com.osglab.account.common.errors.InvalidRequestException
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsAcquisitionChannel
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsBatch
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsDurationBucket
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventType
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsEventTimeException
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsExecutionMode
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFailureCategory
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsFeature
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsIngestResult
|
||||
import com.osglab.account.features.analytics.domain.AnalyticsSurface
|
||||
import com.osglab.account.features.analytics.models.AnalyticsBatchRequest
|
||||
import com.osglab.account.features.analytics.models.AnalyticsEventRequest
|
||||
import com.osglab.account.features.analytics.repositories.AnalyticsRepository
|
||||
import com.osglab.account.features.analytics.services.AnalyticsMaintenanceService
|
||||
import com.osglab.account.features.analytics.services.DefaultAnalyticsService
|
||||
import io.kotest.assertions.throwables.shouldThrow
|
||||
import io.kotest.matchers.shouldBe
|
||||
import io.kotest.matchers.string.shouldNotContain
|
||||
import java.security.MessageDigest
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
import java.util.UUID
|
||||
import kotlin.test.Test
|
||||
|
||||
class AnalyticsServiceTest {
|
||||
private val now = Instant.parse("2026-08-20T01:00:00Z")
|
||||
private val installationId = "10000000-0000-0000-0000-000000000001"
|
||||
private val accountId = UUID.fromString("20000000-0000-0000-0000-000000000001")
|
||||
private val otherAccountId = UUID.fromString("20000000-0000-0000-0000-000000000002")
|
||||
|
||||
@Test
|
||||
fun `anonymous events are accepted and only the installation digest reaches persistence`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val request = batch(firstOpen())
|
||||
val result = service(repository).ingest(null, request)
|
||||
|
||||
result shouldBe AnalyticsIngestResult(accepted = 1, replayed = 0)
|
||||
repository.lastBatch?.accountId shouldBe null
|
||||
repository.lastBatch?.installationHash shouldBe installationId.sha256Hex()
|
||||
repository.lastBatch.toString() shouldNotContain installationId
|
||||
request.toString() shouldNotContain installationId
|
||||
request.toString() shouldNotContain firstOpen().clientEventId
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `authenticated ingestion links an anonymous installation and rejects another account`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val service = service(repository)
|
||||
val request = batch(sessionStarted())
|
||||
|
||||
service.ingest(null, request) shouldBe AnalyticsIngestResult(1, 0)
|
||||
service.ingest(accountId, request) shouldBe AnalyticsIngestResult(0, 1)
|
||||
service.ingest(accountId, request) shouldBe AnalyticsIngestResult(0, 1)
|
||||
shouldThrow<ConflictException> {
|
||||
service.ingest(otherAccountId, request)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `complete event catalog accepts only its declared dimensions`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val validEvents = listOf(
|
||||
firstOpen(),
|
||||
sessionStarted(id = uuid(2)),
|
||||
event(
|
||||
id = uuid(3),
|
||||
type = AnalyticsEventType.KEYBOARD_ACTIVATED,
|
||||
surface = AnalyticsSurface.KEYBOARD,
|
||||
),
|
||||
event(
|
||||
id = uuid(4),
|
||||
type = AnalyticsEventType.AI_FEATURE_STARTED,
|
||||
feature = AnalyticsFeature.POLISH,
|
||||
executionMode = AnalyticsExecutionMode.LOCAL,
|
||||
),
|
||||
event(
|
||||
id = uuid(5),
|
||||
type = AnalyticsEventType.AI_FEATURE_SUCCEEDED,
|
||||
feature = AnalyticsFeature.AI_ASSISTANT,
|
||||
executionMode = AnalyticsExecutionMode.BYOK,
|
||||
durationBucket = AnalyticsDurationBucket.S1_TO_3,
|
||||
),
|
||||
event(
|
||||
id = uuid(6),
|
||||
type = AnalyticsEventType.AI_FEATURE_FAILED,
|
||||
feature = AnalyticsFeature.TRANSCRIPTION,
|
||||
executionMode = AnalyticsExecutionMode.MANAGED,
|
||||
failureCategory = AnalyticsFailureCategory.TIMEOUT,
|
||||
durationBucket = AnalyticsDurationBucket.S10_TO_30,
|
||||
),
|
||||
event(id = uuid(7), type = AnalyticsEventType.PURCHASE_VIEWED),
|
||||
event(id = uuid(8), type = AnalyticsEventType.PURCHASE_STARTED),
|
||||
event(
|
||||
id = uuid(9),
|
||||
type = AnalyticsEventType.PURCHASE_CANCELLED,
|
||||
failureCategory = AnalyticsFailureCategory.CANCELLED,
|
||||
),
|
||||
event(id = uuid(10), type = AnalyticsEventType.REFERRAL_SHARED),
|
||||
event(
|
||||
id = uuid(11),
|
||||
type = AnalyticsEventType.INVITE_OPENED,
|
||||
surface = AnalyticsSurface.INVITE_WEB,
|
||||
acquisitionChannel = AnalyticsAcquisitionChannel.REFERRAL,
|
||||
),
|
||||
)
|
||||
|
||||
service(repository).ingest(null, batch(events = validEvents)) shouldBe
|
||||
AnalyticsIngestResult(validEvents.size, 0)
|
||||
|
||||
listOf(
|
||||
firstOpen().copy(acquisitionChannel = null),
|
||||
sessionStarted().copy(surface = AnalyticsSurface.INVITE_WEB),
|
||||
event(
|
||||
type = AnalyticsEventType.KEYBOARD_ACTIVATED,
|
||||
surface = AnalyticsSurface.APP,
|
||||
),
|
||||
event(
|
||||
type = AnalyticsEventType.AI_FEATURE_STARTED,
|
||||
feature = AnalyticsFeature.POLISH,
|
||||
executionMode = AnalyticsExecutionMode.LOCAL,
|
||||
durationBucket = AnalyticsDurationBucket.LT_1S,
|
||||
),
|
||||
event(
|
||||
type = AnalyticsEventType.AI_FEATURE_SUCCEEDED,
|
||||
feature = AnalyticsFeature.POLISH,
|
||||
executionMode = AnalyticsExecutionMode.LOCAL,
|
||||
),
|
||||
event(
|
||||
type = AnalyticsEventType.AI_FEATURE_FAILED,
|
||||
feature = AnalyticsFeature.POLISH,
|
||||
executionMode = AnalyticsExecutionMode.LOCAL,
|
||||
),
|
||||
event(
|
||||
type = AnalyticsEventType.PURCHASE_CANCELLED,
|
||||
failureCategory = AnalyticsFailureCategory.NETWORK,
|
||||
),
|
||||
event(
|
||||
type = AnalyticsEventType.INVITE_OPENED,
|
||||
surface = AnalyticsSurface.INVITE_WEB,
|
||||
acquisitionChannel = AnalyticsAcquisitionChannel.UNKNOWN,
|
||||
),
|
||||
).forEach { invalidEvent ->
|
||||
shouldThrow<InvalidRequestException> {
|
||||
service(repository).ingest(null, batch(invalidEvent))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `batch UUID release identifier and timestamp validation use stable errors`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val service = service(repository)
|
||||
|
||||
listOf(
|
||||
AnalyticsBatchRequest("not-a-uuid", listOf(sessionStarted())),
|
||||
batch(sessionStarted().copy(clientEventId = "not-a-uuid")),
|
||||
batch(sessionStarted().copy(appVersion = "")),
|
||||
batch(sessionStarted().copy(appVersion = "1.0 beta")),
|
||||
batch(sessionStarted().copy(osVersion = "版本")),
|
||||
batch(sessionStarted().copy(osVersion = "x".repeat(33))),
|
||||
batch(sessionStarted().copy(occurredAt = "2026-08-20T01:00:00+01:00")),
|
||||
AnalyticsBatchRequest(installationId, emptyList()),
|
||||
AnalyticsBatchRequest(installationId, List(51) { sessionStarted() }),
|
||||
).forEach { invalid ->
|
||||
shouldThrow<InvalidRequestException> {
|
||||
service.ingest(null, invalid)
|
||||
}.code shouldBe "invalid_request"
|
||||
}
|
||||
repository.eventCount shouldBe 0
|
||||
|
||||
listOf(
|
||||
now.minusSeconds(35L * 24 * 60 * 60 + 1),
|
||||
now.plusSeconds(5L * 60 + 1),
|
||||
).forEach { outside ->
|
||||
shouldThrow<AnalyticsEventTimeException> {
|
||||
service.ingest(null, batch(sessionStarted().copy(occurredAt = outside.toString())))
|
||||
}.code shouldBe "event_time_invalid"
|
||||
}
|
||||
|
||||
val maximumBatch = List(50) { index -> sessionStarted(uuid(100 + index)) }
|
||||
service.ingest(null, batch(maximumBatch)) shouldBe AnalyticsIngestResult(50, 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `time boundaries are inclusive and batch persistence is atomic on conflict`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val service = service(repository)
|
||||
val oldest = sessionStarted(uuid(20)).copy(
|
||||
occurredAt = now.minusSeconds(35L * 24 * 60 * 60).toString()
|
||||
)
|
||||
val newest = sessionStarted(uuid(21)).copy(occurredAt = now.plusSeconds(5 * 60).toString())
|
||||
|
||||
service.ingest(null, batch(events = listOf(oldest, newest))) shouldBe
|
||||
AnalyticsIngestResult(2, 0)
|
||||
|
||||
val original = sessionStarted(uuid(30))
|
||||
service.ingest(null, batch(original)) shouldBe AnalyticsIngestResult(1, 0)
|
||||
service.ingest(null, batch(original)) shouldBe AnalyticsIngestResult(0, 1)
|
||||
val eventCountBeforeConflict = repository.eventCount
|
||||
shouldThrow<ConflictException> {
|
||||
service.ingest(
|
||||
null,
|
||||
batch(
|
||||
events = listOf(
|
||||
sessionStarted(uuid(31)),
|
||||
original.copy(surface = AnalyticsSurface.KEYBOARD),
|
||||
)
|
||||
),
|
||||
)
|
||||
}
|
||||
repository.eventCount shouldBe eventCountBeforeConflict
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `maintenance purges only anonymous installations older than ninety days`(): Unit =
|
||||
kotlinx.coroutines.runBlocking {
|
||||
val repository = InMemoryAnalyticsRepository()
|
||||
val maintenance = AnalyticsMaintenanceService(
|
||||
repository = repository,
|
||||
clock = Clock.fixed(now, ZoneOffset.UTC),
|
||||
anonymousRetention = Duration.ofDays(90),
|
||||
)
|
||||
|
||||
maintenance.purgeStaleAnonymousInstallations() shouldBe 0
|
||||
repository.lastPurgeBefore shouldBe now.minus(Duration.ofDays(90))
|
||||
repository.lastPurgeLimit shouldBe 1_000
|
||||
}
|
||||
|
||||
private fun service(repository: AnalyticsRepository) =
|
||||
DefaultAnalyticsService(repository, Clock.fixed(now, ZoneOffset.UTC))
|
||||
|
||||
private fun batch(
|
||||
event: AnalyticsEventRequest,
|
||||
) = batch(events = listOf(event))
|
||||
|
||||
private fun batch(
|
||||
events: List<AnalyticsEventRequest>,
|
||||
) = AnalyticsBatchRequest(installationId = installationId, events = events)
|
||||
|
||||
private fun firstOpen() = event(
|
||||
id = uuid(1),
|
||||
type = AnalyticsEventType.FIRST_OPEN,
|
||||
acquisitionChannel = AnalyticsAcquisitionChannel.APP_STORE_ORGANIC,
|
||||
appVersion = "1.2.3",
|
||||
osVersion = "18.6",
|
||||
)
|
||||
|
||||
private fun sessionStarted(id: String = uuid(12)) = event(
|
||||
id = id,
|
||||
type = AnalyticsEventType.SESSION_STARTED,
|
||||
)
|
||||
|
||||
private fun event(
|
||||
id: String = uuid(40),
|
||||
type: AnalyticsEventType,
|
||||
surface: AnalyticsSurface = AnalyticsSurface.APP,
|
||||
acquisitionChannel: AnalyticsAcquisitionChannel? = null,
|
||||
feature: AnalyticsFeature? = null,
|
||||
executionMode: AnalyticsExecutionMode? = null,
|
||||
failureCategory: AnalyticsFailureCategory? = null,
|
||||
durationBucket: AnalyticsDurationBucket? = null,
|
||||
appVersion: String? = null,
|
||||
osVersion: String? = null,
|
||||
) = AnalyticsEventRequest(
|
||||
clientEventId = id,
|
||||
eventType = type,
|
||||
occurredAt = now.toString(),
|
||||
surface = surface,
|
||||
acquisitionChannel = acquisitionChannel,
|
||||
feature = feature,
|
||||
executionMode = executionMode,
|
||||
failureCategory = failureCategory,
|
||||
durationBucket = durationBucket,
|
||||
appVersion = appVersion,
|
||||
osVersion = osVersion,
|
||||
)
|
||||
|
||||
private fun uuid(number: Int): String =
|
||||
"30000000-0000-0000-0000-${number.toString().padStart(12, '0')}"
|
||||
}
|
||||
|
||||
private class InMemoryAnalyticsRepository : AnalyticsRepository {
|
||||
private val linkedAccounts = mutableMapOf<String, UUID?>()
|
||||
private val payloads = mutableMapOf<Pair<String, UUID>, String>()
|
||||
var lastBatch: AnalyticsBatch? = null
|
||||
private set
|
||||
val eventCount: Int get() = payloads.size
|
||||
var lastPurgeBefore: Instant? = null
|
||||
private set
|
||||
var lastPurgeLimit: Int? = null
|
||||
private set
|
||||
|
||||
override suspend fun recordInvitePageOpen(occurredAt: Instant) = Unit
|
||||
override suspend fun purgeAnonymousInstallations(before: Instant, limit: Int): Int {
|
||||
lastPurgeBefore = before
|
||||
lastPurgeLimit = limit
|
||||
return 0
|
||||
}
|
||||
|
||||
override suspend fun ingest(batch: AnalyticsBatch): AnalyticsIngestResult {
|
||||
val accountsCopy = linkedAccounts.toMutableMap()
|
||||
val payloadsCopy = payloads.toMutableMap()
|
||||
val existingAccount = accountsCopy[batch.installationHash]
|
||||
if (batch.installationHash !in accountsCopy) {
|
||||
accountsCopy[batch.installationHash] = batch.accountId
|
||||
} else if (batch.accountId != null) {
|
||||
when {
|
||||
existingAccount == null -> accountsCopy[batch.installationHash] = batch.accountId
|
||||
existingAccount != batch.accountId ->
|
||||
throw ConflictException("Installation is linked to another account")
|
||||
}
|
||||
}
|
||||
|
||||
var accepted = 0
|
||||
var replayed = 0
|
||||
batch.events.forEach { event ->
|
||||
val key = batch.installationHash to event.clientEventId
|
||||
val existingHash = payloadsCopy[key]
|
||||
when {
|
||||
existingHash == null -> {
|
||||
payloadsCopy[key] = event.payloadHash
|
||||
accepted += 1
|
||||
}
|
||||
existingHash == event.payloadHash -> replayed += 1
|
||||
else -> throw ConflictException("Client event ID was reused with another payload")
|
||||
}
|
||||
}
|
||||
linkedAccounts.clear()
|
||||
linkedAccounts.putAll(accountsCopy)
|
||||
payloads.clear()
|
||||
payloads.putAll(payloadsCopy)
|
||||
lastBatch = batch
|
||||
return AnalyticsIngestResult(accepted, replayed)
|
||||
}
|
||||
}
|
||||
|
||||
private fun String.sha256Hex(): String =
|
||||
MessageDigest.getInstance("SHA-256")
|
||||
.digest(toByteArray(Charsets.UTF_8))
|
||||
.joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and 0xff) }
|
||||
@@ -22,9 +22,14 @@ class InviteWebRoutesTest {
|
||||
|
||||
@Test
|
||||
fun `valid referral renders a bilingual first-party page with hardened headers`() = testApplication {
|
||||
var recordedOpens = 0
|
||||
application {
|
||||
routing {
|
||||
configureInviteWebRoutes(ReferralLookupPort { true }, config)
|
||||
configureInviteWebRoutes(
|
||||
ReferralLookupPort { true },
|
||||
config,
|
||||
InviteOpenRecorder { recordedOpens += 1 },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +54,7 @@ class InviteWebRoutesTest {
|
||||
body shouldNotContain "branch.io"
|
||||
body shouldNotContain "appsflyer"
|
||||
body shouldNotContain "adjust.com"
|
||||
recordedOpens shouldBe 1
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -108,6 +114,21 @@ class InviteWebRoutesTest {
|
||||
body shouldContain "temporarily unavailable"
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `analytics counter failure never makes a valid invitation unavailable`() = testApplication {
|
||||
application {
|
||||
routing {
|
||||
configureInviteWebRoutes(
|
||||
ReferralLookupPort { true },
|
||||
config,
|
||||
InviteOpenRecorder { error("analytics unavailable") },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
client.get("/i/$VALID_CODE").status shouldBe HttpStatusCode.OK
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `lookup timeout fails closed with retry guidance`() = testApplication {
|
||||
application {
|
||||
|
||||
Reference in New Issue
Block a user