Harden admin deployment and local acceptance

Enforce mTLS and least-privilege runtime boundaries while adding repeatable MySQL 8.4 and Docker smoke checks that require no production secrets.
This commit is contained in:
Rocky
2026-08-17 15:20:46 +08:00
parent 1a9c518f96
commit 405a2cfc0f
17 changed files with 1480 additions and 37 deletions
+564
View File
@@ -0,0 +1,564 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
COMPOSE_FILE="$ROOT_DIR/compose.smoke.yaml"
SMOKE_DIR="$ROOT_DIR/deploy/smoke"
WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/osg-account-smoke.XXXXXX")"
SECRET_DIR="$WORK_DIR/secrets"
DIAGNOSTICS_FILE="$WORK_DIR/diagnostics.log"
COMPOSE_ENV="$SECRET_DIR/compose.env"
RUNTIME_ENV="$SECRET_DIR/runtime.env"
ADMIN_ENV="$SECRET_DIR/admin.env"
ADMIN_GENERATED_ENV="$SECRET_DIR/admin.generated.env"
ADMIN_HANDOFF="$SECRET_DIR/admin-handoff.txt"
SUPER_COOKIE_JAR="$SECRET_DIR/super.cookies"
ANALYST_COOKIE_JAR="$SECRET_DIR/analyst.cookies"
RESPONSE_BODY="$WORK_DIR/response.json"
ACTUAL_PRIVILEGES="$WORK_DIR/runtime-privileges.tsv"
RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$"
PROJECT_NAME="osg-account-smoke-$RUN_ID"
SMOKE_ACCOUNT_ID="10000000-0000-0000-0000-000000000001"
FAILURES=0
mkdir -m 700 "$SECRET_DIR"
touch "$ADMIN_ENV"
chmod 600 "$ADMIN_ENV"
log() {
printf '[smoke] %s\n' "$*"
}
fail() {
printf '[smoke] FAIL: %s\n' "$*" >&2
return 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || {
printf '[smoke] missing required command: %s\n' "$1" >&2
exit 1
}
}
random_hex() {
openssl rand -hex "$1"
}
random_base64_key() {
openssl rand -base64 32 | tr -d '\n'
}
free_loopback_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
compose() {
docker compose \
--project-name "$PROJECT_NAME" \
--env-file "$COMPOSE_ENV" \
--file "$COMPOSE_FILE" \
"$@"
}
mysql_root() {
compose exec -T \
-e "MYSQL_PWD=$MYSQL_ROOT_PASSWORD" \
mysql mysql --protocol=tcp --host=127.0.0.1 --user=root "$@"
}
mysql_runtime() {
compose exec -T \
-e "MYSQL_PWD=$MYSQL_RUNTIME_PASSWORD" \
mysql mysql --protocol=tcp --host=127.0.0.1 --user=osg_smoke_runtime "$@"
}
capture_diagnostics() {
{
printf '%s\n' '=== compose ps ==='
compose --profile setup ps --all || true
printf '%s\n' '=== mysql/app logs (last 200 lines) ==='
compose --profile setup logs --no-color --tail 200 \
mysql schema-migrator account-server || true
} >"$DIAGNOSTICS_FILE" 2>&1
chmod 600 "$DIAGNOSTICS_FILE"
}
cleanup() {
local status=$?
trap - EXIT INT TERM
if ((status != 0)); then
capture_diagnostics
fi
compose --profile setup down --volumes --remove-orphans --rmi local >/dev/null 2>&1 || true
rm -f \
"$COMPOSE_ENV" \
"$RUNTIME_ENV" \
"$ADMIN_ENV" \
"$ADMIN_GENERATED_ENV" \
"$ADMIN_HANDOFF" \
"$SUPER_COOKIE_JAR" \
"$ANALYST_COOKIE_JAR" \
"$RESPONSE_BODY" \
"$ACTUAL_PRIVILEGES" \
"$WORK_DIR/denied.err"
rmdir "$SECRET_DIR" 2>/dev/null || true
if ((status == 0)); then
rm -f "$DIAGNOSTICS_FILE"
rmdir "$WORK_DIR" 2>/dev/null || true
log "PASS: containers, volume, image, and temporary credentials removed"
else
printf '[smoke] non-sensitive diagnostics kept at %s\n' "$DIAGNOSTICS_FILE" >&2
fi
exit "$status"
}
trap cleanup EXIT INT TERM
wait_for_service_http() {
local service=$1
local url=$2
local attempts=${3:-90}
local attempt
for ((attempt = 1; attempt <= attempts; attempt++)); do
if compose exec -T "$service" wget -q -O /dev/null "$url" 2>/dev/null; then
return 0
fi
if [[ "$(compose ps --all --format json "$service" 2>/dev/null || true)" == *'"State":"exited"'* ]]; then
return 1
fi
sleep 2
done
return 1
}
wait_for_host_http() {
local path=$1
local attempts=${2:-30}
local attempt status
for ((attempt = 1; attempt <= attempts; attempt++)); do
status="$(curl --silent --output /dev/null --write-out '%{http_code}' \
"http://127.0.0.1:$APP_PORT$path" 2>/dev/null || true)"
if [[ "$status" == "200" ]]; then
return 0
fi
sleep 1
done
return 1
}
http_request() {
local expected=$1
local method=$2
local path=$3
local cookie_jar=$4
local body=$5
shift 5
local -a command=(
curl --silent --show-error
--output "$RESPONSE_BODY"
--write-out '%{http_code}'
--request "$method"
"http://127.0.0.1:$APP_PORT$path"
)
local header
if [[ "$cookie_jar" != "-" ]]; then
command+=(--cookie "$cookie_jar" --cookie-jar "$cookie_jar")
fi
if [[ -n "$body" ]]; then
command+=(--header 'Content-Type: application/json' --data "$body")
fi
for header in "$@"; do
command+=(--header "$header")
done
local status
status="$("${command[@]}")"
[[ "$status" == "$expected" ]] || fail "$method $path returned $status, expected $expected"
}
json_value() {
local file=$1
shift
python3 - "$file" "$@" <<'PY'
import json
import sys
value = json.load(open(sys.argv[1], encoding="utf-8"))
for key in sys.argv[2:]:
if key.isdigit():
value = value[int(key)]
else:
value = value[key]
if value is None:
print("")
elif isinstance(value, bool):
print(str(value).lower())
else:
print(value)
PY
}
totp_code() {
local secret=$1
python3 - "$secret" <<'PY'
import base64
import hashlib
import hmac
import struct
import sys
import time
secret = base64.b32decode(sys.argv[1], casefold=True)
counter = int(time.time()) // 30
digest = hmac.new(secret, struct.pack(">Q", counter), hashlib.sha1).digest()
offset = digest[-1] & 0x0F
number = struct.unpack(">I", digest[offset:offset + 4])[0] & 0x7FFFFFFF
print(f"{number % 1_000_000:06d}")
PY
}
url_encode() {
python3 - "$1" <<'PY'
import sys
import urllib.parse
print(urllib.parse.quote(sys.argv[1], safe=""))
PY
}
assert_json_value() {
local expected=$1
shift
local actual
actual="$(json_value "$RESPONSE_BODY" "$@")"
[[ "$actual" == "$expected" ]] || fail "JSON value '$actual' did not equal '$expected'"
}
verify_privilege_matrix() {
mysql_root --batch --skip-column-names information_schema >"$ACTUAL_PRIVILEGES" <<'SQL'
SELECT TABLE_NAME, PRIVILEGE_TYPE
FROM TABLE_PRIVILEGES
WHERE GRANTEE = '''osg_smoke_runtime''@''%'''
AND TABLE_SCHEMA = 'osg_account_smoke'
ORDER BY TABLE_NAME, PRIVILEGE_TYPE;
SQL
python3 - "$SMOKE_DIR/runtime-grants.sql" "$ACTUAL_PRIVILEGES" <<'PY'
import re
import sys
grant_pattern = re.compile(
r"^GRANT\s+(.+?)\s+ON\s+osg_account_smoke\.([a-z0-9_]+)\s+TO\s+'osg_smoke_runtime'@'%';$",
re.IGNORECASE,
)
expected = set()
for raw_line in open(sys.argv[1], encoding="utf-8"):
match = grant_pattern.match(raw_line.strip())
if match:
for privilege in match.group(1).split(","):
expected.add((match.group(2).lower(), privilege.strip().upper()))
actual = set()
for raw_line in open(sys.argv[2], encoding="utf-8"):
table, privilege = raw_line.rstrip("\n").split("\t")
actual.add((table.lower(), privilege.upper()))
missing = sorted(expected - actual)
unexpected = sorted(actual - expected)
if missing or unexpected:
print(f"missing privileges: {missing}", file=sys.stderr)
print(f"unexpected privileges: {unexpected}", file=sys.stderr)
raise SystemExit(1)
PY
local elevated_count
elevated_count="$(mysql_root --batch --skip-column-names information_schema <<'SQL'
SELECT
(SELECT COUNT(*) FROM USER_PRIVILEGES
WHERE GRANTEE = '''osg_smoke_runtime''@''%''' AND PRIVILEGE_TYPE <> 'USAGE')
+ (SELECT COUNT(*) FROM SCHEMA_PRIVILEGES
WHERE GRANTEE = '''osg_smoke_runtime''@''%''')
+ (SELECT COUNT(*) FROM mysql.procs_priv
WHERE User = 'osg_smoke_runtime' AND Host = '%');
SQL
)"
[[ "$elevated_count" == "0" ]] || fail "runtime user received global, schema, or routine privileges"
}
expect_runtime_denied() {
local description=$1
local statement=$2
if mysql_runtime osg_account_smoke --execute "$statement" \
>/dev/null 2>"$WORK_DIR/denied.err"; then
fail "$description unexpectedly succeeded"
fi
}
verify_immutable_history_denials() {
expect_runtime_denied \
"credit ledger UPDATE" \
"UPDATE credit_ledger SET amount_delta = amount_delta WHERE 1 = 0"
expect_runtime_denied \
"credit ledger DELETE" \
"DELETE FROM credit_ledger WHERE 1 = 0"
expect_runtime_denied \
"admin audit UPDATE" \
"UPDATE admin_audit_log SET outcome = outcome WHERE 1 = 0"
expect_runtime_denied \
"admin audit DELETE" \
"DELETE FROM admin_audit_log WHERE 1 = 0"
expect_runtime_denied \
"admin grant UPDATE" \
"UPDATE admin_credit_grants SET amount = amount WHERE 1 = 0"
expect_runtime_denied \
"admin grant DELETE" \
"DELETE FROM admin_credit_grants WHERE 1 = 0"
expect_runtime_denied \
"Flyway metadata read" \
"SELECT version FROM flyway_schema_history LIMIT 1"
}
verify_session_cleanup_permission() {
mysql_runtime osg_account_smoke \
--execute "DELETE FROM admin_sessions WHERE expires_at < UTC_TIMESTAMP() AND 1 = 0"
}
verify_ledger_pagination() {
local cursor first_id second_id encoded_cursor
http_request 200 GET "/v1/admin/users/$SMOKE_ACCOUNT_ID/ledger" "$SUPER_COOKIE_JAR" "" \
'X-OSG-mTLS-Verified: SUCCESS'
[[ "$(json_value "$RESPONSE_BODY" items | tr -d '\n')" != "" ]] || fail "ledger page was empty"
[[ "$(python3 - "$RESPONSE_BODY" <<'PY'
import json
import sys
print(len(json.load(open(sys.argv[1], encoding="utf-8"))["items"]))
PY
)" == "100" ]] || fail "first ledger page did not contain 100 entries"
first_id="$(json_value "$RESPONSE_BODY" items 0 entryId)"
cursor="$(json_value "$RESPONSE_BODY" nextCursor)"
[[ -n "$cursor" ]] || fail "first ledger page omitted nextCursor with more than 100 entries"
encoded_cursor="$(url_encode "$cursor")"
http_request 200 GET \
"/v1/admin/users/$SMOKE_ACCOUNT_ID/ledger?cursor=$encoded_cursor" \
"$SUPER_COOKIE_JAR" "" \
'X-OSG-mTLS-Verified: SUCCESS'
second_id="$(json_value "$RESPONSE_BODY" items 0 entryId)"
[[ -n "$second_id" && "$second_id" != "$first_id" ]] || fail "ledger cursor repeated the first page"
}
require_command curl
require_command docker
require_command openssl
require_command python3
APP_PORT="$(free_loopback_port)"
MYSQL_ROOT_PASSWORD="$(random_hex 24)"
MYSQL_RUNTIME_PASSWORD="$(random_hex 24)"
MYSQL_MIGRATION_PASSWORD="$(random_hex 24)"
JWT_SECRET="$(random_hex 32)"
FIELD_ENCRYPTION_KEY="$(random_base64_key)"
IDENTITY_HMAC_KEY="$(random_base64_key)"
cat >"$COMPOSE_ENV" <<EOF
SMOKE_RUN_ID=$RUN_ID
SMOKE_APP_PORT=$APP_PORT
SMOKE_MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD
SMOKE_MYSQL_MIGRATION_PASSWORD=$MYSQL_MIGRATION_PASSWORD
SMOKE_RUNTIME_ENV=$RUNTIME_ENV
SMOKE_ADMIN_ENV=$ADMIN_ENV
SMOKE_SECRET_DIR=$SECRET_DIR
SMOKE_HOST_UID=$(id -u)
SMOKE_HOST_GID=$(id -g)
EOF
cat >"$RUNTIME_ENV" <<EOF
APP_ENV=test
PORT=8080
PUBLIC_BASE_URL=http://127.0.0.1:$APP_PORT
INVITE_BASE_URL=https://osglab.com/i
APP_STORE_URL=https://apps.apple.com/us/app/smoke/id1
DATABASE_URL=jdbc:mysql://mysql:3306/osg_account_smoke?useUnicode=true&characterEncoding=utf8&connectionTimeZone=UTC&forceConnectionTimeZoneToSession=true
DATABASE_USER=osg_smoke_runtime
DATABASE_PASSWORD=$MYSQL_RUNTIME_PASSWORD
DATABASE_POOL_SIZE=4
DATABASE_MIGRATION_USER=osg_smoke_migrator
DATABASE_MIGRATION_PASSWORD=$MYSQL_MIGRATION_PASSWORD
JWT_ISSUER=http://127.0.0.1:$APP_PORT
JWT_AUDIENCE=osg-smoke
JWT_SECRET=$JWT_SECRET
ACCESS_TOKEN_MINUTES=15
REFRESH_TOKEN_DAYS=1
GATEWAY_GRANT_DAYS=1
FIELD_ENCRYPTION_KEY=$FIELD_ENCRYPTION_KEY
IDENTITY_HMAC_KEY=$IDENTITY_HMAC_KEY
IDENTITY_TOMBSTONE_RETENTION_DAYS=1
APPLE_TEAM_ID=
APPLE_KEY_ID=
APPLE_CLIENT_ID=com.osgkeyboard.smoke
APPLE_PRIVATE_KEY_PEM=
APPLE_JWKS_URL=http://127.0.0.1:9/apple/jwks
APPLE_TOKEN_URL=http://127.0.0.1:9/apple/token
APPLE_REVOKE_URL=http://127.0.0.1:9/apple/revoke
APPLE_INTEGRITY_ENVIRONMENT=development
ENFORCE_DEVICE_CHECK=false
ENFORCE_APP_ATTEST=false
APP_ATTEST_CHALLENGE_TTL_SECONDS=30
VOLCENGINE_API_KEY=
VOLCENGINE_APP_ID=
VOLCENGINE_ACCESS_TOKEN=
VOLCENGINE_RESOURCE_ID=smoke-disabled
VOLCENGINE_ASR_ENDPOINT=ws://127.0.0.1:9/volcengine
DEEPSEEK_API_KEY=
DEEPSEEK_MODEL=smoke-disabled
DEEPSEEK_ENDPOINT=http://127.0.0.1:9/deepseek
SIGNUP_TRIAL_CREDITS=100
REFERRAL_INVITER_CREDITS=100
REFERRAL_INVITEE_CREDITS=100
REFERRAL_BINDING_DAYS=1
ADMIN_SESSION_HOURS=1
ADMIN_MAXIMUM_MANUAL_GRANT=1000
EOF
chmod 600 "$COMPOSE_ENV" "$RUNTIME_ENV"
log "validating isolated Compose model"
compose --profile setup config --quiet
log "building local application image"
compose --profile setup build credential-generator
log "generating one-time administrator credentials"
compose --profile setup run --rm --no-deps credential-generator >/dev/null
mv "$ADMIN_GENERATED_ENV" "$ADMIN_ENV"
chmod 600 "$ADMIN_ENV" "$ADMIN_HANDOFF"
ADMIN_PASSWORD="$(awk -F '' '/^密码:/{print $2}' "$ADMIN_HANDOFF")"
ADMIN_TOTP_SECRET="$(awk -F '' '/^TOTP 密钥:/{print $2}' "$ADMIN_HANDOFF")"
[[ -n "$ADMIN_PASSWORD" && -n "$ADMIN_TOTP_SECRET" ]] || fail "administrator credential generation failed"
log "starting disposable MySQL 8.4"
compose up --detach --wait --wait-timeout 120 mysql
log "creating isolated migration and runtime users"
mysql_root <<SQL
CREATE USER 'osg_smoke_migrator'@'%' IDENTIFIED BY '$MYSQL_MIGRATION_PASSWORD';
CREATE USER 'osg_smoke_runtime'@'%' IDENTIFIED BY '$MYSQL_RUNTIME_PASSWORD';
GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, INDEX, REFERENCES, TRIGGER
ON osg_account_smoke.* TO 'osg_smoke_migrator'@'%';
SQL
log "applying Flyway migrations with the dedicated migrator"
compose --profile setup up --detach schema-migrator
wait_for_service_http schema-migrator 'http://127.0.0.1:8080/health/ready' ||
fail "schema migrator did not become ready"
MIGRATIONS="$(mysql_root --batch --skip-column-names osg_account_smoke <<'SQL'
SELECT CONCAT(version, ':', success)
FROM flyway_schema_history
WHERE version IS NOT NULL
ORDER BY installed_rank;
SQL
)"
EXPECTED_MIGRATIONS=$'1:1\n2:1\n3:1\n4:1\n5:1\n6:1\n7:1\n8:1'
[[ "$MIGRATIONS" == "$EXPECTED_MIGRATIONS" ]] ||
fail "Flyway history was not exactly successful V1-V8"
compose --profile setup stop schema-migrator >/dev/null
log "installing exact runtime grants and disposable fixture"
mysql_root osg_account_smoke <"$SMOKE_DIR/runtime-grants.sql"
mysql_root osg_account_smoke <"$SMOKE_DIR/fixture.sql"
verify_privilege_matrix
log "starting application with runtime-only database access"
compose up --detach account-server
wait_for_service_http account-server 'http://127.0.0.1:8080/health/ready' ||
fail "account server did not become ready"
wait_for_host_http '/health/ready' ||
fail "account server loopback port did not become ready"
http_request 200 GET '/health/ready' - ''
assert_json_value UP status
log "verifying hidden admin edge and authenticated session"
http_request 404 GET '/v1/admin/auth/session' - ''
http_request 200 GET '/v1/admin/auth/session' - '' 'X-OSG-mTLS-Verified: SUCCESS'
assert_json_value false authenticated
ADMIN_TOTP_CODE="$(totp_code "$ADMIN_TOTP_SECRET")"
http_request 200 POST '/v1/admin/auth/login' "$SUPER_COOKIE_JAR" \
"{\"username\":\"smoke-admin\",\"password\":\"$ADMIN_PASSWORD\",\"totpCode\":\"$ADMIN_TOTP_CODE\"}" \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT"
assert_json_value SUPER_ADMIN role
CSRF_TOKEN="$(json_value "$RESPONSE_BODY" csrfToken)"
log "verifying CSRF, RBAC, core reads, and append-only grant path"
http_request 403 POST '/v1/admin/credits/grants' "$SUPER_COOKIE_JAR" \
"{\"userId\":\"$SMOKE_ACCOUNT_ID\",\"amount\":7,\"reason\":\"smoke grant\"}" \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT" \
'Idempotency-Key: smoke-grant-no-csrf'
assert_json_value CSRF_INVALID code
http_request 200 POST '/v1/admin/credits/grants' "$SUPER_COOKIE_JAR" \
"{\"userId\":\"$SMOKE_ACCOUNT_ID\",\"amount\":7,\"reason\":\"smoke grant\"}" \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT" \
"X-CSRF-Token: $CSRF_TOKEN" \
'Idempotency-Key: smoke-grant-success'
assert_json_value 108 balanceAfter
for path in \
'/v1/admin/overview?range=7d' \
'/v1/admin/referrals?range=7d' \
'/v1/admin/users' \
"/v1/admin/users/$SMOKE_ACCOUNT_ID" \
'/v1/admin/operators/summary' \
'/v1/admin/operators' \
'/v1/admin/audit'; do
http_request 200 GET "$path" "$SUPER_COOKIE_JAR" '' 'X-OSG-mTLS-Verified: SUCCESS'
done
http_request 201 POST '/v1/admin/operators' "$SUPER_COOKIE_JAR" \
'{"username":"smoke-analyst","password":"smoke-analyst-password","role":"ANALYST"}' \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT" \
"X-CSRF-Token: $CSRF_TOKEN"
ANALYST_TOTP_SECRET="$(json_value "$RESPONSE_BODY" totpSecret)"
ANALYST_TOTP_CODE="$(totp_code "$ANALYST_TOTP_SECRET")"
http_request 200 POST '/v1/admin/auth/login' "$ANALYST_COOKIE_JAR" \
"{\"username\":\"smoke-analyst\",\"password\":\"smoke-analyst-password\",\"totpCode\":\"$ANALYST_TOTP_CODE\"}" \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT"
http_request 403 GET '/v1/admin/users' "$ANALYST_COOKIE_JAR" '' \
'X-OSG-mTLS-Verified: SUCCESS'
assert_json_value INSUFFICIENT_PERMISSION code
if ! verify_ledger_pagination; then
FAILURES=$((FAILURES + 1))
printf '[smoke] ledger pagination verification failed\n' >&2
fi
verify_immutable_history_denials
verify_session_cleanup_permission
log "verifying logout and session revocation"
http_request 204 POST '/v1/admin/auth/logout' "$SUPER_COOKIE_JAR" '' \
'X-OSG-mTLS-Verified: SUCCESS' \
"Origin: http://127.0.0.1:$APP_PORT" \
"X-CSRF-Token: $CSRF_TOKEN"
ACTIVE_SUPER_SESSIONS="$(mysql_runtime --batch --skip-column-names osg_account_smoke <<'SQL'
SELECT COUNT(*)
FROM admin_sessions AS session
JOIN admin_operators AS operator ON operator.id = session.operator_id
WHERE operator.username = 'smoke-admin'
AND session.revoked_at IS NULL;
SQL
)"
[[ "$ACTIVE_SUPER_SESSIONS" == "0" ]] || fail "logout did not revoke the super-administrator session"
((FAILURES == 0)) || fail "$FAILURES smoke verification(s) failed"