diff --git a/.env.example b/.env.example index be803df..fc0af9d 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,21 @@ RABBITMQ_DLQ=sync_queue.dlq RABBITMQ_NF_QUEUE=not_found_parse_queue RABBITMQ_NF_DLQ=not_found_parse_queue.dlq +# ===== 投稿数据迁移队列 ===== +RABBITMQ_MIGRATION_EXCHANGE=ttml.migration +RABBITMQ_MIGRATION_QUEUE=migration.user +RABBITMQ_MIGRATION_DLQ=migration.user.dlq +RABBITMQ_MIGRATION_CLOSE_QUEUE=migration.close_pr +RABBITMQ_MIGRATION_CLOSE_DLQ=migration.close_pr.dlq +# 迁移起始 PR 号（含），该号之前的 PR 不迁移 +MIGRATION_PR_START_NUMBER=7390 +# 迁移时 GitHub API 请求并发上限（≤10） +MIGRATION_CONCURRENCY=10 +# 每处理多少条 PR 刷新一次任务进度 +MIGRATION_BATCH_SIZE=20 +# 迁移完成后是否自动关闭已迁移的 GitHub PR（默认关闭） +MIGRATION_CLOSE_PR_ENABLED=false + # ===== MeiliSearch ===== MEILISEARCH_HOST=http://localhost:7700 MEILISEARCH_API_KEY=meili-master-key @@ -49,6 +64,11 @@ GITHUB_APP_REPO_OWNER=amll-dev GITHUB_APP_REPO_NAME=amll-ttml-db GITHUB_APP_UPLOAD_FOLDER=raw-lyrics +# ===== GitHub OAuth===== +GITHUB_OAUTH_CLIENT_ID= +GITHUB_OAUTH_CLIENT_SECRET= +GITHUB_OAUTH_REDIRECT_URI=http://localhost:8080/api/v1/auth/github/callback + # ===== Submission 投稿模块 ===== SUBMISSION_AUTO_REJECT_INTERVAL=1h SUBMISSION_AUTO_REJECT_AFTER=96h diff --git a/.gitignore b/.gitignore index 444acab..ace0093 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,8 @@ pnpm-debug.log* # Environment .env .env.local + +# Secrets / keys +*.pem +*.key +*private-key* diff --git a/android/app/src/main/java/dev/amll/hub/android/data/remote/AuthRepository.kt b/android/app/src/main/java/dev/amll/hub/android/data/remote/AuthRepository.kt index bb6b2f8..8d83469 100644 --- a/android/app/src/main/java/dev/amll/hub/android/data/remote/AuthRepository.kt +++ b/android/app/src/main/java/dev/amll/hub/android/data/remote/AuthRepository.kt @@ -3,11 +3,15 @@ package dev.amll.hub.android.data.remote import dev.amll.hub.android.data.local.AuthStore import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json import retrofit2.HttpException import java.io.IOException import javax.inject.Inject import javax.inject.Singleton +/** 解析错误响应体用的轻量 Json */ +private val errorJson = Json { ignoreUnknownKeys = true } + /** * 认证仓库：统一处理后端 {code, message, data} 包装与错误映射。 * 对应网页端 lib/api.ts 的 request() 管道。 @@ -31,7 +35,7 @@ class AuthRepository @Inject constructor( } suspend fun sendCode(checkType: String, dest: String) { - call { api.sendCode(SendCodeRequest(checkType = checkType, dest = dest.trim())) } + request { api.sendCode(SendCodeRequest(checkType = checkType, dest = dest.trim())) } } suspend fun profile(): UserProfile = call { api.profile() } @@ -44,24 +48,41 @@ class AuthRepository @Inject constructor( /** 确认把当前账号授权给该票据 */ suspend fun confirmQrLogin(ticket: String) { - call { api.confirmQr(TicketRequest(ticket)) } + request { api.confirmQr(TicketRequest(ticket)) } } /** 取消本次扫码登录 */ suspend fun cancelQrLogin(ticket: String) { - call { api.cancelQr(TicketRequest(ticket)) } + request { api.cancelQr(TicketRequest(ticket)) } } /** - * 调用后端接口并解包统一响应。 - * code !== 200 时抛出携带后端 message 的 [ApiException]，UI 直接展示即可。 + * 需要数据体的接口：`data` 缺失视为异常。 */ - private suspend fun call(block: suspend () -> ApiResponse): T = + private suspend fun call(block: suspend () -> ApiResponse): T = + request(block) ?: throw ApiException(200, "服务端返回空数据") + + /** + * 统一请求管道：解包 {code, message, data} 并映射错误。 + * + * - `data` 允许为空：后端 `OKWithMsg(c, nil, ...)`（发码、扫码确认/取消等）因 + * `json:"data,omitempty"` 会直接省略 data 字段。网页端 request() 同样只校验 code， + * 不要求 data 非空 —— 必须保持一致，否则发码接口永远报「服务端返回空数据」。 + * - HTTP 非 2xx 时尽量取出响应体里的后端 message（如「验证码错误」）， + * 而不是统一映射成「服务异常」，否则用户无法判断错在哪。 + */ + private suspend fun request(block: suspend () -> ApiResponse): T? = withContext(Dispatchers.IO) { val resp = try { block() } catch (e: HttpException) { - throw if (e.code() == 401) UnauthorizedException() else NetworkException("服务异常（HTTP ${e.code()}）") + val backendMsg = errorMessage(e) + if (e.code() == 401) { + // 401 既可能是 token 失效，也可能是「用户名或密码错误」， + // 用后端文案覆盖默认提示；类型仍是 UnauthorizedException。 + throw if (backendMsg == null) UnauthorizedException() else UnauthorizedException(backendMsg) + } + throw ApiException(e.code(), backendMsg ?: "服务异常（HTTP ${e.code()}）") } catch (e: IOException) { throw NetworkException() } @@ -70,6 +91,13 @@ class AuthRepository @Inject constructor( if (resp.code == 401) throw UnauthorizedException(resp.message) throw ApiException(resp.code, resp.message.ifBlank { "请求失败（${resp.code}）" }) } - resp.data ?: throw ApiException(resp.code, "服务端返回空数据") + resp.data } + + /** 后端错误响应体形如 {"code":400,"message":"验证码错误"}，取出 message 供 UI 展示 */ + private fun errorMessage(e: HttpException): String? = runCatching { + e.response()?.errorBody()?.string() + ?.let { errorJson.decodeFromString>(it).message } + ?.takeIf { it.isNotBlank() } + }.getOrNull() } diff --git a/android/app/src/main/java/dev/amll/hub/android/data/remote/Dto.kt b/android/app/src/main/java/dev/amll/hub/android/data/remote/Dto.kt index 482d905..bb6197d 100644 --- a/android/app/src/main/java/dev/amll/hub/android/data/remote/Dto.kt +++ b/android/app/src/main/java/dev/amll/hub/android/data/remote/Dto.kt @@ -41,7 +41,8 @@ data class SendCodeRequest( val checkType: String, val dest: String, val method: String = "login", - val captchaType: String = "none", + /** 空串表示交由后端按 Casdoor 配置自动补齐人机验证（勿传 "none"） */ + val captchaType: String = "", val captchaToken: String = "", ) diff --git a/android/app/src/main/java/dev/amll/hub/android/nav/AmllNavGraph.kt b/android/app/src/main/java/dev/amll/hub/android/nav/AmllNavGraph.kt index bc3999f..ab451bf 100644 --- a/android/app/src/main/java/dev/amll/hub/android/nav/AmllNavGraph.kt +++ b/android/app/src/main/java/dev/amll/hub/android/nav/AmllNavGraph.kt @@ -1,19 +1,30 @@ package dev.amll.hub.android.nav +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect -import androidx.navigation.NavHostController +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.hilt.navigation.compose.hiltViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.navigation.NavType import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.rememberNavController import androidx.navigation.navArgument +import dev.amll.hub.android.data.remote.UserProfile import dev.amll.hub.android.ui.screen.HomeScreen import dev.amll.hub.android.ui.screen.LoginScreen import dev.amll.hub.android.ui.screen.ScanScreen +import dev.amll.hub.android.ui.session.SessionState +import dev.amll.hub.android.ui.session.SessionViewModel object Routes { - const val LOGIN = "login" const val HOME = "home" /** 扫码页，ticket 可为空（手动进入扫码器）或由 deep link 带入 */ @@ -22,42 +33,65 @@ object Routes { if (ticket.isNullOrBlank()) "scan" else "scan?ticket=$ticket" } +/** + * 顶层导航，完全由 [SessionViewModel] 的持久化登录态驱动： + * - Loading：读本地 token 期间的启动占位，避免登录页一闪而过 + * - LoggedIn：首页 + 扫码授权页 + * - LoggedOut：仅登录页，未登录不提供「扫码登录网页端」入口 + * + * 登录 / 登出只更新登录态，分区切换由状态完成，无需手动推栈。 + */ @Composable fun AmllNavGraph( - navController: NavHostController = rememberNavController(), - /** deep link 带进来的 ticket，非空时直接跳扫码确认页 */ + /** deep link 带进来的 ticket，非空时登录后直接弹扫码确认页 */ pendingTicket: String? = null, onTicketConsumed: () -> Unit = {}, + sessionViewModel: SessionViewModel = hiltViewModel(), ) { - // deep link 优先于当前页面：用户点二维码就是为了来授权的 + val session by sessionViewModel.state.collectAsStateWithLifecycle() + + when (val state = session) { + SessionState.Loading -> SplashPane() + + is SessionState.LoggedIn -> LoggedInNav( + profile = state.profile, + pendingTicket = pendingTicket, + onTicketConsumed = onTicketConsumed, + onLoggedOut = { sessionViewModel.logout() }, + onRefreshProfile = { sessionViewModel.refreshProfile() }, + ) + + SessionState.LoggedOut -> LoginScreen( + onLoggedIn = { sessionViewModel.onLoggedIn(it) }, + ) + } +} + +/** 登录后的导航分区：首页 + 扫码授权页 */ +@Composable +private fun LoggedInNav( + profile: UserProfile?, + pendingTicket: String?, + onTicketConsumed: () -> Unit, + onLoggedOut: () -> Unit, + onRefreshProfile: () -> Unit, +) { + val navController = rememberNavController() + + // 已登录时 deep link 直接进扫码确认页；未登录时 ticket 会被保留，登录后再处理 LaunchedEffect(pendingTicket) { - val ticket = pendingTicket ?: return@LaunchedEffect + val ticket = pendingTicket?.takeIf { it.isNotBlank() } ?: return@LaunchedEffect onTicketConsumed() - navController.navigate(Routes.scan(ticket)) { - popUpTo(Routes.LOGIN) { inclusive = true } - } + navController.navigate(Routes.scan(ticket)) } - NavHost(navController = navController, startDestination = Routes.LOGIN) { - composable(Routes.LOGIN) { - LoginScreen( - onLoggedIn = { - navController.navigate(Routes.HOME) { - popUpTo(Routes.LOGIN) { inclusive = true } - } - }, - onScanQr = { navController.navigate(Routes.scan()) }, - ) - } - + NavHost(navController = navController, startDestination = Routes.HOME) { composable(Routes.HOME) { HomeScreen( + profile = profile, onScanQr = { navController.navigate(Routes.scan()) }, - onLoggedOut = { - navController.navigate(Routes.LOGIN) { - popUpTo(Routes.HOME) { inclusive = true } - } - }, + onLoggedOut = onLoggedOut, + onRefreshProfile = onRefreshProfile, ) } @@ -75,12 +109,24 @@ fun AmllNavGraph( initialTicket = entry.arguments?.getString("ticket"), onBack = { navController.popBackStack() }, onDone = { - // 扫码页带参数，popUpTo 用 route 模式匹配不到，直接清栈 navController.navigate(Routes.HOME) { - popUpTo(0) { inclusive = true } + popUpTo(Routes.HOME) { inclusive = true } } }, ) } } } + +/** 读取本地登录态期间的启动占位，避免登录页一闪而过 */ +@Composable +private fun SplashPane() { + Surface(color = MaterialTheme.colorScheme.background) { + Box( + contentAlignment = Alignment.Center, + modifier = Modifier.fillMaxSize(), + ) { + CircularProgressIndicator() + } + } +} diff --git a/android/app/src/main/java/dev/amll/hub/android/ui/screen/HomeScreen.kt b/android/app/src/main/java/dev/amll/hub/android/ui/screen/HomeScreen.kt index 035897e..861cf2a 100644 --- a/android/app/src/main/java/dev/amll/hub/android/ui/screen/HomeScreen.kt +++ b/android/app/src/main/java/dev/amll/hub/android/ui/screen/HomeScreen.kt @@ -21,13 +21,10 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.unit.dp -import androidx.hilt.navigation.compose.hiltViewModel -import androidx.lifecycle.compose.collectAsStateWithLifecycle import coil3.compose.AsyncImage import dev.amll.hub.android.data.remote.UserProfile @@ -36,13 +33,15 @@ import dev.amll.hub.android.data.remote.UserProfile */ @Composable fun HomeScreen( + profile: UserProfile?, onScanQr: () -> Unit, onLoggedOut: () -> Unit, - viewModel: LoginViewModel = hiltViewModel(), + onRefreshProfile: () -> Unit, ) { - val state by viewModel.uiState.collectAsStateWithLifecycle() - - LaunchedEffect(Unit) { viewModel.restoreSession() } + // 登录态已恢复但资料缺失（如离线启动）时补拉一次 + LaunchedEffect(profile) { + if (profile == null) onRefreshProfile() + } Surface(color = MaterialTheme.colorScheme.background) { Column( @@ -58,7 +57,7 @@ fun HomeScreen( ) Spacer(Modifier.height(24.dp)) - state.profile?.let { ProfileCard(it) } + profile?.let { ProfileCard(it) } Spacer(Modifier.height(24.dp)) @@ -78,10 +77,7 @@ fun HomeScreen( Spacer(Modifier.height(8.dp)) OutlinedButton( - onClick = { - viewModel.logout() - onLoggedOut() - }, + onClick = onLoggedOut, modifier = Modifier.fillMaxWidth(), ) { Icon( diff --git a/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginScreen.kt b/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginScreen.kt index 7ed9acb..75b3d95 100644 --- a/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginScreen.kt +++ b/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginScreen.kt @@ -41,12 +41,12 @@ import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle +import dev.amll.hub.android.data.remote.UserProfile import dev.amll.hub.android.ui.component.ErrorBanner @Composable fun LoginScreen( - onLoggedIn: () -> Unit, - onScanQr: () -> Unit, + onLoggedIn: (UserProfile) -> Unit, viewModel: LoginViewModel = hiltViewModel(), ) { val state by viewModel.uiState.collectAsStateWithLifecycle() @@ -105,7 +105,7 @@ fun LoginScreen( Spacer(Modifier.height(24.dp)) Button( - onClick = { viewModel.submit { onLoggedIn() } }, + onClick = { viewModel.submit { onLoggedIn(it.user) } }, enabled = state.canSubmit, modifier = Modifier .fillMaxWidth() @@ -121,12 +121,6 @@ fun LoginScreen( Text("登录") } } - - Spacer(Modifier.height(12.dp)) - - TextButton(onClick = onScanQr) { - Text("扫码登录网页端") - } } } } diff --git a/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginViewModel.kt b/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginViewModel.kt index 0ab3249..7c2e6e9 100644 --- a/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginViewModel.kt +++ b/android/app/src/main/java/dev/amll/hub/android/ui/screen/LoginViewModel.kt @@ -7,7 +7,6 @@ import dev.amll.hub.android.data.remote.ApiException import dev.amll.hub.android.data.remote.AuthRepository import dev.amll.hub.android.data.remote.LoginResult import dev.amll.hub.android.data.remote.UnauthorizedException -import dev.amll.hub.android.data.remote.UserProfile import kotlinx.coroutines.Job import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -37,8 +36,6 @@ data class LoginUiState( /** 验证码发送成功后的倒计时秒数，0 表示未在倒计时 */ val countdown: Int = 0, val error: String? = null, - val profile: UserProfile? = null, - val loggedIn: Boolean = false, ) { val canSubmit: Boolean get() = !submitting && when (tab) { @@ -57,19 +54,6 @@ class LoginViewModel @Inject constructor( private var countdownJob: Job? = null - /** 冷启动时恢复登录态，有 token 就直接拉一次资料 */ - fun restoreSession() { - viewModelScope.launch { - val token = repo.currentToken() - if (token.isNullOrBlank()) return@launch - runCatching { repo.profile() } - .onSuccess { user -> - _uiState.update { it.copy(profile = user, loggedIn = true) } - } - .onFailure { if (it !is UnauthorizedException) repo.logout() } - } - } - fun switchTab(tab: LoginTab) = _uiState.update { it.copy(tab = tab, error = null, countdown = 0) } @@ -128,7 +112,7 @@ class LoginViewModel @Inject constructor( } .onSuccess { result -> _uiState.update { - it.copy(submitting = false, loggedIn = true, password = "", code = "") + it.copy(submitting = false, password = "", code = "") } onSuccess(result) } @@ -137,13 +121,6 @@ class LoginViewModel @Inject constructor( } } } - - fun logout() { - viewModelScope.launch { - repo.logout() - _uiState.update { LoginUiState() } - } - } } /** 把异常转成可直接展示给用户的中文文案 */ diff --git a/android/app/src/main/java/dev/amll/hub/android/ui/session/SessionViewModel.kt b/android/app/src/main/java/dev/amll/hub/android/ui/session/SessionViewModel.kt new file mode 100644 index 0000000..aa1bafc --- /dev/null +++ b/android/app/src/main/java/dev/amll/hub/android/ui/session/SessionViewModel.kt @@ -0,0 +1,82 @@ +package dev.amll.hub.android.ui.session + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import dagger.hilt.android.lifecycle.HiltViewModel +import dev.amll.hub.android.data.remote.AuthRepository +import dev.amll.hub.android.data.remote.UnauthorizedException +import dev.amll.hub.android.data.remote.UserProfile +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import javax.inject.Inject + +/** App 级登录态，决定冷启动落在登录页还是首页 */ +sealed interface SessionState { + /** 正在读取本地 token */ + data object Loading : SessionState + + /** 已登录；profile 允许为空（启动时先放行，联网后再补拉资料） */ + data class LoggedIn(val profile: UserProfile?) : SessionState + + /** 未登录，或后端明确判定登录态失效（401） */ + data object LoggedOut : SessionState +} + +/** + * 全局登录态。修复「重启即退出登录」： + * - token 由 [dev.amll.hub.android.data.local.AuthStore] 持久化在 DataStore，重启后仍在 + * - 冷启动只读本地 token（不阻塞等网络），有 token 直接进首页，资料随后补拉 + * - 只有后端返回 401 才清除本地登录态；离线等临时故障保留登录态 + */ +@HiltViewModel +class SessionViewModel @Inject constructor( + private val repo: AuthRepository, +) : ViewModel() { + + private val _state = MutableStateFlow(SessionState.Loading) + val state: StateFlow = _state.asStateFlow() + + init { + viewModelScope.launch { + val token = repo.currentToken() + _state.value = if (token.isNullOrBlank()) { + SessionState.LoggedOut + } else { + SessionState.LoggedIn(null) + } + } + } + + /** 已登录但资料缺失时补拉；顺带校验 token，失效则退回登录页 */ + fun refreshProfile() { + val current = _state.value as? SessionState.LoggedIn ?: return + viewModelScope.launch { + runCatching { repo.profile() } + .onSuccess { profile -> _state.value = SessionState.LoggedIn(profile) } + .onFailure { e -> + if (e is UnauthorizedException) { + repo.logout() + _state.value = SessionState.LoggedOut + } else { + // 网络异常：token 依然有效，保留登录态与已有资料 + _state.value = SessionState.LoggedIn(current.profile) + } + } + } + } + + /** 登录成功后由 UI 回传资料 */ + fun onLoggedIn(profile: UserProfile?) { + _state.value = SessionState.LoggedIn(profile) + } + + /** 退出登录 */ + fun logout() { + viewModelScope.launch { + repo.logout() + _state.value = SessionState.LoggedOut + } + } +} diff --git a/android/app/src/test/java/dev/amll/hub/android/AuthDtoTest.kt b/android/app/src/test/java/dev/amll/hub/android/AuthDtoTest.kt index 9f67477..43c9faf 100644 --- a/android/app/src/test/java/dev/amll/hub/android/AuthDtoTest.kt +++ b/android/app/src/test/java/dev/amll/hub/android/AuthDtoTest.kt @@ -4,6 +4,7 @@ import kotlinx.serialization.json.Json import dev.amll.hub.android.data.remote.ApiResponse import dev.amll.hub.android.data.remote.LoginResult import dev.amll.hub.android.data.remote.QrTicketStatus +import dev.amll.hub.android.data.remote.SendCodeRequest import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNull @@ -50,4 +51,24 @@ class AuthDtoTest { val status = json.decodeFromString(body) assertTrue(status.isScanned) } + + @Test + fun `无 data 字段的成功响应应可解析`() { + // 后端 OKWithMsg(c, nil, ...)（发码 / 扫码确认取消）因 `data,omitempty` + // 会直接省略 data 字段，客户端不能把 data 缺失当成错误 + val resp = json.decodeFromString>( + """{"code":200,"message":"验证码已发送"}""" + ) + assertEquals(200, resp.code) + assertEquals("验证码已发送", resp.message) + assertNull(resp.data) + } + + @Test + fun `发码请求省略可选项，由后端补齐人机验证参数`() { + // Json 未开启 encodeDefaults（默认 false），带默认值的字段不会出现在请求体里。 + // 后端因此走 captchaType == "" 分支，按 Casdoor 配置自动补齐；method 默认 login。 + val body = json.encodeToString(SendCodeRequest(checkType = "phone", dest = "13800000000")) + assertEquals("""{"checkType":"phone","dest":"13800000000"}""", body) + } } diff --git a/backend/cmd/server/server.go b/backend/cmd/server/server.go index 1185771..55cb812 100644 --- a/backend/cmd/server/server.go +++ b/backend/cmd/server/server.go @@ -52,7 +52,7 @@ func Run() { logrus.Fatalf("init minio: %v", err) } - mq, err := infrastructure.NewRabbitMQ(cfg.RabbitMQ) + mq, err := infrastructure.NewRabbitMQ(cfg.RabbitMQ, cfg.Migration) if err != nil { logrus.Fatalf("init rabbitmq: %v", err) } @@ -95,6 +95,10 @@ func Run() { adminRepo := repository.NewAdminRepo(db) notifRepo := repository.NewNotificationRepo(db) + // GitHub 登录/迁移 repository + githubBindingRepo := repository.NewGithubBindingRepo(db) + githubMigrationRepo := repository.NewGithubMigrationRepo(db) + // 搜索IP投稿 repository searchIpRepo := repository.NewSearchIPRepo(db) @@ -118,6 +122,20 @@ func Run() { cfg.Casdoor.Organization, ) + // GitHub OAuth 登录/绑定 + 投稿数据迁移 + githubOAuthClient := infrastructure.NewGithubOAuthClient(cfg.GitHubOAuth) + githubBindSvc := service.NewGithubBindService( + githubOAuthClient, + authSvc, + githubBindingRepo, + redisClient, + cfg.Casdoor.JWTSecret, + cfg.Casdoor.JWTTTL, + cfg.Casdoor.Organization, + cfg.GitHubOAuth.SiteURL, + ) + migrationSvc := service.NewMigrationService(githubBindingRepo, githubMigrationRepo, mq, cfg.Migration) + // 投稿模块 service fileSvc := service.NewFileService(cfg, minioClient) githubSvc := service.NewGitHubService(githubApp) @@ -224,6 +242,8 @@ func Run() { onlineSearchH := handler.NewOnlineSearchHandler(onlineSearchSvc) cloudMusicH := handler.NewCloudMusicHandler(cloudMusicSvc) authH := handler.NewAuthHandler(authSvc, reviewerCache, adminCache) + authGithubH := handler.NewAuthGithubHandler(githubBindSvc, reviewerCache, adminCache) + migrationH := handler.NewMigrationHandler(migrationSvc) // 投稿 handler submissionH := handler.NewSubmissionHandler(submissionSvc, reviewerCache) @@ -252,6 +272,8 @@ func Run() { OnlineSearch: onlineSearchH, CloudMusic: cloudMusicH, Auth: authH, + AuthGithub: authGithubH, + Migration: migrationH, Submission: submissionH, Review: reviewH, Revision: revisionH, diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index c5eb897..14095dc 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -22,6 +22,8 @@ type Config struct { MeiliSearch MeiliSearchConfig GitHub GitHubConfig GitHubApp GitHubAppConfig + GitHubOAuth GitHubOAuthConfig + Migration MigrationConfig Sync SyncConfig NCM NCMConfig OnlineSearch OnlineSearchConfig @@ -125,6 +127,39 @@ type GitHubAppConfig struct { UploadFolder string } +// GitHubOAuthConfig 自建 GitHub OAuth 登录配置 +type GitHubOAuthConfig struct { + ClientID string + ClientSecret string + RedirectURI string + // SiteURL 前端站点地址，OAuth 回调后跳转目标 + SiteURL string +} + +// Enabled 是否已配置 GitHub OAuth +func (c GitHubOAuthConfig) Enabled() bool { + return c.ClientID != "" && c.ClientSecret != "" +} + +// MigrationConfig 投稿数据迁移配置（后端只负责创建任务与投递消息） +type MigrationConfig struct { + // Exchange 迁移交换机（ttml.migration） + Exchange string + // Queue migration.user 队列 + Queue string + // DLQ migration.user.dlq + DLQ string + // CloseQueue migration.close_pr 队列 + CloseQueue string + // CloseDLQ migration.close_pr.dlq + CloseDLQ string + // StartPrNumber 迁移起始 PR 号（含） + StartPrNumber int + // RepoOwner / RepoName 迁移来源仓库 + RepoOwner string + RepoName string +} + // SubmissionConfig 投稿模块配置 type SubmissionConfig struct { // AutoRejectInterval 自动拒绝任务执行间隔 @@ -261,6 +296,19 @@ func Load() (*Config, error) { v.SetDefault("GITHUB_APP_REPO_NAME", "amll-ttml-db") v.SetDefault("GITHUB_APP_UPLOAD_FOLDER", "raw-lyrics") + // GitHub OAuth（自建登录/绑定） + v.SetDefault("GITHUB_OAUTH_CLIENT_ID", "") + v.SetDefault("GITHUB_OAUTH_CLIENT_SECRET", "") + v.SetDefault("GITHUB_OAUTH_REDIRECT_URI", "http://localhost:8080/api/v1/auth/github/callback") + + // 投稿数据迁移（后端只投递消息） + v.SetDefault("RABBITMQ_MIGRATION_EXCHANGE", "ttml.migration") + v.SetDefault("RABBITMQ_MIGRATION_QUEUE", "migration.user") + v.SetDefault("RABBITMQ_MIGRATION_DLQ", "migration.user.dlq") + v.SetDefault("RABBITMQ_MIGRATION_CLOSE_QUEUE", "migration.close_pr") + v.SetDefault("RABBITMQ_MIGRATION_CLOSE_DLQ", "migration.close_pr.dlq") + v.SetDefault("MIGRATION_PR_START_NUMBER", 7390) + // Submission 投稿模块 v.SetDefault("SUBMISSION_AUTO_REJECT_INTERVAL", "1h") v.SetDefault("SUBMISSION_AUTO_REJECT_AFTER", "96h") @@ -354,6 +402,22 @@ func Load() (*Config, error) { RepoName: v.GetString("GITHUB_APP_REPO_NAME"), UploadFolder: v.GetString("GITHUB_APP_UPLOAD_FOLDER"), }, + GitHubOAuth: GitHubOAuthConfig{ + ClientID: v.GetString("GITHUB_OAUTH_CLIENT_ID"), + ClientSecret: v.GetString("GITHUB_OAUTH_CLIENT_SECRET"), + RedirectURI: v.GetString("GITHUB_OAUTH_REDIRECT_URI"), + SiteURL: strings.TrimSuffix(v.GetString("SITE_URL"), "/"), + }, + Migration: MigrationConfig{ + Exchange: v.GetString("RABBITMQ_MIGRATION_EXCHANGE"), + Queue: v.GetString("RABBITMQ_MIGRATION_QUEUE"), + DLQ: v.GetString("RABBITMQ_MIGRATION_DLQ"), + CloseQueue: v.GetString("RABBITMQ_MIGRATION_CLOSE_QUEUE"), + CloseDLQ: v.GetString("RABBITMQ_MIGRATION_CLOSE_DLQ"), + StartPrNumber: v.GetInt("MIGRATION_PR_START_NUMBER"), + RepoOwner: v.GetString("GITHUB_APP_REPO_OWNER"), + RepoName: v.GetString("GITHUB_APP_REPO_NAME"), + }, Submission: SubmissionConfig{ AutoRejectInterval: v.GetDuration("SUBMISSION_AUTO_REJECT_INTERVAL"), AutoRejectAfter: v.GetDuration("SUBMISSION_AUTO_REJECT_AFTER"), @@ -387,6 +451,10 @@ func Load() (*Config, error) { logrus.Warnf("GitHub App config incomplete (GITHUB_APP_ID/GITHUB_INSTALLATION_ID/GITHUB_PRIVATE_KEY_PATH), submission approve upload will be disabled") } + if !cfg.GitHubOAuth.Enabled() { + logrus.Warnf("GitHub OAuth config incomplete (GITHUB_OAUTH_CLIENT_ID/GITHUB_OAUTH_CLIENT_SECRET), GitHub login/binding will be disabled") + } + if os.Getenv("APP_ENV") == "production" { if cfg.MinIO.AccessKey == "minioadmin" { logrus.Warnf("MINIO_ACCESS_KEY is default 'minioadmin' in production") diff --git a/backend/internal/handler/auth_github.go b/backend/internal/handler/auth_github.go new file mode 100644 index 0000000..f7448ad --- /dev/null +++ b/backend/internal/handler/auth_github.go @@ -0,0 +1,222 @@ +package handler + +import ( + "context" + "errors" + "net/http" + + "github.com/amll-dev/amll-hub/backend/internal/middleware" + "github.com/amll-dev/amll-hub/backend/internal/pkg" + "github.com/amll-dev/amll-hub/backend/internal/service" + "github.com/gin-gonic/gin" + logrus "github.com/sirupsen/logrus" +) + +// AuthGithubHandler GitHub OAuth 登录/绑定 handler +type AuthGithubHandler struct { + svc *service.GithubBindService + reviewerCache *middleware.ReviewerCache + adminCache *middleware.AdminCache +} + +// NewAuthGithubHandler 创建 AuthGithubHandler +func NewAuthGithubHandler(svc *service.GithubBindService, rc *middleware.ReviewerCache, ac *middleware.AdminCache) *AuthGithubHandler { + return &AuthGithubHandler{svc: svc, reviewerCache: rc, adminCache: ac} +} + +// Login GET /api/v1/auth/github/login +// 跳转 GitHub 授权页（匿名登录/注册） +func (h *AuthGithubHandler) Login(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + authorizeURL, err := h.svc.StartLogin(ctx) + if err != nil { + writeGithubErr(c, err) + return + } + c.Redirect(http.StatusFound, authorizeURL) +} + +// BindURL GET /api/v1/auth/github/bind-url +func (h *AuthGithubHandler) BindURL(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + authorizeURL, err := h.svc.StartBind(ctx, username) + if err != nil { + writeGithubErr(c, err) + return + } + pkg.OK(c, gin.H{"url": authorizeURL}) +} + +// Callback GET /api/v1/auth/github/callback +// GitHub 回调：校验 state → 换 token → 签发 JWT 或进入绑定页 +func (h *AuthGithubHandler) Callback(c *gin.Context) { + code := c.Query("code") + state := c.Query("state") + if code == "" || state == "" { + pkg.BadRequest(c, "缺少 code 或 state 参数") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), longTimeout) + defer cancel() + + redirectURL, err := h.svc.HandleCallback(ctx, code, state) + if err != nil { + writeGithubErr(c, err) + return + } + c.Redirect(http.StatusFound, redirectURL) +} + +// BindInfo POST /api/v1/auth/github/bind/info +// 绑定页预填信息（票据走请求体，避免出现在 URL 与访问日志中） +func (h *AuthGithubHandler) BindInfo(c *gin.Context) { + var req struct { + Token string `json:"token"` + } + if err := c.ShouldBindJSON(&req); err != nil || req.Token == "" { + pkg.BadRequest(c, "缺少绑定凭证") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + info, err := h.svc.GetBindInfo(ctx, req.Token) + if err != nil { + writeGithubErr(c, err) + return + } + pkg.OK(c, info) +} + +// BindLogin POST /api/v1/auth/github/bind/login +// 已有账号登录并绑定 +func (h *AuthGithubHandler) BindLogin(c *gin.Context) { + var req service.GithubBindLoginRequest + if err := c.ShouldBindJSON(&req); err != nil { + pkg.BadRequest(c, "token、账号、密码必填") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), longTimeout) + defer cancel() + + result, err := h.svc.BindLogin(ctx, req) + if err != nil { + writeGithubErr(c, err) + return + } + h.enrichReviewer(c, &result.User) + pkg.OK(c, result) +} + +// BindRegister POST /api/v1/auth/github/bind/register +// 注册新账号并绑定 +func (h *AuthGithubHandler) BindRegister(c *gin.Context) { + var req service.GithubBindRegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + pkg.BadRequest(c, "参数错误") + return + } + if req.Token == "" || req.Username == "" || req.Password == "" || + req.Email == "" || req.EmailCode == "" || req.DisplayName == "" { + pkg.BadRequest(c, "token、用户名、昵称、密码、邮箱、邮箱验证码必填") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), longTimeout) + defer cancel() + + result, err := h.svc.BindRegister(ctx, req) + if err != nil { + writeGithubErr(c, err) + return + } + h.enrichReviewer(c, &result.User) + pkg.OK(c, result) +} + +// Unbind DELETE /api/v1/user/github/binding +func (h *AuthGithubHandler) Unbind(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + if err := h.svc.Unbind(ctx, username); err != nil { + writeGithubErr(c, err) + return + } + pkg.OKWithMsg(c, nil, "已解绑 GitHub") +} + +// enrichReviewer 复用 AuthHandler 的审核员/管理员标记逻辑 +func (h *AuthGithubHandler) enrichReviewer(c *gin.Context, p *service.UserProfile) { + if p == nil || p.Name == "" { + return + } + if h.reviewerCache != nil { + ok, _ := h.reviewerCache.IsReviewer(c.Request.Context(), p.Name) + p.IsReviewer = ok + } + if h.adminCache != nil { + ok, _ := h.adminCache.IsAdmin(c.Request.Context(), p.Name) + p.IsAdmin = ok + } +} + +// writeGithubErr 统一处理 GitHub 登录/绑定错误 +func writeGithubErr(c *gin.Context, err error) { + switch { + case errors.Is(err, service.ErrGithubOAuthDisabled): + pkg.Fail(c, http.StatusServiceUnavailable, http.StatusServiceUnavailable, "站点尚未配置 GitHub 登录，请联系管理员") + case errors.Is(err, service.ErrOAuthStateInvalid): + pkg.BadRequest(c, "登录已过期，请重新发起") + case errors.Is(err, service.ErrBindTokenExpired): + pkg.Fail(c, http.StatusGone, http.StatusGone, "授权已过期，请重新登录") + case errors.Is(err, service.ErrAccountAlreadyBound): + pkg.Fail(c, http.StatusConflict, http.StatusConflict, "该账号已绑定其他 GitHub，请先解绑") + case errors.Is(err, service.ErrGithubAlreadyBound): + pkg.Fail(c, http.StatusConflict, http.StatusConflict, "该 GitHub 账号已绑定其他站点账号") + case errors.Is(err, service.ErrAccountLocked): + pkg.Fail(c, http.StatusTooManyRequests, http.StatusTooManyRequests, "账户已锁定，请稍后再试") + case errors.Is(err, service.ErrInvalidCredentials): + pkg.Fail(c, http.StatusUnauthorized, http.StatusUnauthorized, "用户名或密码错误") + case errors.Is(err, service.ErrInvalidCode): + pkg.BadRequest(c, "验证码错误") + case errors.Is(err, service.ErrCodeExpired): + pkg.BadRequest(c, "验证码已失效，请重新获取") + case errors.Is(err, service.ErrUserAlreadyExists): + pkg.Fail(c, http.StatusConflict, http.StatusConflict, "用户名或邮箱已存在") + case errors.Is(err, service.ErrInvalidInput): + pkg.BadRequest(c, "请求参数非法") + default: + writeUpstreamErrOrInternal(c, err) + } +} + +// writeUpstreamErrOrInternal 上游错误给 502，其余给 500（并去掉错误前缀细节） +func writeUpstreamErrOrInternal(c *gin.Context, err error) { + if errors.Is(err, service.ErrUpstreamUnavailable) { + logrus.WithError(err).Warn("github oauth upstream unavailable") + pkg.Fail(c, http.StatusBadGateway, http.StatusBadGateway, "GitHub 服务暂不可用，请稍后再试") + return + } + logrus.WithError(err).Warn("github oauth unknown error") + pkg.InternalError(c, "内部错误") +} diff --git a/backend/internal/handler/migration.go b/backend/internal/handler/migration.go new file mode 100644 index 0000000..d831be8 --- /dev/null +++ b/backend/internal/handler/migration.go @@ -0,0 +1,164 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "strconv" + + "github.com/amll-dev/amll-hub/backend/internal/middleware" + "github.com/amll-dev/amll-hub/backend/internal/model" + "github.com/amll-dev/amll-hub/backend/internal/pkg" + "github.com/amll-dev/amll-hub/backend/internal/service" + "github.com/gin-gonic/gin" + logrus "github.com/sirupsen/logrus" +) + +// MigrationHandler 投稿数据迁移 handler +type MigrationHandler struct { + svc *service.MigrationService +} + +// NewMigrationHandler 创建 MigrationHandler +func NewMigrationHandler(svc *service.MigrationService) *MigrationHandler { + return &MigrationHandler{svc: svc} +} + +// Status GET /api/v1/migration/github/status +func (h *MigrationHandler) Status(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + st, err := h.svc.Status(ctx, username) + if err != nil { + writeMigrationErr(c, err) + return + } + pkg.OK(c, st) +} + +// Start POST /api/v1/migration/github/start +func (h *MigrationHandler) Start(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + task, err := h.svc.Start(ctx, username, submitterInfoFromContext(c, username)) + if err != nil { + writeMigrationErr(c, err) + return + } + pkg.OK(c, task) +} + +// GetTask GET /api/v1/migration/github/tasks/:id +func (h *MigrationHandler) GetTask(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + pkg.BadRequest(c, "任务 ID 非法") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + task, err := h.svc.GetTask(ctx, username, id) + if err != nil { + writeMigrationErr(c, err) + return + } + pkg.OK(c, task) +} + +// Retry POST /api/v1/migration/github/tasks/:id/retry +func (h *MigrationHandler) Retry(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + pkg.BadRequest(c, "任务 ID 非法") + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + task, err := h.svc.Retry(ctx, username, id, submitterInfoFromContext(c, username)) + if err != nil { + writeMigrationErr(c, err) + return + } + pkg.OK(c, task) +} + +// ListMigrated GET /api/v1/migration/github/migrated +// 返回该用户已迁移的 PR 列表（含 PR 标题）与总数 +func (h *MigrationHandler) ListMigrated(c *gin.Context) { + username := middleware.GetUserName(c) + if username == "" { + pkg.Unauthorized(c) + return + } + + limit := 100 + if v := c.Query("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + limit = n + } + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultTimeout) + defer cancel() + + res, err := h.svc.ListMigrated(ctx, username, limit) + if err != nil { + writeMigrationErr(c, err) + return + } + pkg.OK(c, res) +} + +// submitterInfoFromContext 从 JWT 注入的 context 构造投稿者信息 +func submitterInfoFromContext(c *gin.Context, username string) model.UserInfo { + return model.UserInfo{ + Username: username, + DisplayName: middleware.GetUserDisplayName(c), + Avatar: middleware.GetUserAvatar(c), + } +} + +// writeMigrationErr 统一处理迁移模块错误 +func writeMigrationErr(c *gin.Context, err error) { + switch { + case errors.Is(err, service.ErrNotBound): + pkg.Fail(c, http.StatusForbidden, http.StatusForbidden, "请先绑定 GitHub 账号") + case errors.Is(err, service.ErrMigrationInProgress): + pkg.Fail(c, http.StatusConflict, http.StatusConflict, "已有进行中的迁移任务，请等待完成") + case errors.Is(err, service.ErrMigrationTaskNotFound): + pkg.NotFound(c, "迁移任务不存在") + case errors.Is(err, service.ErrUpstreamUnavailable): + writeUpstreamErr(c, err, "迁移服务暂不可用，请稍后再试") + default: + logrus.WithError(err).Warn("migration service unknown error") + pkg.InternalError(c, "内部错误") + } +} diff --git a/backend/internal/infrastructure/github_oauth.go b/backend/internal/infrastructure/github_oauth.go new file mode 100644 index 0000000..e41070f --- /dev/null +++ b/backend/internal/infrastructure/github_oauth.go @@ -0,0 +1,156 @@ +package infrastructure + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/amll-dev/amll-hub/backend/internal/config" +) + +// GithubOAuthClient 自建 GitHub OAuth 客户端（不依赖 Casdoor 第三方登录） +type GithubOAuthClient struct { + cfg config.GitHubOAuthConfig + http *http.Client +} + +// GithubUser GitHub 用户基本信息 +type GithubUser struct { + ID int64 `json:"id"` + Login string `json:"login"` + Name string `json:"name"` + AvatarURL string `json:"avatar_url"` + Email string `json:"email"` +} + +// NewGithubOAuthClient 创建 GithubOAuthClient +func NewGithubOAuthClient(cfg config.GitHubOAuthConfig) *GithubOAuthClient { + return &GithubOAuthClient{ + cfg: cfg, + http: &http.Client{Timeout: 15 * time.Second}, + } +} + +// Enabled 是否已配置 +func (c *GithubOAuthClient) Enabled() bool { return c.cfg.Enabled() } + +// AuthorizeURL 生成 GitHub 授权页跳转地址 +func (c *GithubOAuthClient) AuthorizeURL(state string) string { + q := url.Values{} + q.Set("client_id", c.cfg.ClientID) + q.Set("redirect_uri", c.cfg.RedirectURI) + q.Set("scope", "read:user user:email") + q.Set("state", state) + return "https://github.com/login/oauth/authorize?" + q.Encode() +} + +// ExchangeCode 用授权码换取 access token +func (c *GithubOAuthClient) ExchangeCode(ctx context.Context, code string) (string, error) { + if !c.Enabled() { + return "", fmt.Errorf("github oauth not configured") + } + form := url.Values{} + form.Set("client_id", c.cfg.ClientID) + form.Set("client_secret", c.cfg.ClientSecret) + form.Set("code", code) + form.Set("redirect_uri", c.cfg.RedirectURI) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + "https://github.com/login/oauth/access_token", + strings.NewReader(form.Encode())) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return "", fmt.Errorf("exchange code: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read token response: %w", err) + } + var parsed struct { + AccessToken string `json:"access_token"` + Error string `json:"error"` + Description string `json:"error_description"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", fmt.Errorf("decode token response: %w", err) + } + if parsed.AccessToken == "" { + return "", fmt.Errorf("github exchange code failed: %s %s", parsed.Error, parsed.Description) + } + return parsed.AccessToken, nil +} + +// GetUser 获取当前授权用户信息 +func (c *GithubOAuthClient) GetUser(ctx context.Context, accessToken string) (*GithubUser, error) { + var user GithubUser + if err := c.getJSON(ctx, accessToken, "https://api.github.com/user", &user); err != nil { + return nil, err + } + return &user, nil +} + +// GetPrimaryEmail 取主邮箱（primary && verified）；无可用邮箱返回空串 +func (c *GithubOAuthClient) GetPrimaryEmail(ctx context.Context, accessToken string) string { + var emails []struct { + Email string `json:"email"` + Primary bool `json:"primary"` + Verified bool `json:"verified"` + } + if err := c.getJSON(ctx, accessToken, "https://api.github.com/user/emails", &emails); err != nil { + return "" + } + for _, e := range emails { + if e.Primary && e.Verified { + return e.Email + } + } + for _, e := range emails { + if e.Verified { + return e.Email + } + } + return "" +} + +// getJSON 发送带 token 的 GET 请求并解析 JSON +func (c *GithubOAuthClient) getJSON(ctx context.Context, accessToken, apiURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+accessToken) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("User-Agent", "amll-hub") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("github api %s: %w", apiURL, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read github api response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("github api %s status=%d body=%s", apiURL, resp.StatusCode, string(body)) + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("decode github api response: %w", err) + } + return nil +} diff --git a/backend/internal/infrastructure/rabbitmq.go b/backend/internal/infrastructure/rabbitmq.go index c900673..c4f9edc 100644 --- a/backend/internal/infrastructure/rabbitmq.go +++ b/backend/internal/infrastructure/rabbitmq.go @@ -23,6 +23,7 @@ type RabbitMQ struct { NFQueue amqp.Queue // 无歌词解析队列 NFDLQ amqp.Queue // 无歌词死信队列 cfg config.RabbitMQConfig + migCfg config.MigrationConfig } // 无歌词解析相关常量 @@ -35,9 +36,17 @@ const ( NFDLQName = "not_found_parse_queue.dlq" ) +// 投稿数据迁移相关常量（独立 exchange + DLQ） +const ( + MigDLXExchange = "ttml.migration.dlx" + MigDLQRouting = "migration.failed" + MigRoutingUser = "migration.user" + MigRoutingClose = "migration.close_pr" +) + // NewRabbitMQ 初始化 RabbitMQ，声明主队列 + 死信队列 -func NewRabbitMQ(cfg config.RabbitMQConfig) (*RabbitMQ, error) { - r := &RabbitMQ{cfg: cfg} +func NewRabbitMQ(cfg config.RabbitMQConfig, migCfg config.MigrationConfig) (*RabbitMQ, error) { + r := &RabbitMQ{cfg: cfg, migCfg: migCfg} if err := r.connect(); err != nil { return nil, err } @@ -179,9 +188,65 @@ func (r *RabbitMQ) declareTopology(ch *amqp.Channel) (queue, dlq, nfQueue, nfDlq err = fmt.Errorf("bind nf queue: %w", err) return } + + // === 投稿数据迁移队列（独立交换机/队列/DLQ；后端只声明+投递，消费在 Worker） === + if err = r.declareMigrationTopology(ch); err != nil { + return + } return } +// declareMigrationTopology 声明迁移交换机、两个队列及其 DLQ。 +// 拓扑与 Worker 侧保持一致：exchange=ttml.migration(direct)，DLX=ttml.migration.dlx。 +func (r *RabbitMQ) declareMigrationTopology(ch *amqp.Channel) error { + if err := ch.ExchangeDeclare( + MigDLXExchange, "direct", true, false, false, false, nil, + ); err != nil { + return fmt.Errorf("declare migration dlx exchange: %w", err) + } + + declDLQ := func(name string) error { + q, err := ch.QueueDeclare(name, true, false, false, false, nil) + if err != nil { + return fmt.Errorf("declare migration dlq %s: %w", name, err) + } + return ch.QueueBind(q.Name, MigDLQRouting, MigDLXExchange, false, nil) + } + if err := declDLQ(r.migCfg.DLQ); err != nil { + return err + } + if err := declDLQ(r.migCfg.CloseDLQ); err != nil { + return err + } + + if err := ch.ExchangeDeclare( + r.migCfg.Exchange, "direct", true, false, false, false, nil, + ); err != nil { + return fmt.Errorf("declare migration exchange: %w", err) + } + + declQueue := func(name, routingKey string) error { + q, err := ch.QueueDeclare( + name, true, false, false, false, + amqp.Table{ + "x-dead-letter-exchange": MigDLXExchange, + "x-dead-letter-routing-key": MigDLQRouting, + }, + ) + if err != nil { + return fmt.Errorf("declare migration queue %s: %w", name, err) + } + return ch.QueueBind(q.Name, routingKey, r.migCfg.Exchange, false, nil) + } + if err := declQueue(r.migCfg.Queue, MigRoutingUser); err != nil { + return err + } + if err := declQueue(r.migCfg.CloseQueue, MigRoutingClose); err != nil { + return err + } + return nil +} + // 关闭旧连接并重新建立 func (r *RabbitMQ) reconnect() error { r.mu.Lock() @@ -323,6 +388,36 @@ func (r *RabbitMQ) PublishNotFoundParse(msg NotFoundParseMessage) error { ) } +// PublishMigrationUser 投递一条按用户迁移任务到 migration.user 队列 +func (r *RabbitMQ) PublishMigrationUser(body []byte, messageID string) error { + return r.publish( + r.migCfg.Exchange, + MigRoutingUser, + amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: messageID, + Timestamp: time.Now(), + Body: body, + }, + ) +} + +// PublishMigrationClosePr 投递一条关闭 PR 任务到 migration.close_pr 队列 +func (r *RabbitMQ) PublishMigrationClosePr(body []byte, messageID string) error { + return r.publish( + r.migCfg.Exchange, + MigRoutingClose, + amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: messageID, + Timestamp: time.Now(), + Body: body, + }, + ) +} + // Close 关闭连接 func (r *RabbitMQ) Close() error { r.mu.Lock() diff --git a/backend/internal/middleware/auth.go b/backend/internal/middleware/auth.go index 7c9fcd4..ba645dd 100644 --- a/backend/internal/middleware/auth.go +++ b/backend/internal/middleware/auth.go @@ -22,33 +22,44 @@ const ( // Auth JWT 校验中间件 func Auth(jwtSecret string) gin.HandlerFunc { return func(c *gin.Context) { - authHeader := c.GetHeader("Authorization") - if authHeader == "" { + token := bearerToken(c.GetHeader("Authorization")) + if token == "" { pkg.Fail(c, http.StatusUnauthorized, http.StatusUnauthorized, "missing token") c.Abort() return } - - parts := strings.SplitN(authHeader, " ", 2) - if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { - pkg.Fail(c, http.StatusUnauthorized, http.StatusUnauthorized, "invalid authorization header") + if !setClaims(c, token, jwtSecret) { c.Abort() return } + c.Next() + } +} - claims, err := pkg.ParseJWT(parts[1], jwtSecret) - if err != nil { - pkg.Fail(c, http.StatusUnauthorized, http.StatusUnauthorized, "invalid or expired token") - c.Abort() - return - } +// bearerToken 从 Authorization 头解析出 Bearer token（格式非法返回空串） +func bearerToken(authHeader string) string { + if authHeader == "" { + return "" + } + parts := strings.SplitN(authHeader, " ", 2) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + return "" + } + return parts[1] +} - c.Set(UserIDKey, claims.Sub) - c.Set(UserNameKey, claims.Name) - c.Set(UserDisplayNameKey, claims.DisplayName) - c.Set(UserAvatarKey, claims.Avatar) - c.Next() +// setClaims 解析 JWT 并写入 context；失败时写响应并返回 false +func setClaims(c *gin.Context, token, jwtSecret string) bool { + claims, err := pkg.ParseJWT(token, jwtSecret) + if err != nil { + pkg.Fail(c, http.StatusUnauthorized, http.StatusUnauthorized, "invalid or expired token") + return false } + c.Set(UserIDKey, claims.Sub) + c.Set(UserNameKey, claims.Name) + c.Set(UserDisplayNameKey, claims.DisplayName) + c.Set(UserAvatarKey, claims.Avatar) + return true } // GetUserID 从 context 获取用户 ID diff --git a/backend/internal/model/github_binding.go b/backend/internal/model/github_binding.go new file mode 100644 index 0000000..41c24d0 --- /dev/null +++ b/backend/internal/model/github_binding.go @@ -0,0 +1,18 @@ +package model + +import "time" + +// UserGithubBinding GitHub 账号绑定表。 +// username 为站点 Casdoor 用户名（= JWT claims.name），与 GitHub 数字 ID 双向唯一。 +type UserGithubBinding struct { + ID int64 `gorm:"primaryKey;autoIncrement" json:"id"` + Username string `gorm:"type:varchar(100);not null" json:"username"` + GithubID int64 `gorm:"column:github_id;not null" json:"githubId"` + GithubLogin string `gorm:"column:github_login;type:varchar(100);not null" json:"githubLogin"` + GithubEmail string `gorm:"column:github_email;type:varchar(255)" json:"githubEmail,omitempty"` + GithubAvatar string `gorm:"column:github_avatar;type:varchar(500)" json:"githubAvatar,omitempty"` + BoundAt time.Time `gorm:"column:bound_at;not null;default:CURRENT_TIMESTAMP" json:"boundAt"` + UpdatedAt time.Time `gorm:"not null;default:CURRENT_TIMESTAMP" json:"updatedAt"` +} + +func (UserGithubBinding) TableName() string { return "user_github_bindings" } diff --git a/backend/internal/model/github_migration.go b/backend/internal/model/github_migration.go new file mode 100644 index 0000000..516b4cd --- /dev/null +++ b/backend/internal/model/github_migration.go @@ -0,0 +1,54 @@ +package model + +import "time" + +// 迁移任务状态 +const ( + MigrationStatusPending = "pending" + MigrationStatusRunning = "running" + MigrationStatusCompleted = "completed" + MigrationStatusFailed = "failed" +) + +// GithubMigratedPr 已迁移 PR 记录（全局唯一，用于去重跳过） +type GithubMigratedPr struct { + ID int64 `gorm:"primaryKey;autoIncrement" json:"id"` + PrNumber int `gorm:"column:pr_number;not null" json:"prNumber"` + Username string `gorm:"type:varchar(100);not null" json:"username"` + SubmissionID *int64 `gorm:"column:submission_id" json:"submissionId,omitempty"` + MigratedAt time.Time `gorm:"column:migrated_at;not null;default:CURRENT_TIMESTAMP" json:"migratedAt"` +} + +func (GithubMigratedPr) TableName() string { return "github_migrated_prs" } + +// GithubMigratedPrItem 已迁移 PR 概览（迁移页展示用，含 PR 标题与对应稿件） +type GithubMigratedPrItem struct { + PrNumber int `gorm:"column:pr_number" json:"prNumber"` + PrTitle string `gorm:"column:pr_title" json:"prTitle"` + SubmissionID *int64 `gorm:"column:submission_id" json:"submissionId,omitempty"` + SubmissionTitle string `gorm:"column:submission_title" json:"submissionTitle"` + FileName string `gorm:"column:file_name" json:"fileName"` + MigratedAt time.Time `gorm:"column:migrated_at" json:"migratedAt"` +} + +// GithubMigrationTask 迁移任务与进度 +type GithubMigrationTask struct { + ID int64 `gorm:"primaryKey;autoIncrement" json:"id"` + Username string `gorm:"type:varchar(100);not null" json:"username"` + GithubLogin string `gorm:"column:github_login;type:varchar(100);not null" json:"githubLogin"` + Status string `gorm:"type:varchar(20);not null;default:'pending'" json:"status"` + TotalPrs int `gorm:"column:total_prs;not null;default:0" json:"totalPrs"` + ProcessedPrs int `gorm:"column:processed_prs;not null;default:0" json:"processedPrs"` + CreatedCount int `gorm:"column:created_count;not null;default:0" json:"createdCount"` + SkippedCount int `gorm:"column:skipped_count;not null;default:0" json:"skippedCount"` + FailedCount int `gorm:"column:failed_count;not null;default:0" json:"failedCount"` + Cursor string `gorm:"type:varchar(64)" json:"cursor,omitempty"` + Error string `gorm:"type:text" json:"error,omitempty"` + ClosePrStatus string `gorm:"column:close_pr_status;type:varchar(20);not null;default:'pending'" json:"closePrStatus"` + ClosedPrCount int `gorm:"column:closed_pr_count;not null;default:0" json:"closedPrCount"` + CreatedAt time.Time `gorm:"not null;default:CURRENT_TIMESTAMP" json:"createdAt"` + UpdatedAt time.Time `gorm:"not null;default:CURRENT_TIMESTAMP" json:"updatedAt"` + CompletedAt *time.Time `gorm:"column:completed_at;type:timestamptz" json:"completedAt,omitempty"` +} + +func (GithubMigrationTask) TableName() string { return "github_migration_tasks" } diff --git a/backend/internal/repository/github_binding_repo.go b/backend/internal/repository/github_binding_repo.go new file mode 100644 index 0000000..a8cedf9 --- /dev/null +++ b/backend/internal/repository/github_binding_repo.go @@ -0,0 +1,57 @@ +package repository + +import ( + "context" + "errors" + + "github.com/amll-dev/amll-hub/backend/internal/model" + "gorm.io/gorm" +) + +// GithubBindingRepo GitHub 绑定表数据访问 +type GithubBindingRepo struct { + db *gorm.DB +} + +// NewGithubBindingRepo 创建 GithubBindingRepo +func NewGithubBindingRepo(db *gorm.DB) *GithubBindingRepo { + return &GithubBindingRepo{db: db} +} + +// GetByUsername 按站点用户名查询绑定，未找到返回 (nil, nil) +func (r *GithubBindingRepo) GetByUsername(ctx context.Context, username string) (*model.UserGithubBinding, error) { + var b model.UserGithubBinding + err := r.db.WithContext(ctx).Where("username = ?", username).First(&b).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &b, nil +} + +// GetByGithubID 按 GitHub 数字 ID 查询绑定，未找到返回 (nil, nil) +func (r *GithubBindingRepo) GetByGithubID(ctx context.Context, githubID int64) (*model.UserGithubBinding, error) { + var b model.UserGithubBinding + err := r.db.WithContext(ctx).Where("github_id = ?", githubID).First(&b).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &b, nil +} + +// Create 写入绑定记录 +func (r *GithubBindingRepo) Create(ctx context.Context, b *model.UserGithubBinding) error { + return r.db.WithContext(ctx).Create(b).Error +} + +// DeleteByUsername 解绑 +func (r *GithubBindingRepo) DeleteByUsername(ctx context.Context, username string) error { + return r.db.WithContext(ctx). + Where("username = ?", username). + Delete(&model.UserGithubBinding{}).Error +} diff --git a/backend/internal/repository/github_migration_repo.go b/backend/internal/repository/github_migration_repo.go new file mode 100644 index 0000000..3a0ec17 --- /dev/null +++ b/backend/internal/repository/github_migration_repo.go @@ -0,0 +1,149 @@ +package repository + +import ( + "context" + "errors" + + "github.com/amll-dev/amll-hub/backend/internal/model" + "gorm.io/gorm" +) + +// GithubMigrationRepo 迁移任务/已迁 PR 数据访问。 +// 说明：迁移过程中的进度更新由 Worker 直接写库完成，此处只提供后端创建/查询/重试用到的操作。 +type GithubMigrationRepo struct { + db *gorm.DB +} + +// NewGithubMigrationRepo 创建 GithubMigrationRepo +func NewGithubMigrationRepo(db *gorm.DB) *GithubMigrationRepo { + return &GithubMigrationRepo{db: db} +} + +// CreateTask 创建迁移任务 +func (r *GithubMigrationRepo) CreateTask(ctx context.Context, t *model.GithubMigrationTask) error { + return r.db.WithContext(ctx).Create(t).Error +} + +// GetTask 按 ID 查询任务，未找到返回 (nil, nil) +func (r *GithubMigrationRepo) GetTask(ctx context.Context, id int64) (*model.GithubMigrationTask, error) { + var t model.GithubMigrationTask + err := r.db.WithContext(ctx).First(&t, id).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &t, nil +} + +// GetActiveTask 查询该用户是否有进行中的任务（pending/running） +func (r *GithubMigrationRepo) GetActiveTask(ctx context.Context, username string) (*model.GithubMigrationTask, error) { + var t model.GithubMigrationTask + err := r.db.WithContext(ctx). + Where("username = ? AND status IN ?", username, []string{ + model.MigrationStatusPending, model.MigrationStatusRunning, + }). + Order("id DESC"). + First(&t).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &t, nil +} + +// GetLatestTask 查询该用户最近一次任务 +func (r *GithubMigrationRepo) GetLatestTask(ctx context.Context, username string) (*model.GithubMigrationTask, error) { + var t model.GithubMigrationTask + err := r.db.WithContext(ctx). + Where("username = ?", username). + Order("id DESC"). + First(&t).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &t, nil +} + +// ResetForRetry 把任务重置为待执行。 +// resetCursor=true 表示「重新迁移」：从头发起扫描（已被 github_migrated_prs 记录命中的会跳过）； +// resetCursor=false 表示失败续传：保留 cursor 从断点继续。 +func (r *GithubMigrationRepo) ResetForRetry(ctx context.Context, id int64, resetCursor bool) error { + updates := map[string]any{ + "status": model.MigrationStatusPending, + "error": "", + "updated_at": gorm.Expr("NOW()"), + } + if resetCursor { + updates["cursor"] = nil + updates["total_prs"] = 0 + updates["processed_prs"] = 0 + } + return r.db.WithContext(ctx). + Model(&model.GithubMigrationTask{}). + Where("id = ?", id). + Updates(updates).Error +} + +// MarkFailed 投递失败等场景下把任务标记为失败 +func (r *GithubMigrationRepo) MarkFailed(ctx context.Context, id int64, msg string) error { + return r.db.WithContext(ctx). + Model(&model.GithubMigrationTask{}). + Where("id = ?", id). + Updates(map[string]any{ + "status": model.MigrationStatusFailed, + "error": msg, + "updated_at": gorm.Expr("NOW()"), + }).Error +} + +// HasMigratedPr 判断 PR 是否已被迁移（去重） +func (r *GithubMigrationRepo) HasMigratedPr(ctx context.Context, prNumber int) (bool, error) { + var count int64 + if err := r.db.WithContext(ctx). + Model(&model.GithubMigratedPr{}). + Where("pr_number = ?", prNumber). + Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +// CountMigratedPrs 统计该用户已迁移的 PR 数 +func (r *GithubMigrationRepo) CountMigratedPrs(ctx context.Context, username string) (int64, error) { + var count int64 + if err := r.db.WithContext(ctx). + Model(&model.GithubMigratedPr{}). + Where("username = ?", username). + Count(&count).Error; err != nil { + return 0, err + } + return count, nil +} + +// ListMigratedPrs 查询该用户已迁移的 PR（按 PR 号倒序，附带 PR 标题与对应稿件信息）。 +// PR 标题取自迁移时写入 submissions.metadata 的 github_pr_title。 +func (r *GithubMigrationRepo) ListMigratedPrs(ctx context.Context, username string, limit int) ([]model.GithubMigratedPrItem, error) { + const q = `SELECT g.pr_number AS pr_number, + COALESCE(s.metadata->>'github_pr_title', '') AS pr_title, + g.submission_id AS submission_id, + COALESCE(s.title, '') AS submission_title, + COALESCE(s.file_name, '') AS file_name, + g.migrated_at AS migrated_at + FROM github_migrated_prs g + LEFT JOIN submissions s ON s.id = g.submission_id + WHERE g.username = ? + ORDER BY g.pr_number DESC + LIMIT ?` + items := make([]model.GithubMigratedPrItem, 0, limit) + if err := r.db.WithContext(ctx).Raw(q, username, limit).Scan(&items).Error; err != nil { + return nil, err + } + return items, nil +} diff --git a/backend/internal/router/router.go b/backend/internal/router/router.go index 76b4bf1..df571c7 100644 --- a/backend/internal/router/router.go +++ b/backend/internal/router/router.go @@ -25,6 +25,8 @@ type RouterDeps struct { OnlineSearch *handler.OnlineSearchHandler CloudMusic *handler.CloudMusicHandler Auth *handler.AuthHandler + AuthGithub *handler.AuthGithubHandler + Migration *handler.MigrationHandler Submission *handler.SubmissionHandler Review *handler.ReviewHandler Revision *handler.RevisionHandler @@ -56,6 +58,8 @@ func New(deps RouterDeps) *gin.Engine { onlineSearchH := deps.OnlineSearch cloudMusicH := deps.CloudMusic authH := deps.Auth + authGithubH := deps.AuthGithub + migrationH := deps.Migration submissionH := deps.Submission reviewH := deps.Review revisionH := deps.Revision @@ -178,6 +182,13 @@ func New(deps RouterDeps) *gin.Engine { auth.POST("/qrcode", authH.CreateQRCode) auth.GET("/qrcode/status", authH.QRCodeStatus) + // GitHub OAuth 登录/绑定（公开部分） + auth.GET("/github/login", authGithubH.Login) + auth.GET("/github/callback", authGithubH.Callback) + auth.POST("/github/bind/info", authGithubH.BindInfo) + auth.POST("/github/bind/login", authGithubH.BindLogin) + auth.POST("/github/bind/register", authGithubH.BindRegister) + // 受保护接口 protected := auth.Group("") protected.Use(middleware.Auth(jwtSecret)) @@ -192,6 +203,7 @@ func New(deps RouterDeps) *gin.Engine { protected.POST("/qrcode/scanned", authH.MarkQRCodeScanned) protected.POST("/qrcode/confirm", authH.ConfirmQRCode) protected.POST("/qrcode/cancel", authH.CancelQRCode) + protected.GET("/github/bind-url", authGithubH.BindURL) } // 投稿模块 @@ -252,6 +264,16 @@ func New(deps RouterDeps) *gin.Engine { // 个人中心活动统计 sub.GET("/users/me/activity", userActivityH.Get) + // GitHub 绑定解绑 + sub.DELETE("/user/github/binding", authGithubH.Unbind) + + // 投稿数据迁移（个人中心 → 数据迁移） + sub.GET("/migration/github/status", migrationH.Status) + sub.GET("/migration/github/migrated", migrationH.ListMigrated) + sub.POST("/migration/github/start", migrationH.Start) + sub.GET("/migration/github/tasks/:id", migrationH.GetTask) + sub.POST("/migration/github/tasks/:id/retry", migrationH.Retry) + // 搜索IP显示投稿 sub.POST("/search-ip/submissions", searchIpH.Create) sub.GET("/search-ip/submissions", searchIpH.List) diff --git a/backend/internal/service/errors.go b/backend/internal/service/errors.go index adc7f3f..afbb033 100644 --- a/backend/internal/service/errors.go +++ b/backend/internal/service/errors.go @@ -120,3 +120,31 @@ var ErrImageNotFound = errors.New("search ip image not found") // ErrNotificationNotFound 消息不存在或不属于当前用户 var ErrNotificationNotFound = errors.New("notification not found") + +// GitHub 登录/绑定模块 + +// ErrGithubOAuthDisabled 未配置 GitHub OAuth（client_id / client_secret 缺失） +var ErrGithubOAuthDisabled = errors.New("github oauth disabled") + +// ErrOAuthStateInvalid OAuth state 无效或已过期（可能是伪造/重放） +var ErrOAuthStateInvalid = errors.New("github oauth state invalid") + +// ErrBindTokenExpired 绑定令牌已过期，需重新发起 GitHub 授权 +var ErrBindTokenExpired = errors.New("github bind token expired") + +// ErrAccountAlreadyBound 该站点账号已绑定其他 GitHub +var ErrAccountAlreadyBound = errors.New("account already bound to another github") + +// ErrGithubAlreadyBound 该 GitHub 账号已被其他站点账号绑定 +var ErrGithubAlreadyBound = errors.New("github already bound to another account") + +// ErrNotBound 当前用户尚未绑定 GitHub +var ErrNotBound = errors.New("github not bound") + +// 投稿数据迁移模块 + +// ErrMigrationInProgress 已有进行中的迁移任务 +var ErrMigrationInProgress = errors.New("migration already in progress") + +// ErrMigrationTaskNotFound 迁移任务不存在或不属于当前用户 +var ErrMigrationTaskNotFound = errors.New("migration task not found") diff --git a/backend/internal/service/github_bind_service.go b/backend/internal/service/github_bind_service.go new file mode 100644 index 0000000..e989d96 --- /dev/null +++ b/backend/internal/service/github_bind_service.go @@ -0,0 +1,356 @@ +package service + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/amll-dev/amll-hub/backend/internal/infrastructure" + "github.com/amll-dev/amll-hub/backend/internal/model" + "github.com/amll-dev/amll-hub/backend/internal/pkg" + "github.com/amll-dev/amll-hub/backend/internal/repository" + "github.com/redis/go-redis/v9" + "github.com/sirupsen/logrus" +) + +// GitHub OAuth state / 绑定令牌 的 Redis 前缀与有效期 +const ( + githubOAuthStatePrefix = "oauth:state:" + githubBindPrefix = "bind:" + githubOAuthStateTTL = 10 * time.Minute + githubBindTokenTTL = 15 * time.Minute +) + +// githubBindPayload 暂存在 Redis 中的 GitHub 绑定待选信息 +type githubBindPayload struct { + GithubID int64 `json:"github_id"` + GithubLogin string `json:"github_login"` + GithubEmail string `json:"github_email"` + GithubAvatar string `json:"github_avatar"` +} + +// GithubBindInfo 绑定页展示用的预填信息 +type GithubBindInfo struct { + GithubLogin string `json:"githubLogin"` + GithubEmail string `json:"githubEmail"` + GithubAvatar string `json:"githubAvatar"` +} + +// GithubBindLoginRequest 已有账号登录并绑定 +type GithubBindLoginRequest struct { + Token string `json:"token" binding:"required"` + Account string `json:"account" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// GithubBindRegisterRequest 注册新账号并绑定 +type GithubBindRegisterRequest struct { + RegisterRequest + Token string `json:"token" binding:"required"` +} + +// GithubBindService GitHub 登录/绑定业务逻辑 +type GithubBindService struct { + oauth *infrastructure.GithubOAuthClient + auth *AuthService + repo *repository.GithubBindingRepo + rdb *redis.Client + jwtSecret string + jwtTTL time.Duration + org string + siteURL string +} + +// NewGithubBindService 创建 GithubBindService +func NewGithubBindService( + oauth *infrastructure.GithubOAuthClient, + auth *AuthService, + repo *repository.GithubBindingRepo, + rdb *redis.Client, + jwtSecret string, + jwtTTL time.Duration, + org string, + siteURL string, +) *GithubBindService { + return &GithubBindService{ + oauth: oauth, + auth: auth, + repo: repo, + rdb: rdb, + jwtSecret: jwtSecret, + jwtTTL: jwtTTL, + org: org, + siteURL: siteURL, + } +} + +// randomToken 生成随机十六进制串 +func randomToken(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + // rand.Read 在现代 Go 上几乎不会失败；退化为时间戳保证调用方逻辑继续 + return fmt.Sprintf("%x", time.Now().UnixNano()) + } + return hex.EncodeToString(b) +} + +// StartLogin 生成 state 并返回 GitHub 授权地址（匿名登录/注册场景） +func (s *GithubBindService) StartLogin(ctx context.Context) (string, error) { + return s.startAuth(ctx, "") +} + +// StartBind 已登录用户主动绑定：state 中记录站点用户名，回调时直接写绑定 +func (s *GithubBindService) StartBind(ctx context.Context, username string) (string, error) { + return s.startAuth(ctx, username) +} + +func (s *GithubBindService) startAuth(ctx context.Context, username string) (string, error) { + if !s.oauth.Enabled() { + return "", ErrGithubOAuthDisabled + } + state := randomToken(16) + // value 为空表示匿名登录；非空表示「已登录用户绑定」场景携带的站点用户名 + if err := s.rdb.Set(ctx, githubOAuthStatePrefix+state, username, githubOAuthStateTTL).Err(); err != nil { + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + return s.oauth.AuthorizeURL(state), nil +} + +// HandleCallback 处理 GitHub 回调，返回应跳转的前端地址 +func (s *GithubBindService) HandleCallback(ctx context.Context, code, state string) (string, error) { + if !s.oauth.Enabled() { + return "", ErrGithubOAuthDisabled + } + // 一次性校验 state + key := githubOAuthStatePrefix + state + username, err := s.rdb.GetDel(ctx, key).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return "", ErrOAuthStateInvalid + } + // Redis 异常时 fail-closed，避免绕过 state 校验 + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + + accessToken, err := s.oauth.ExchangeCode(ctx, code) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + ghUser, err := s.oauth.GetUser(ctx, accessToken) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + email := ghUser.Email + if email == "" { + email = s.oauth.GetPrimaryEmail(ctx, accessToken) + } + + // 已登录用户绑定分支：直接把 GitHub 写入该站点账号 + if username != "" { + if err := s.writeBinding(ctx, username, ghUser, email); err != nil { + return "", err + } + return fmt.Sprintf("%s/profile/migration?githubBound=1", s.siteURL), nil + } + + // 匿名登录分支：GitHub 已绑定则直接签发 JWT 登录 + if b, err := s.repo.GetByGithubID(ctx, ghUser.ID); err != nil { + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } else if b != nil { + token, err := s.issueJWT(ctx, b.Username) + if err != nil { + return "", err + } + return fmt.Sprintf("%s/auth/github/callback#token=%s", s.siteURL, token), nil + } + + // 未绑定：暂存待绑定信息，进入绑定页 + bindToken := randomToken(16) + payload := githubBindPayload{ + GithubID: ghUser.ID, + GithubLogin: ghUser.Login, + GithubEmail: email, + GithubAvatar: ghUser.AvatarURL, + } + data, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("marshal bind payload: %w", err) + } + if err := s.rdb.Set(ctx, githubBindPrefix+bindToken, data, githubBindTokenTTL).Err(); err != nil { + return "", fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + // 绑定票据同样放在 fragment 回传，避免落入访问日志 + return fmt.Sprintf("%s/bind-account#token=%s", s.siteURL, bindToken), nil +} + +// GetBindInfo 读取绑定令牌对应的 GitHub 信息 +func (s *GithubBindService) GetBindInfo(ctx context.Context, token string) (*GithubBindInfo, error) { + payload, err := s.loadBindPayload(ctx, token) + if err != nil { + return nil, err + } + return &GithubBindInfo{ + GithubLogin: payload.GithubLogin, + GithubEmail: payload.GithubEmail, + GithubAvatar: payload.GithubAvatar, + }, nil +} + +// BindLogin 用已有站点账号登录并绑定 +func (s *GithubBindService) BindLogin(ctx context.Context, req GithubBindLoginRequest) (*LoginResult, error) { + payload, err := s.loadBindPayload(ctx, req.Token) + if err != nil { + return nil, err + } + + result, err := s.auth.Login(ctx, req.Account, req.Password) + if err != nil { + return nil, err + } + username := result.User.Name + + binding := &model.UserGithubBinding{ + Username: username, + GithubID: payload.GithubID, + GithubLogin: payload.GithubLogin, + GithubEmail: payload.GithubEmail, + GithubAvatar: payload.GithubAvatar, + } + if err := s.createBinding(ctx, binding); err != nil { + return nil, err + } + s.rdb.Del(ctx, githubBindPrefix+req.Token) + return result, nil +} + +// BindRegister 注册新站点账号并绑定 +func (s *GithubBindService) BindRegister(ctx context.Context, req GithubBindRegisterRequest) (*LoginResult, error) { + payload, err := s.loadBindPayload(ctx, req.Token) + if err != nil { + return nil, err + } + if err := s.auth.Register(ctx, req.RegisterRequest); err != nil { + return nil, err + } + result, err := s.auth.Login(ctx, req.Username, req.Password) + if err != nil { + return nil, err + } + + binding := &model.UserGithubBinding{ + Username: result.User.Name, + GithubID: payload.GithubID, + GithubLogin: payload.GithubLogin, + GithubEmail: payload.GithubEmail, + GithubAvatar: payload.GithubAvatar, + } + if err := s.createBinding(ctx, binding); err != nil { + return nil, err + } + + // 新注册账号尚无头像，直接用 GitHub 头像作为站点头像（失败不阻塞注册/绑定）； + // 若用户随后在注册页另选了图片，前端会再上传一次覆盖这里的结果 + if payload.GithubAvatar != "" { + userID := s.org + "/" + result.User.Name + if _, err := s.auth.UpdateProfile(ctx, userID, UpdateProfileRequest{Avatar: payload.GithubAvatar}); err != nil { + logrus.WithError(err).WithField("username", result.User.Name). + Warn("绑定注册：同步 GitHub 头像失败（不影响注册与绑定）") + } else { + result.User.Avatar = payload.GithubAvatar + } + } + + s.rdb.Del(ctx, githubBindPrefix+req.Token) + return result, nil +} + +// GetBinding 查询某站点用户的绑定（未绑定返回 nil, nil） +func (s *GithubBindService) GetBinding(ctx context.Context, username string) (*model.UserGithubBinding, error) { + return s.repo.GetByUsername(ctx, username) +} + +// Unbind 解绑当前用户的 GitHub +func (s *GithubBindService) Unbind(ctx context.Context, username string) error { + if err := s.repo.DeleteByUsername(ctx, username); err != nil { + return fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + return nil +} + +// loadBindPayload 读取并解析 Redis 中的待绑定信息 +func (s *GithubBindService) loadBindPayload(ctx context.Context, token string) (*githubBindPayload, error) { + data, err := s.rdb.Get(ctx, githubBindPrefix+token).Bytes() + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, ErrBindTokenExpired + } + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + var payload githubBindPayload + if err := json.Unmarshal(data, &payload); err != nil { + return nil, ErrBindTokenExpired + } + return &payload, nil +} + +// writeBinding 已登录用户绑定分支：直接写入绑定（含冲突校验） +func (s *GithubBindService) writeBinding(ctx context.Context, username string, gh *infrastructure.GithubUser, email string) error { + return s.createBinding(ctx, &model.UserGithubBinding{ + Username: username, + GithubID: gh.ID, + GithubLogin: gh.Login, + GithubEmail: email, + GithubAvatar: gh.AvatarURL, + }) +} + +// createBinding 写入绑定，前置双向唯一校验 +func (s *GithubBindService) createBinding(ctx context.Context, b *model.UserGithubBinding) error { + if existing, err := s.repo.GetByUsername(ctx, b.Username); err != nil { + return fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } else if existing != nil { + if existing.GithubID == b.GithubID { + // 幂等：同一 GitHub 重复绑定视为成功 + return nil + } + return ErrAccountAlreadyBound + } + if existing, err := s.repo.GetByGithubID(ctx, b.GithubID); err != nil { + return fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } else if existing != nil { + return ErrGithubAlreadyBound + } + if err := s.repo.Create(ctx, b); err != nil { + // 并发下唯一索引兜底 + return ErrAccountAlreadyBound + } + return nil +} + +// issueJWT 为已绑定用户签发站点 JWT +func (s *GithubBindService) issueJWT(ctx context.Context, username string) (string, error) { + claims := &pkg.Claims{ + Sub: s.org + "/" + username, + Name: username, + } + // 尽量取 Casdoor 资料补全展示字段；失败则退化为 GitHub 绑定信息 + if profile, err := s.auth.GetProfile(ctx, s.org+"/"+username); err == nil { + claims.DisplayName = profile.DisplayName + claims.Email = profile.Email + claims.Avatar = profile.Avatar + } else if b, err := s.repo.GetByUsername(ctx, username); err == nil && b != nil { + logrus.WithField("username", username).Debug("fallback to github binding info for claims") + claims.DisplayName = b.GithubLogin + claims.Email = b.GithubEmail + claims.Avatar = b.GithubAvatar + } + token, err := pkg.SignJWT(claims, s.jwtSecret, s.jwtTTL) + if err != nil { + return "", fmt.Errorf("签发 token 失败: %w", err) + } + return token, nil +} diff --git a/backend/internal/service/migration_service.go b/backend/internal/service/migration_service.go new file mode 100644 index 0000000..cc992bb --- /dev/null +++ b/backend/internal/service/migration_service.go @@ -0,0 +1,219 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/amll-dev/amll-hub/backend/internal/config" + "github.com/amll-dev/amll-hub/backend/internal/infrastructure" + "github.com/amll-dev/amll-hub/backend/internal/model" + "github.com/amll-dev/amll-hub/backend/internal/repository" + "github.com/sirupsen/logrus" +) + +// MigrationUserMessage 投递到 migration.user 的消息体 +type MigrationUserMessage struct { + TaskID int64 `json:"task_id"` + Username string `json:"username"` + SubmitterInfo model.UserInfo `json:"submitter_info"` + GithubLogin string `json:"github_login"` + GithubID int64 `json:"github_id"` + SincePrNumber int `json:"since_pr_number"` +} + +// MigrationStatus 迁移页状态概览 +type MigrationStatus struct { + Bound bool `json:"bound"` + GithubLogin string `json:"githubLogin,omitempty"` + GithubEmail string `json:"githubEmail,omitempty"` + GithubAvatar string `json:"githubAvatar,omitempty"` + BoundAt *time.Time `json:"boundAt,omitempty"` + HasActive bool `json:"hasActive"` + ActiveTask *model.GithubMigrationTask `json:"activeTask,omitempty"` + LatestTask *model.GithubMigrationTask `json:"latestTask,omitempty"` + StartPrNumber int `json:"startPrNumber"` +} + +// MigrationList 已迁移 PR 列表（迁移页展示） +type MigrationList struct { + Total int `json:"total"` + Items []model.GithubMigratedPrItem `json:"items"` +} + +// MigrationService 投稿数据迁移业务逻辑（后端只创建任务并投递 MQ） +type MigrationService struct { + bindRepo *repository.GithubBindingRepo + migRepo *repository.GithubMigrationRepo + mq *infrastructure.RabbitMQ + cfg config.MigrationConfig +} + +// NewMigrationService 创建 MigrationService +func NewMigrationService( + bindRepo *repository.GithubBindingRepo, + migRepo *repository.GithubMigrationRepo, + mq *infrastructure.RabbitMQ, + cfg config.MigrationConfig, +) *MigrationService { + return &MigrationService{bindRepo: bindRepo, migRepo: migRepo, mq: mq, cfg: cfg} +} + +// Status 返回绑定状态与最近任务概览 +func (s *MigrationService) Status(ctx context.Context, username string) (*MigrationStatus, error) { + binding, err := s.bindRepo.GetByUsername(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + + st := &MigrationStatus{StartPrNumber: s.cfg.StartPrNumber} + if binding != nil { + st.Bound = true + st.GithubLogin = binding.GithubLogin + st.GithubEmail = binding.GithubEmail + st.GithubAvatar = binding.GithubAvatar + boundAt := binding.BoundAt + st.BoundAt = &boundAt + } + + active, err := s.migRepo.GetActiveTask(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + if active != nil { + st.HasActive = true + st.ActiveTask = active + } + + latest, err := s.migRepo.GetLatestTask(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + st.LatestTask = latest + return st, nil +} + +// Start 创建迁移任务并投递 migration.user +func (s *MigrationService) Start(ctx context.Context, username string, submitter model.UserInfo) (*model.GithubMigrationTask, error) { + binding, err := s.bindRepo.GetByUsername(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + if binding == nil { + return nil, ErrNotBound + } + + active, err := s.migRepo.GetActiveTask(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + if active != nil { + return nil, ErrMigrationInProgress + } + + task := &model.GithubMigrationTask{ + Username: username, + GithubLogin: binding.GithubLogin, + Status: model.MigrationStatusPending, + ClosePrStatus: model.MigrationStatusPending, + } + if err := s.migRepo.CreateTask(ctx, task); err != nil { + return nil, fmt.Errorf("create migration task: %w", err) + } + if submitter.Username == "" { + submitter.Username = username + } + if err := s.publish(ctx, task, binding.GithubID, submitter); err != nil { + _ = s.migRepo.MarkFailed(ctx, task.ID, err.Error()) + return nil, err + } + return task, nil +} + +// ListMigrated 查询该用户已迁移的 PR 列表与总数 +func (s *MigrationService) ListMigrated(ctx context.Context, username string, limit int) (*MigrationList, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + + total, err := s.migRepo.CountMigratedPrs(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + items, err := s.migRepo.ListMigratedPrs(ctx, username, limit) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + return &MigrationList{Total: int(total), Items: items}, nil +} + +// GetTask 查询任务进度（仅本人） +func (s *MigrationService) GetTask(ctx context.Context, username string, id int64) (*model.GithubMigrationTask, error) { + task, err := s.migRepo.GetTask(ctx, id) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + if task == nil || task.Username != username { + return nil, ErrMigrationTaskNotFound + } + return task, nil +} + +// Retry 重试任务：失败任务从 cursor 续传；已完成任务从头重新扫描（已迁移的会跳过） +func (s *MigrationService) Retry(ctx context.Context, username string, id int64, submitter model.UserInfo) (*model.GithubMigrationTask, error) { + task, err := s.GetTask(ctx, username, id) + if err != nil { + return nil, err + } + if task.Status == model.MigrationStatusRunning || task.Status == model.MigrationStatusPending { + return nil, ErrMigrationInProgress + } + + binding, err := s.bindRepo.GetByUsername(ctx, username) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + if binding == nil { + return nil, ErrNotBound + } + + // 失败续传保留 cursor；已完成任务重新迁移则从头扫描 + resetCursor := task.Status == model.MigrationStatusCompleted + if err := s.migRepo.ResetForRetry(ctx, task.ID, resetCursor); err != nil { + return nil, fmt.Errorf("reset migration task: %w", err) + } + + if submitter.Username == "" { + submitter.Username = username + } + if err := s.publish(ctx, task, binding.GithubID, submitter); err != nil { + _ = s.migRepo.MarkFailed(ctx, task.ID, err.Error()) + return nil, err + } + return s.GetTask(ctx, username, id) +} + +// publish 投递迁移任务消息 +func (s *MigrationService) publish(ctx context.Context, task *model.GithubMigrationTask, githubID int64, submitter model.UserInfo) error { + msg := MigrationUserMessage{ + TaskID: task.ID, + Username: task.Username, + SubmitterInfo: submitter, + GithubLogin: task.GithubLogin, + GithubID: githubID, + SincePrNumber: s.cfg.StartPrNumber, + } + body, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("marshal migration message: %w", err) + } + if err := s.mq.PublishMigrationUser(body, fmt.Sprintf("migration.user:%d", task.ID)); err != nil { + logrus.WithError(err).WithField("task_id", task.ID).Error("publish migration.user failed") + return fmt.Errorf("%w: %v", ErrUpstreamUnavailable, err) + } + return nil +} diff --git a/backend/migrations/001_init.up.sql b/backend/migrations/001_init.up.sql index ef195a1..7b01698 100644 --- a/backend/migrations/001_init.up.sql +++ b/backend/migrations/001_init.up.sql @@ -436,3 +436,70 @@ CREATE INDEX idx_notifications_created ON notifications (created_at DESC); ALTER TABLE notifications ADD CONSTRAINT ck_notifications_type CHECK (type IN ('system', 'review', 'submission', 'comment')); + +-- GitHub 绑定表：一个站点用户（JWT claims.name）最多绑定一个 GitHub 账号 +CREATE TABLE user_github_bindings ( + id BIGSERIAL PRIMARY KEY, + username VARCHAR(100) NOT NULL, + github_id BIGINT NOT NULL, + github_login VARCHAR(100) NOT NULL, + github_email VARCHAR(255), + github_avatar VARCHAR(500), + bound_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- 一个站点用户只能绑定一个 GitHub +CREATE UNIQUE INDEX uq_user_github_bindings_username ON user_github_bindings(username); +-- 一个 GitHub 账号只能绑定一个站点用户 +CREATE UNIQUE INDEX uq_user_github_bindings_github_id ON user_github_bindings(github_id); + +CREATE TRIGGER trg_user_github_bindings_updated_at + BEFORE UPDATE ON user_github_bindings + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +-- 已迁移 PR 记录表：全局去重核心，任何用户迁移时命中 pr_number 即跳过 +CREATE TABLE github_migrated_prs ( + id BIGSERIAL PRIMARY KEY, + pr_number INT NOT NULL, + username VARCHAR(100) NOT NULL, + submission_id BIGINT, + migrated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- 全局唯一：同一 PR 只迁移一次 +CREATE UNIQUE INDEX uq_github_migrated_prs_number ON github_migrated_prs(pr_number); +CREATE INDEX idx_github_migrated_prs_username ON github_migrated_prs(username); + +-- 迁移任务与进度表 +CREATE TABLE github_migration_tasks ( + id BIGSERIAL PRIMARY KEY, + username VARCHAR(100) NOT NULL, + github_login VARCHAR(100) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + total_prs INT NOT NULL DEFAULT 0, + processed_prs INT NOT NULL DEFAULT 0, + created_count INT NOT NULL DEFAULT 0, + skipped_count INT NOT NULL DEFAULT 0, + failed_count INT NOT NULL DEFAULT 0, + cursor VARCHAR(64), + error TEXT, + close_pr_status VARCHAR(20) NOT NULL DEFAULT 'pending', + closed_pr_count INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +COMMENT ON COLUMN github_migration_tasks.status IS 'pending/running/completed/failed'; +COMMENT ON COLUMN github_migration_tasks.close_pr_status IS 'pending/running/completed/failed'; +COMMENT ON COLUMN github_migration_tasks.cursor IS '断点游标（GraphQL endCursor），重试时从此继续'; + +CREATE INDEX idx_github_migration_tasks_user ON github_migration_tasks(username, created_at DESC); +CREATE INDEX idx_github_migration_tasks_status ON github_migration_tasks(status); + +CREATE TRIGGER trg_github_migration_tasks_updated_at + BEFORE UPDATE ON github_migration_tasks + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); diff --git a/frontend/src/components/AuthDialog.tsx b/frontend/src/components/AuthDialog.tsx index b13ed5c..4c27098 100644 --- a/frontend/src/components/AuthDialog.tsx +++ b/frontend/src/components/AuthDialog.tsx @@ -12,7 +12,7 @@ import { import { useForm } from 'react-hook-form'; import { zodResolver } from '@hookform/resolvers/zod'; import { AnimatePresence, motion } from 'framer-motion'; -import { ArrowLeft, Eye, EyeOff, Loader2, Mail, X } from 'lucide-react'; +import { ArrowLeft, Eye, EyeOff, Github, Loader2, Mail, X } from 'lucide-react'; import { z } from 'zod'; import { useAuth } from '@/hooks/useAuth'; import { useAliyunCaptcha } from '@/hooks/useAliyunCaptcha'; @@ -707,6 +707,15 @@ export function AuthDialog() { 其他方式登录
+ {/* GitHub 登录：整页跳转到后端 OAuth 授权入口 */} + handleClose()} + className="flex flex-col items-center gap-1 text-ink-3 transition-colors hover:text-primary" + > + + GitHub 登录 + + +
+ + 或 + +
+ + + 注册新账号并绑定 + + + + 忘记密码？ + + + + + )} + + {infoQuery.isError && !expired && ( +

+ {infoQuery.error instanceof Error ? infoQuery.error.message : '加载失败'} +

+ )} +
+ + ); +} diff --git a/frontend/src/pages/GithubCallback.tsx b/frontend/src/pages/GithubCallback.tsx new file mode 100644 index 0000000..c288394 --- /dev/null +++ b/frontend/src/pages/GithubCallback.tsx @@ -0,0 +1,58 @@ +import { useEffect, useMemo, useState } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; +import { logout } from '@/atoms/auth'; +import { setToken } from '@/lib/auth'; +import { Button, buttonVariants } from '@/components/ui/button'; +import { PageContainer } from '@/components/PageContainer'; + +/** + * GitHub OAuth 回调落地页。 + * 后端已绑定站点账号时 302 到 /auth/github/callback#token=。 + * 令牌放在 fragment 中（不会发送给服务器），本页从 hash 读取并写入， + * 随后由 AuthBoot 拉取 profile 并写入登录态。 + */ +export function GithubCallback() { + const navigate = useNavigate(); + const token = useMemo(() => { + const hash = window.location.hash.replace(/^#/, ''); + return new URLSearchParams(hash).get('token') ?? ''; + }, []); + const [failed, setFailed] = useState(false); + + useEffect(() => { + if (!token) { + setFailed(true); + return; + } + // 换账号登录：先清掉旧登录态与缓存，再写入新 token + logout(); + setToken(token); + navigate('/', { replace: true }); + }, [token, navigate]); + + return ( + + {failed ? ( + <> +

登录失败

+

未获取到登录凭证，请重新发起 GitHub 登录

+ + 返回首页 + + + ) : ( + <> +

正在登录…

+

正在完成 GitHub 授权，请稍候

+ + + )} + + ); +} diff --git a/frontend/src/pages/Register.tsx b/frontend/src/pages/Register.tsx index 283d87c..a8dc913 100644 --- a/frontend/src/pages/Register.tsx +++ b/frontend/src/pages/Register.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import { Link, useNavigate } from 'react-router-dom'; import { useMutation, useQuery } from '@tanstack/react-query'; import { useAtom } from 'jotai'; @@ -19,6 +19,7 @@ import { useAuth } from '@/hooks/useAuth'; import { useAliyunCaptcha } from '@/hooks/useAliyunCaptcha'; import { useCountdown } from '@/hooks/useCountdown'; import { api } from '@/lib/api'; +import { compressAvatar } from '@/lib/image'; import { queryKeys } from '@/lib/query'; import { buttonTap } from '@/lib/motion'; import type { CaptchaConfig } from '@/lib/auth'; @@ -35,6 +36,7 @@ import { import type { Control, FieldPath } from 'react-hook-form'; import type { ReactNode } from 'react'; import { PageContainer } from '@/components/PageContainer'; +import { AvatarUploadDialog } from '@/components/profile/AvatarUploadDialog'; /** 注册表单 schema：字段必填 + 密码长度 + 两次密码一致 */ const registerSchema = z @@ -63,6 +65,11 @@ const codeLabelClass = 'mb-2 block text-sm font-normal text-ink-2'; export function Register() { const navigate = useNavigate(); const { login } = useAuth(); + // 来自 GitHub 绑定流程的凭证（/register#token=，放在 fragment 中避免入日志） + const bindToken = useMemo(() => { + const hash = window.location.hash.replace(/^#/, ''); + return new URLSearchParams(hash).get('token') ?? ''; + }, []); // 页面状态存全局 atoms（卸载时复位，语义同原 useState） const [avatarFile, setAvatarFile] = useAtom(avatarFileAtom); @@ -80,6 +87,10 @@ export function Register() { useEffect(() => () => resetRegisterForm(), []); const fileInputRef = useRef(null); + // 待裁剪的原图：确认缩放后才写回 avatarFile，保证上传的是裁切压缩后的文件 + const [rawAvatarFile, setRawAvatarFile] = useState(null); + const [avatarDialogOpen, setAvatarDialogOpen] = useState(false); + const [avatarProcessing, setAvatarProcessing] = useState(false); const form = useForm({ resolver: zodResolver(registerSchema), @@ -111,6 +122,36 @@ export function Register() { : (captchaQuery.data ?? { type: 'none' }); const captchaLoading = captchaQuery.isFetching; + // GitHub 绑定场景：拉取 GitHub 资料用于预填 + const bindInfoQuery = useQuery({ + queryKey: ['github-bind-info', bindToken], + queryFn: () => api.getGithubBindInfo(bindToken), + enabled: !!bindToken, + retry: false, + }); + const bindInfo = bindInfoQuery.data; + + useEffect(() => { + if (!bindInfo) return; + form.reset({ + username: bindInfo.githubLogin, + displayName: bindInfo.githubLogin, + email: bindInfo.githubEmail || '', + password: '', + confirmPassword: '', + phone: '', + phoneCode: '', + emailCode: '', + }); + }, [bindInfo, form]); + + useEffect(() => { + if (bindInfoQuery.isError) { + setError('授权已过期，请重新使用 GitHub 登录'); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [bindInfoQuery.isError]); + const { captchaVerifyParam, isReady: captchaReady, @@ -125,9 +166,11 @@ export function Register() { // 倒计时由 useCountdown 提供 - // 头像选择 + // 头像选择：先弹裁剪框调缩放，确认后再压缩（与个人中心「更换头像」一致） const handleAvatarChange = (e: React.ChangeEvent) => { const file = e.target.files?.[0]; + // 清空 value，取消弹窗后重选同一张图也能再次触发 change + e.target.value = ''; if (!file) return; if (!file.type.startsWith('image/')) { setError('请选择图片文件'); @@ -138,8 +181,27 @@ export function Register() { return; } setError(''); - setAvatarFile(file); - setAvatarPreview(URL.createObjectURL(file)); + setRawAvatarFile(file); + setAvatarDialogOpen(true); + }; + + const closeAvatarDialog = () => { + setAvatarDialogOpen(false); + setRawAvatarFile(null); + }; + + const handleAvatarConfirm = async (zoom: number) => { + if (!rawAvatarFile) return; + setAvatarProcessing(true); + try { + const compressed = await compressAvatar(rawAvatarFile, zoom); + if (avatarPreview) URL.revokeObjectURL(avatarPreview); + setAvatarFile(compressed); + setAvatarPreview(URL.createObjectURL(compressed)); + closeAvatarDialog(); + } finally { + setAvatarProcessing(false); + } }; // 实际发送验证码（target: phone / email） @@ -222,8 +284,21 @@ export function Register() { }, [captchaVerifyParam, captchaModalOpen]); // 提交注册：注册 → 自动登录 → 上传头像（失败不阻塞） + // GitHub 绑定场景改为调用 bind/register，后端直接返回登录态与绑定结果 const registerMutation = useMutation({ mutationFn: async (vars: RegisterValues) => { + if (bindToken) { + return api.githubBindRegister({ + token: bindToken, + username: vars.username, + password: vars.password, + phone: vars.phone, + code: vars.phoneCode, + email: vars.email, + emailCode: vars.emailCode, + displayName: vars.displayName, + }); + } await api.register({ username: vars.username, password: vars.password, @@ -233,24 +308,23 @@ export function Register() { emailCode: vars.emailCode, displayName: vars.displayName, }); - return { username: vars.username, password: vars.password }; + try { + return await api.login(vars.username, vars.password); + } catch { + throw new Error('注册成功，请手动登录'); + } }, onMutate: () => setError(''), - onSuccess: async ({ username, password: pwd }) => { - try { - const result = await api.login(username, pwd); - login(result.token, result.user); - if (avatarFile) { - try { - await api.uploadAvatar(avatarFile); - } catch { - // 头像上传失败不阻塞 - } + onSuccess: async (result) => { + login(result.token, result.user); + if (avatarFile) { + try { + await api.uploadAvatar(avatarFile); + } catch { + // 头像上传失败不阻塞 } - navigate('/profile'); - } catch { - setError('注册成功，请手动登录'); } + navigate(bindToken ? '/profile/migration' : '/profile'); }, onError: (e: Error) => setError(e.message || '注册失败'), }); @@ -279,9 +353,29 @@ export function Register() {

- 注册账号 + {bindToken ? '注册并绑定 GitHub' : '注册账号'}

+ {bindInfo && ( +
+ {bindInfo.githubAvatar ? ( + {bindInfo.githubLogin} + ) : null} +
+

+ 即将绑定：{bindInfo.githubLogin} +

+ {bindInfo.githubEmail && ( +

{bindInfo.githubEmail}

+ )} +
+
+ )} +
registerMutation.mutate(v))} @@ -301,8 +395,12 @@ export function Register() { onClick={() => fileInputRef.current?.click()} className="relative flex h-20 w-20 items-center justify-center overflow-hidden rounded-full border-2 border-line bg-surface-2 transition-colors hover:border-primary" > - {avatarPreview ? ( - 头像预览 + {avatarPreview || bindInfo?.githubAvatar ? ( + 头像预览 ) : (
@@ -311,7 +409,19 @@ export function Register() { )}
-

头像可选，其余均为必填

+

+ {bindInfo?.githubAvatar && !avatarPreview + ? '将使用你的 GitHub 头像，点击可更换' + : '头像可选，其余均为必填'} +

+ + {/* 两列布局：用户名 + 昵称 */}
diff --git a/frontend/src/pages/profile/ProfileLayout.tsx b/frontend/src/pages/profile/ProfileLayout.tsx index b975347..4cae805 100644 --- a/frontend/src/pages/profile/ProfileLayout.tsx +++ b/frontend/src/pages/profile/ProfileLayout.tsx @@ -1,7 +1,7 @@ import { Suspense } from 'react'; import { NavLink, Outlet, useLocation } from 'react-router-dom'; import { motion } from 'framer-motion'; -import { Home, Loader2, ShieldCheck, User, type LucideIcon } from 'lucide-react'; +import { Github, Home, Loader2, ShieldCheck, User, type LucideIcon } from 'lucide-react'; import { useAuth } from '@/hooks/useAuth'; import { buttonTap } from '@/lib/motion'; import { cn } from '@/lib/utils'; @@ -20,6 +20,7 @@ const navItems: NavItem[] = [ { to: '/profile', label: '首页', icon: Home, end: true }, { to: '/profile/info', label: '我的信息', icon: User }, { to: '/profile/security', label: '账号安全', icon: ShieldCheck }, + { to: '/profile/migration', label: '数据迁移', icon: Github }, ]; /** 个人中心布局 */ diff --git a/frontend/src/pages/profile/ProfileMigration.tsx b/frontend/src/pages/profile/ProfileMigration.tsx new file mode 100644 index 0000000..6d24b1c --- /dev/null +++ b/frontend/src/pages/profile/ProfileMigration.tsx @@ -0,0 +1,374 @@ +import { useEffect, useMemo } from 'react'; +import { Link, useNavigate, useSearchParams } from 'react-router-dom'; +import { useMutation, useQuery } from '@tanstack/react-query'; +import { toast } from 'sonner'; +import { FileText, Github, Loader2, RefreshCw, Unlink, Upload } from 'lucide-react'; +import { useAuth } from '@/hooks/useAuth'; +import { api } from '@/lib/api'; +import { queryKeys } from '@/lib/query'; +import { buttonTap } from '@/lib/motion'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { SectionCard } from '@/components/profile/SectionCard'; +import type { GithubMigrationStatus, GithubMigrationTask } from '@/lib/types'; + +/** 任务状态 → 中文标签 */ +const STATUS_LABEL: Record = { + pending: '排队中', + running: '迁移中', + completed: '已完成', + failed: '失败', +}; + +function formatTime(iso?: string): string { + if (!iso) return '-'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return '-'; + return d.toLocaleString('zh-CN', { hour12: false }); +} + +/** 迁移进度明细 */ +function TaskProgress({ + task, + startPrNumber, +}: { + task: GithubMigrationTask; + startPrNumber: number; +}) { + const percent = + task.totalPrs > 0 ? Math.min(100, Math.round((task.processedPrs / task.totalPrs) * 100)) : 0; + + return ( +
+
+
+ + {STATUS_LABEL[task.status]} + + + #{startPrNumber} 之后的投稿 · 任务 ID {task.id} + +
+ + {task.processedPrs} / {task.totalPrs} + +
+ + {/* 进度条 */} +
+
+
+ +
+
+
迁移的投稿数
+
+ {task.createdCount} +
+
+
+
跳过（已迁移）
+
+ {task.skippedCount} +
+
+
+
失败
+
+ {task.failedCount} +
+
+
+ + {task.completedAt && ( +

完成于 {formatTime(task.completedAt)}

+ )} + + {task.error &&

错误：{task.error}

} +
+ ); +} + +/** 个人中心 → 数据迁移 */ +export function ProfileMigration() { + const { user } = useAuth(); + const navigate = useNavigate(); + const [params] = useSearchParams(); + const justBound = params.get('githubBound') === '1'; + + const statusQuery = useQuery({ + queryKey: queryKeys.githubMigration, + queryFn: () => api.getMigrationStatus(), + enabled: !!user, + // 有进行中任务时轮询进度 + refetchInterval: (query) => (query.state.data?.hasActive ? 2000 : false), + retry: false, + }); + + const status: GithubMigrationStatus | undefined = statusQuery.data; + + // 已迁移投稿列表（含 PR 标题）；迁移进行中时同步轮询 + const migratedQuery = useQuery({ + queryKey: queryKeys.githubMigrated, + queryFn: () => api.getMigratedPrs(100), + enabled: !!user && !!status?.bound, + refetchInterval: status?.hasActive ? 5000 : false, + retry: false, + }); + const migratedList = migratedQuery.data; + + // 绑定回跳提示：显示一次后清掉 query 参数 + useEffect(() => { + if (!justBound) return; + toast.success('GitHub 绑定成功'); + navigate('/profile/migration', { replace: true }); + }, [justBound, navigate]); + + const startMutation = useMutation({ + mutationFn: () => api.startMigration(), + onSuccess: () => { + toast.success('迁移任务已创建'); + void statusQuery.refetch(); + void migratedQuery.refetch(); + }, + onError: (e: Error) => toast.error(e.message || '创建迁移任务失败'), + }); + + const retryMutation = useMutation({ + mutationFn: (id: number) => api.retryMigration(id), + onSuccess: () => { + toast.success('已重新开始迁移'); + void statusQuery.refetch(); + void migratedQuery.refetch(); + }, + onError: (e: Error) => toast.error(e.message || '重试失败'), + }); + + const unbindMutation = useMutation({ + mutationFn: () => api.unbindGithub(), + onSuccess: () => { + toast.success('已解绑 GitHub'); + void statusQuery.refetch(); + }, + onError: (e: Error) => toast.error(e.message || '解绑失败'), + }); + + const currentTask: GithubMigrationTask | undefined = useMemo( + () => status?.activeTask ?? status?.latestTask, + [status] + ); + + if (!user) return null; + + if (statusQuery.isPending) { + return ( +
+ + 加载中… +
+ ); + } + + if (statusQuery.isError && !status) { + return ( + } title="数据迁移"> +

+ {statusQuery.error instanceof Error ? statusQuery.error.message : '加载失败'} +

+ + + ); + } + + if (!status) return null; + + const busy = startMutation.isPending || retryMutation.isPending; + const canStart = status.bound && !status.hasActive && !busy; + const canRetry = status.bound && !status.hasActive && currentTask?.status === 'failed'; + + return ( +
+ {/* 绑定状态 */} + } + title="GitHub 账号" + description="迁移前需先绑定你在 amll-ttml-db 提交歌词所用的 GitHub 账号" + > + {status.bound ? ( +
+
+ {status.githubAvatar ? ( + {status.githubLogin} + ) : ( + + + + )} +
+

{status.githubLogin}

+

+ {status.githubEmail || '未公开邮箱'} · 绑定于 {formatTime(status.boundAt)} +

+
+
+ +
+ ) : ( +
+

尚未绑定 GitHub 账号，绑定后才能开始迁移

+ +
+ )} + + + {/* 迁移任务 */} + } + title="投稿数据迁移" + description={`将 GitHub 上 #${status.startPrNumber} 之后的投稿迁移为站点原生投稿，已迁移的会自动跳过`} + > + {currentTask ? ( +
+ + +
+ + {canRetry && ( + + )} +
+ + {status.hasActive && ( +

+ 迁移进行中，数据量较大时请保持页面打开或稍后回来查看。 +

+ )} + {currentTask.status === 'completed' && ( +

+ 数据截止到 {formatTime(currentTask.completedAt ?? currentTask.updatedAt)}。 + 如需同步此后更新的投稿，请点击「重新迁移」。 +

+ )} +
+ ) : ( +
+

+ {status.bound + ? '尚未执行过迁移，点击下方按钮开始。' + : '请先绑定 GitHub 账号后再开始迁移。'} +

+ +
+ )} + + + {/* 已迁移投稿（含原始 PR 标题） */} + {status.bound && ( + } + title="已迁移投稿" + description="已从 GitHub 迁移到站点的投稿列表，标题为该投稿对应的 PR 标题" + > + {migratedQuery.isPending ? ( +
+ + 加载中… +
+ ) : migratedList && migratedList.total > 0 ? ( + <> +

+ 共{' '} + + {migratedList.total} + {' '} + 个 + {migratedList.total > migratedList.items.length && ( + （仅展示最近 {migratedList.items.length} 条） + )} +

+
    + {migratedList.items.map((item) => ( +
  • +
    +

    + {item.prTitle || item.submissionTitle || `PR #${item.prNumber}`} +

    +

    + PR #{item.prNumber} + {item.submissionTitle && item.prTitle ? ` · ${item.submissionTitle}` : ''} +

    +
    + {(item.submissionId || item.fileName) && ( + + 查看 + + )} +
    • + ))} +
+ + ) : ( +

暂无已迁移的投稿

+ )} + + )} +
+ ); +} diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index a76e666..6bf3e8a 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -3,8 +3,10 @@ import { Layout } from '@/App'; import { RouteErrorBoundary } from '@/components/ErrorBoundary'; import { LyricEditorShell } from '@/components/review/LyricEditorShell'; import { + BindAccount, CreatorCenter, DailyRecommend, + GithubCallback, Home, LyricEditorPage, MessagesPage, @@ -16,6 +18,7 @@ import { ProfileHome, ProfileInfo, ProfileLayout, + ProfileMigration, ProfileSecurity, Ranking, Register, @@ -38,6 +41,12 @@ export const router = createBrowserRouter([ ), errorElement: , }, + // GitHub OAuth 回调落地页：不套站点布局，写入 token 后立即跳转 + { + path: '/auth/github/callback', + element: , + errorElement: , + }, { element: , // 懒加载页面抛错 / chunk 加载失败时整树兜底，避免白屏 @@ -59,8 +68,10 @@ export const router = createBrowserRouter([ { index: true, element: }, { path: 'info', element: }, { path: 'security', element: }, + { path: 'migration', element: }, ], }, + { path: '/bind-account', element: }, { path: '/messages', element: }, { path: '/creator', element: }, { diff --git a/frontend/src/routes.tsx b/frontend/src/routes.tsx index aedba0d..9d5f7e7 100644 --- a/frontend/src/routes.tsx +++ b/frontend/src/routes.tsx @@ -21,6 +21,12 @@ const loadProfileInfo = () => import('@/pages/profile/ProfileInfo').then((m) => ({ default: m.ProfileInfo })); const loadProfileSecurity = () => import('@/pages/profile/ProfileSecurity').then((m) => ({ default: m.ProfileSecurity })); +const loadProfileMigration = () => + import('@/pages/profile/ProfileMigration').then((m) => ({ default: m.ProfileMigration })); +const loadBindAccount = () => + import('@/pages/BindAccount').then((m) => ({ default: m.BindAccount })); +const loadGithubCallback = () => + import('@/pages/GithubCallback').then((m) => ({ default: m.GithubCallback })); const loadMessagesPage = () => import('@/pages/MessagesPage').then((m) => ({ default: m.MessagesPage })); const loadCreatorCenter = () => @@ -53,6 +59,9 @@ export const ProfileLayout = lazy(loadProfileLayout); export const ProfileHome = lazy(loadProfileHome); export const ProfileInfo = lazy(loadProfileInfo); export const ProfileSecurity = lazy(loadProfileSecurity); +export const ProfileMigration = lazy(loadProfileMigration); +export const BindAccount = lazy(loadBindAccount); +export const GithubCallback = lazy(loadGithubCallback); export const MessagesPage = lazy(loadMessagesPage); export const CreatorCenter = lazy(loadCreatorCenter); export const ReviewCenter = lazy(loadReviewCenter); @@ -80,6 +89,8 @@ export const pagePreloads: Record Promise> = { '/profile': loadProfileLayout, '/profile/info': loadProfileInfo, '/profile/security': loadProfileSecurity, + '/profile/migration': loadProfileMigration, + '/bind-account': loadBindAccount, '/messages': loadMessagesPage, '/creator': loadCreatorCenter, '/creator/lyrics/detail': loadSubmissionDetailPage, diff --git a/worker/Cargo.lock b/worker/Cargo.lock index fcc7558..307b659 100644 --- a/worker/Cargo.lock +++ b/worker/Cargo.lock @@ -2699,9 +2699,11 @@ checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ "base64 0.22.1", "js-sys", + "pem", "ring", "serde", "serde_json", + "simple_asn1", ] [[package]] @@ -3204,6 +3206,16 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4565,6 +4577,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.18", + "time", +] + [[package]] name = "slab" version = "0.4.12" @@ -5414,6 +5438,7 @@ dependencies = [ "dotenvy", "futures-lite 2.6.1", "indexmap 2.14.0", + "jsonwebtoken", "lapin", "meilisearch-sdk", "pinyin", diff --git a/worker/Cargo.toml b/worker/Cargo.toml index 3d10c82..521f7b0 100644 --- a/worker/Cargo.toml +++ b/worker/Cargo.toml @@ -51,6 +51,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } config = "0.14" dotenvy = "0.15" +# GitHub App JWT（RS256 签发，用于换取 installation access token） +jsonwebtoken = "9" + # Errors anyhow = "1" diff --git a/worker/src/config.rs b/worker/src/config.rs index 63a3274..6fa9afa 100644 --- a/worker/src/config.rs +++ b/worker/src/config.rs @@ -11,6 +11,9 @@ pub struct Config { pub rabbitmq: RabbitMqConfig, pub meilisearch: MeiliSearchConfig, pub github: GitHubConfig, + pub github_app: GitHubAppConfig, + pub casdoor: CasdoorConfig, + pub migration: MigrationConfig, pub worker: WorkerConfig, pub ncm: NcmConfig, } @@ -102,6 +105,16 @@ pub struct RabbitMqConfig { pub nf_queue: String, #[serde(default = "default_nf_dlq")] pub nf_dlq: String, + /// 迁移队列（消费 migration.user） + #[serde(default = "default_migration_queue")] + pub migration_queue: String, + #[serde(default = "default_migration_dlq")] + pub migration_dlq: String, + /// 关闭 PR 队列（消费 migration.close_pr） + #[serde(default = "default_migration_close_queue")] + pub migration_close_queue: String, + #[serde(default = "default_migration_close_dlq")] + pub migration_close_dlq: String, } fn default_dlq() -> String { @@ -113,6 +126,96 @@ fn default_nf_queue() -> String { fn default_nf_dlq() -> String { "not_found_parse_queue.dlq".to_string() } +fn default_migration_queue() -> String { + "migration.user".to_string() +} +fn default_migration_dlq() -> String { + "migration.user.dlq".to_string() +} +fn default_migration_close_queue() -> String { + "migration.close_pr".to_string() +} +fn default_migration_close_dlq() -> String { + "migration.close_pr.dlq".to_string() +} + +/// 迁移队列的 exchange / DLX（与后端 RabbitMQ 拓扑保持一致） +pub const MIGRATION_EXCHANGE: &str = "ttml.migration"; +pub const MIGRATION_DLX_EXCHANGE: &str = "ttml.migration.dlx"; +pub const MIGRATION_DLQ_ROUTING: &str = "migration.failed"; +pub const MIGRATION_USER_ROUTING: &str = "migration.user"; +pub const MIGRATION_CLOSE_ROUTING: &str = "migration.close_pr"; + +/// GitHub App 配置（迁移统一使用安装令牌） +#[derive(Debug, Clone, Deserialize)] +pub struct GitHubAppConfig { + #[serde(default)] + pub app_id: u64, + #[serde(default)] + pub installation_id: u64, + #[serde(default)] + pub private_key_path: String, +} + +impl GitHubAppConfig { + /// 三项配置齐全才算启用 + pub fn enabled(&self) -> bool { + self.app_id != 0 && self.installation_id != 0 && !self.private_key_path.is_empty() + } +} + +/// Casdoor 配置（迁移写库时把 GitHub login 解析为站点用户展示名/头像） +#[derive(Debug, Clone, Deserialize)] +pub struct CasdoorConfig { + #[serde(default)] + pub endpoint: String, + #[serde(default)] + pub client_id: String, + #[serde(default)] + pub client_secret: String, + #[serde(default)] + pub organization: String, +} + +impl CasdoorConfig { + /// 四项齐全才可用；缺失时回退为 GitHub 原始信息 + pub fn enabled(&self) -> bool { + !self.endpoint.is_empty() + && !self.client_id.is_empty() + && !self.client_secret.is_empty() + && !self.organization.is_empty() + } +} + +/// 投稿数据迁移配置 +#[derive(Debug, Clone, Deserialize)] +pub struct MigrationConfig { + /// 迁移起始 PR 号（小于该号的 PR 丢弃）；实际以消息体 since_pr_number 为准 + #[serde(default = "default_migration_start_pr")] + pub start_pr_number: i64, + /// GitHub 请求并发上限（≤10） + #[serde(default = "default_migration_concurrency")] + pub concurrency: usize, + /// 每处理多少条刷新一次进度与断点游标 + #[serde(default = "default_migration_batch_size")] + pub batch_size: usize, + /// 前端站点地址，用于关闭 PR 时的迁移地址评论 + #[serde(default)] + pub site_url: String, + /// 是否在迁移完成后自动关闭已迁移的 GitHub PR（默认关闭） + #[serde(default)] + pub close_pr_enabled: bool, +} + +fn default_migration_start_pr() -> i64 { + 7390 +} +fn default_migration_concurrency() -> usize { + 10 +} +fn default_migration_batch_size() -> usize { + 20 +} #[derive(Debug, Clone, Deserialize)] pub struct MeiliSearchConfig { @@ -217,7 +320,14 @@ fn find_dotenv() -> Option { pub fn load() -> anyhow::Result { let dotenv_path = find_dotenv(); if let Some(ref path) = dotenv_path { - let _ = dotenvy::from_path(path); + // 解析失败必须显式告警：dotenvy 遇到非法行会中断，其后的变量全部不会生效 + if let Err(e) = dotenvy::from_path(path) { + tracing::warn!( + error = %e, + path = %path.display(), + ".env 解析失败，该行之后的变量不会生效（常见原因：双引号内出现 \\a \\b 等非法转义）" + ); + } tracing::info!(path = %path.display(), "加载 .env 文件"); } else { tracing::warn!("未找到 .env 文件，使用默认配置"); @@ -275,6 +385,22 @@ pub fn load() -> anyhow::Result { .set_override("rabbitmq.dlq", env_or("RABBITMQ_DLQ", "sync_queue.dlq"))? .set_override("rabbitmq.nf_queue", env_or("RABBITMQ_NF_QUEUE", "not_found_parse_queue"))? .set_override("rabbitmq.nf_dlq", env_or("RABBITMQ_NF_DLQ", "not_found_parse_queue.dlq"))? + .set_override( + "rabbitmq.migration_queue", + env_or("RABBITMQ_MIGRATION_QUEUE", "migration.user"), + )? + .set_override( + "rabbitmq.migration_dlq", + env_or("RABBITMQ_MIGRATION_DLQ", "migration.user.dlq"), + )? + .set_override( + "rabbitmq.migration_close_queue", + env_or("RABBITMQ_MIGRATION_CLOSE_QUEUE", "migration.close_pr"), + )? + .set_override( + "rabbitmq.migration_close_dlq", + env_or("RABBITMQ_MIGRATION_CLOSE_DLQ", "migration.close_pr.dlq"), + )? .set_override( "meilisearch.host", env_or("MEILISEARCH_HOST", "http://localhost:7700"), @@ -287,6 +413,59 @@ pub fn load() -> anyhow::Result { env_or("GITHUB_REPO", "amll-dev/amll-ttml-db"), )? .set_override("github.branch", env_or("GITHUB_BRANCH", "main"))? + .set_override( + "github_app.app_id", + env_or("GITHUB_APP_ID", "0").parse::().unwrap_or(0), + )? + .set_override( + "github_app.installation_id", + env_or("GITHUB_INSTALLATION_ID", "0") + .parse::() + .unwrap_or(0), + )? + .set_override( + "github_app.private_key_path", + env_or("GITHUB_PRIVATE_KEY_PATH", ""), + )? + .set_override("casdoor.endpoint", env_or("CASDOOR_ENDPOINT", ""))? + .set_override("casdoor.client_id", env_or("CASDOOR_CLIENT_ID", ""))? + .set_override( + "casdoor.client_secret", + env_or("CASDOOR_CLIENT_SECRET", ""), + )? + .set_override( + "casdoor.organization", + env_or("CASDOOR_ORGANIZATION", "amll-ttml-user"), + )? + .set_override( + "migration.start_pr_number", + env_or("MIGRATION_PR_START_NUMBER", "7390") + .parse::() + .unwrap_or(7390), + )? + .set_override( + "migration.concurrency", + env_or("MIGRATION_CONCURRENCY", "10") + .parse::() + .unwrap_or(10), + )? + .set_override( + "migration.batch_size", + env_or("MIGRATION_BATCH_SIZE", "20") + .parse::() + .unwrap_or(20), + )? + .set_override("migration.site_url", env_or("SITE_URL", ""))? + .set_override( + "migration.close_pr_enabled", + matches!( + env_or("MIGRATION_CLOSE_PR_ENABLED", "false") + .trim() + .to_ascii_lowercase() + .as_str(), + "1" | "true" | "yes" | "on" + ), + )? .set_override( "worker.concurrency", env_or("WORKER_CONCURRENCY", "20") diff --git a/worker/src/infra/casdoor.rs b/worker/src/infra/casdoor.rs new file mode 100644 index 0000000..2f1a27d --- /dev/null +++ b/worker/src/infra/casdoor.rs @@ -0,0 +1,73 @@ +use anyhow::{Context, Result}; +use serde::Deserialize; + +use crate::config::CasdoorConfig; + +/// Casdoor 用户（仅取站点 UserInfo 需要的字段） +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CasdoorUser { + #[serde(default)] + pub name: String, + #[serde(default)] + pub display_name: String, + #[serde(default)] + pub avatar: String, +} + +#[derive(Debug, Deserialize)] +struct CasdoorResponse { + #[serde(default)] + status: String, + #[serde(default)] + data: Option, +} + +/// Casdoor HTTP 客户端 +/// +/// 对齐 backend/internal/infrastructure/casdoor.go 的 GetUser： +/// `GET {endpoint}/api/get-user?id={organization}/{name}`，Basic Auth(client_id:client_secret) +pub struct CasdoorClient { + cfg: CasdoorConfig, + http: reqwest::Client, +} + +impl CasdoorClient { + pub fn new(cfg: CasdoorConfig) -> Result { + let http = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(15)) + .build() + .context("build casdoor http client")?; + Ok(Self { cfg, http }) + } + + /// 按用户名查询用户；用户不存在时返回 `Ok(None)`（Casdoor 以 status=error 表示） + pub async fn get_user(&self, owner: &str, name: &str) -> Result> { + let url = format!( + "{}/api/get-user?id={}/{}", + self.cfg.endpoint.trim_end_matches('/'), + urlencoding::encode(owner), + urlencoding::encode(name) + ); + let resp = self + .http + .get(&url) + .basic_auth(&self.cfg.client_id, Some(&self.cfg.client_secret)) + .send() + .await + .with_context(|| format!("请求 Casdoor 用户失败: {}", url))?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if !status.is_success() { + anyhow::bail!("Casdoor 返回错误状态: status={} body={}", status, body); + } + + let parsed: CasdoorResponse = serde_json::from_str(&body) + .with_context(|| format!("解析 Casdoor 响应失败: {}", body))?; + if parsed.status != "ok" { + return Ok(None); + } + Ok(parsed.data) + } +} diff --git a/worker/src/infra/github_app.rs b/worker/src/infra/github_app.rs new file mode 100644 index 0000000..41fe97a --- /dev/null +++ b/worker/src/infra/github_app.rs @@ -0,0 +1,138 @@ +use anyhow::{Context, Result}; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use crate::config::GitHubAppConfig; + +/// GitHub App 客户端：RS256 签发 App JWT → 换取 installation access token → 内存缓存 +/// +/// 逻辑对齐后端 backend/internal/infrastructure/github_app.go： +/// - App JWT 有效期 ≤ 10 分钟（iss = AppID） +/// - 安装令牌在过期前 5 分钟自动刷新 +pub struct GithubAppClient { + cfg: GitHubAppConfig, + key: Option, + http: reqwest::Client, + cached: Mutex>, +} + +struct CachedToken { + token: String, + expires_at: chrono::DateTime, +} + +#[derive(Serialize)] +struct AppClaims { + iat: i64, + exp: i64, + iss: String, +} + +#[derive(Deserialize)] +struct TokenResponse { + token: String, + expires_at: chrono::DateTime, +} + +impl GithubAppClient { + /// 创建客户端：私钥缺失或不可读时返回错误（调用方据此决定是否启用迁移功能） + pub fn new(cfg: GitHubAppConfig) -> Result { + let key = if cfg.private_key_path.is_empty() { + None + } else { + let pem = std::fs::read(&cfg.private_key_path) + .with_context(|| format!("读取 GitHub App 私钥失败: {}", cfg.private_key_path))?; + let key = + EncodingKey::from_rsa_pem(&pem).context("解析 GitHub App RSA 私钥失败（需 PEM 格式）")?; + Some(key) + }; + + let http = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(15)) + .build() + .context("build github app http client")?; + + Ok(Self { + cfg, + key, + http, + cached: Mutex::new(None), + }) + } + + /// 是否可用（AppID / InstallationID / 私钥均就绪） + pub fn enabled(&self) -> bool { + self.cfg.app_id != 0 && self.cfg.installation_id != 0 && self.key.is_some() + } + + /// 获取 installation access token（带缓存，过期前 5 分钟刷新） + pub async fn installation_token(&self) -> Result { + if !self.enabled() { + anyhow::bail!( + "GitHub App 未配置：需要 GITHUB_APP_ID / GITHUB_INSTALLATION_ID / GITHUB_PRIVATE_KEY_PATH" + ); + } + + let mut guard = self.cached.lock().await; + if let Some(cached) = guard.as_ref() + && chrono::Utc::now() < cached.expires_at - chrono::Duration::minutes(5) + { + return Ok(cached.token.clone()); + } + + let jwt = self.app_jwt()?; + let url = format!( + "https://api.github.com/app/installations/{}/access_tokens", + self.cfg.installation_id + ); + let resp = self + .http + .post(&url) + .header("Authorization", format!("Bearer {}", jwt)) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + .header("User-Agent", "amll-ttml-worker") + .send() + .await + .context("请求 GitHub installation token 失败")?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if !status.is_success() { + anyhow::bail!( + "换取 GitHub installation token 失败: status={} body={}", + status, + body + ); + } + + let parsed: TokenResponse = + serde_json::from_str(&body).context("解析 installation token 响应失败")?; + if parsed.token.is_empty() { + anyhow::bail!("GitHub installation token 为空"); + } + + *guard = Some(CachedToken { + token: parsed.token.clone(), + expires_at: parsed.expires_at, + }); + Ok(parsed.token) + } + + /// 签发 GitHub App JWT（iat 回拨 60s，exp 9 分钟） + fn app_jwt(&self) -> Result { + let key = self + .key + .as_ref() + .ok_or_else(|| anyhow::anyhow!("GitHub App 私钥未加载"))?; + let now = chrono::Utc::now().timestamp(); + let claims = AppClaims { + iat: now - 60, + exp: now + 9 * 60, + iss: self.cfg.app_id.to_string(), + }; + let header = Header::new(Algorithm::RS256); + jsonwebtoken::encode(&header, &claims, key).context("签发 GitHub App JWT 失败") + } +} diff --git a/worker/src/infra/mod.rs b/worker/src/infra/mod.rs index b63a36e..8da841d 100644 --- a/worker/src/infra/mod.rs +++ b/worker/src/infra/mod.rs @@ -1,3 +1,5 @@ +pub mod casdoor; +pub mod github_app; pub mod meilisearch_client; pub mod postgres; pub mod rabbitmq; diff --git a/worker/src/infra/rabbitmq.rs b/worker/src/infra/rabbitmq.rs index a364ed1..d6ee7b9 100644 --- a/worker/src/infra/rabbitmq.rs +++ b/worker/src/infra/rabbitmq.rs @@ -2,13 +2,17 @@ use std::sync::Arc; use anyhow::{Context, Result}; use lapin::{ - options::{BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicQosOptions, - ExchangeDeclareOptions, QueueBindOptions, QueueDeclareOptions}, + options::{BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicPublishOptions, + BasicQosOptions, ConfirmSelectOptions, ExchangeDeclareOptions, QueueBindOptions, + QueueDeclareOptions}, types::{AMQPValue, FieldTable}, - Channel, Connection, ConnectionProperties, ExchangeKind, + BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind, }; -use crate::config::Config; +use crate::config::{ + Config, MIGRATION_CLOSE_ROUTING, MIGRATION_DLX_EXCHANGE, MIGRATION_DLQ_ROUTING, + MIGRATION_EXCHANGE, MIGRATION_USER_ROUTING, +}; /// 初始化 RabbitMQ 连接与队列声明 pub struct RabbitMq { @@ -18,6 +22,41 @@ pub struct RabbitMq { pub channel: Channel, /// not_found 消费者使用的独立 channel（避免与主消费者共享同一 channel） pub nf_channel: Channel, + /// 迁移（migration.user）消费者 channel，QoS=1 + pub mig_channel: Channel, + /// 关闭 PR（migration.close_pr）消费者 channel，QoS=1 + pub mig_close_channel: Channel, + /// 迁移发布 channel（开启 publisher confirm，用于投递 migration.close_pr） + pub mig_publish_channel: Channel, +} + +/// 以 publisher confirm 模式发布一条 JSON 消息 +pub async fn publish_json( + channel: &Channel, + exchange: &str, + routing_key: &str, + body: &[u8], + message_id: &str, +) -> Result<()> { + channel + .basic_publish( + exchange, + routing_key, + BasicPublishOptions { + mandatory: true, + ..Default::default() + }, + body, + BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2) + .with_message_id(message_id.into()), + ) + .await + .context("basic_publish")? + .await + .context("publish confirm")?; + Ok(()) } /// 通用消费循环骨架 @@ -275,5 +314,139 @@ pub async fn init_rabbitmq(cfg: &Config) -> Result { .await .context("nf channel qos")?; - Ok(RabbitMq { conn, channel, nf_channel }) + // === 投稿数据迁移队列（独立交换机/队列/DLQ）=== + declare_migration_topology(&channel, cfg).await?; + + // 迁移消费者使用独立 channel（QoS=1） + let mig_channel = conn.create_channel().await.context("create migration channel")?; + mig_channel + .basic_qos(1, BasicQosOptions { global: false }) + .await + .context("migration channel qos")?; + + let mig_close_channel = conn + .create_channel() + .await + .context("create migration close channel")?; + mig_close_channel + .basic_qos(1, BasicQosOptions { global: false }) + .await + .context("migration close channel qos")?; + + // 迁移发布 channel：开启 publisher confirm，保证 migration.close_pr 投递可靠 + let mig_publish_channel = conn + .create_channel() + .await + .context("create migration publish channel")?; + mig_publish_channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .context("enable publisher confirm")?; + + Ok(RabbitMq { + conn, + channel, + nf_channel, + mig_channel, + mig_close_channel, + mig_publish_channel, + }) +} + +/// 声明迁移交换机 / DLX / DLQ / 队列 +async fn declare_migration_topology(channel: &Channel, cfg: &Config) -> Result<()> { + channel + .exchange_declare( + MIGRATION_DLX_EXCHANGE, + ExchangeKind::Direct, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .context("declare migration dlx exchange")?; + + for dlq in [ + &cfg.rabbitmq.migration_dlq, + &cfg.rabbitmq.migration_close_dlq, + ] { + channel + .queue_declare( + dlq, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .with_context(|| format!("declare migration dlq {}", dlq))?; + channel + .queue_bind( + dlq, + MIGRATION_DLX_EXCHANGE, + MIGRATION_DLQ_ROUTING, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .with_context(|| format!("bind migration dlq {}", dlq))?; + } + + channel + .exchange_declare( + MIGRATION_EXCHANGE, + ExchangeKind::Direct, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .context("declare migration exchange")?; + + let mut args = FieldTable::default(); + args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString(MIGRATION_DLX_EXCHANGE.into()), + ); + args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(MIGRATION_DLQ_ROUTING.into()), + ); + + for (queue, routing) in [ + (&cfg.rabbitmq.migration_queue, MIGRATION_USER_ROUTING), + ( + &cfg.rabbitmq.migration_close_queue, + MIGRATION_CLOSE_ROUTING, + ), + ] { + channel + .queue_declare( + queue, + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + args.clone(), + ) + .await + .with_context(|| format!("declare migration queue {}", queue))?; + channel + .queue_bind( + queue, + MIGRATION_EXCHANGE, + routing, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .with_context(|| format!("bind migration queue {}", queue))?; + } + + Ok(()) } diff --git a/worker/src/main.rs b/worker/src/main.rs index ac1fe82..93bd76d 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -2,6 +2,7 @@ mod app; mod config; mod db; mod infra; +mod migration; mod not_found; mod search; mod storage; @@ -91,6 +92,62 @@ async fn main() -> Result<()> { } }); + // 启动迁移消费者（GitHub App 未配置时不启动，不影响既有同步功能） + let mut mig_handles = Vec::new(); + match build_github_app(&cfg) { + Some(github_app) => { + // migration.user 消费者 + let mig_app = app.clone(); + let mig_shutdown = shutdown.clone(); + let mig_channel = mq.mig_channel.clone(); + let mig_publish_channel = mq.mig_publish_channel.clone(); + let mig_queue = cfg.rabbitmq.migration_queue.clone(); + let gh = github_app.clone(); + mig_handles.push(tokio::spawn(async move { + if let Err(e) = migration::consumer::consume_user_loop( + mig_channel, + mig_publish_channel, + mig_queue, + mig_app, + gh, + mig_shutdown, + ) + .await + { + error!(error = %e, "migration consumer exited with error"); + } + })); + + // migration.close_pr 消费者：仅在「自动关闭 PR」开关打开时启动 + if cfg.migration.close_pr_enabled { + let close_app = app.clone(); + let close_shutdown = shutdown.clone(); + let close_channel = mq.mig_close_channel.clone(); + let close_queue = cfg.rabbitmq.migration_close_queue.clone(); + mig_handles.push(tokio::spawn(async move { + if let Err(e) = migration::consumer::consume_close_pr_loop( + close_channel, + close_queue, + close_app, + github_app, + close_shutdown, + ) + .await + { + error!(error = %e, "migration close_pr consumer exited with error"); + } + })); + } else { + info!("自动关闭 PR 已禁用（MIGRATION_CLOSE_PR_ENABLED=false），不启动 migration.close_pr 消费者"); + } + } + None => { + warn!( + "GitHub App 未配置（GITHUB_APP_ID / GITHUB_INSTALLATION_ID / GITHUB_PRIVATE_KEY_PATH），投稿数据迁移功能不可用" + ); + } + } + // 主消费循环（sync 任务） worker::consumer::consume_loop(channel, queue_name, app, shutdown.clone()) .await @@ -98,11 +155,33 @@ async fn main() -> Result<()> { // 等待 not_found 消费者退出 let _ = nf_handle.await; + for handle in mig_handles { + let _ = handle.await; + } info!("ttml-worker exited gracefully"); Ok(()) } +/// 构建 GitHub App 客户端；未配置或私钥不可用时返回 None（不阻塞既有同步功能） +fn build_github_app(cfg: &Arc) -> Option> { + if !cfg.github_app.enabled() { + return None; + } + match infra::github_app::GithubAppClient::new(cfg.github_app.clone()) { + Ok(client) => { + if !client.enabled() { + return None; + } + Some(Arc::new(client)) + } + Err(e) => { + warn!(error = %e, "GitHub App 私钥加载失败，投稿数据迁移功能不可用"); + None + } + } +} + async fn run_health_server(port: u16) -> Result<()> { use axum::{routing::{get, post}, Router}; let app = Router::new() diff --git a/worker/src/migration/consumer.rs b/worker/src/migration/consumer.rs new file mode 100644 index 0000000..1ffb5c2 --- /dev/null +++ b/worker/src/migration/consumer.rs @@ -0,0 +1,811 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use serde::Deserialize; +use tracing::{info, warn}; + +use crate::app::AppState; +use crate::config::{CasdoorConfig, MIGRATION_CLOSE_ROUTING, MIGRATION_EXCHANGE}; +use crate::infra::casdoor::CasdoorClient; +use crate::infra::github_app::GithubAppClient; +use crate::infra::rabbitmq; +use crate::migration::github_api::{GithubApi, extract_body_authors}; +use crate::migration::mapper; +use crate::migration::store::{ + CommentInsert, FileHistoryInsert, MigrationStore, ReviewHistoryInsert, SubmissionInsert, +}; + +/// 待审核 TTML 对象存储前缀（与 backend file_service.go 约定一致） +const PENDING_PREFIX: &str = "tg-lyrics/"; +/// 已通过 TTML 对象存储前缀 +const APPROVED_PREFIX: &str = "shtg-lyrics/"; +/// TTML 对象存储 content-type +const TTML_CONTENT_TYPE: &str = "application/ttml+xml"; +/// 关闭 PR：每批数量与批间隔 +const CLOSE_BATCH_SIZE: usize = 50; +const CLOSE_BATCH_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, Deserialize)] +struct MigrationUserMessage { + task_id: i64, + username: String, + #[serde(default)] + submitter_info: SubmitterInfo, + github_login: String, + #[serde(default)] + since_pr_number: i64, +} + +#[derive(Debug, Deserialize, Default, Clone)] +struct SubmitterInfo { + #[serde(default)] + username: String, + #[serde(default, rename = "displayName")] + display_name: String, + #[serde(default)] + avatar: String, +} + +impl SubmitterInfo { + /// 转换为站点 UserInfo JSON（字段缺失时用站点 username 兜底） + fn to_json(&self, fallback: &str) -> serde_json::Value { + let username = if self.username.is_empty() { + fallback.to_string() + } else { + self.username.clone() + }; + let display_name = if self.display_name.is_empty() { + username.clone() + } else { + self.display_name.clone() + }; + serde_json::json!({ + "username": username, + "displayName": display_name, + "avatar": self.avatar, + }) + } +} + +#[derive(Debug, Deserialize)] +struct ClosePrMessage { + task_id: i64, + username: String, +} + +/// 单条 PR 处理结果 +enum Outcome { + Created, + Skipped, +} + +/// 解析后的身份：站点用户名 + UserInfo +#[derive(Clone)] +struct ResolvedIdentity { + username: String, + info: serde_json::Value, +} + +impl ResolvedIdentity { + /// 未绑定 hub 用户时保持 GitHub 原样 + fn raw(login: &str) -> Self { + let login = login.trim(); + Self { + username: login.to_string(), + info: mapper::user_info(login, None), + } + } +} + +/// GitHub login → 站点用户身份的解析器 +struct LoginResolver { + store: MigrationStore, + casdoor: Option, + organization: String, + cache: HashMap, +} + +impl LoginResolver { + fn new(store: MigrationStore, cfg: &CasdoorConfig) -> Self { + let casdoor = if cfg.enabled() { + match CasdoorClient::new(cfg.clone()) { + Ok(client) => Some(client), + Err(e) => { + warn!(error = %e, "Casdoor 客户端初始化失败，回退为 GitHub 原始信息"); + None + } + } + } else { + warn!("Casdoor 未配置（CASDOOR_ENDPOINT/CLIENT_ID/CLIENT_SECRET/ORGANIZATION），回退为 GitHub 原始信息"); + None + }; + Self { + store, + casdoor, + organization: cfg.organization.clone(), + cache: HashMap::new(), + } + } + + /// 预热一批 GitHub login（结果进缓存；查询失败静默降级为 GitHub 原样） + async fn warm(&mut self, logins: &[String]) { + let mut wanted: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for login in logins { + let key = login.trim().to_ascii_lowercase(); + if key.is_empty() || self.cache.contains_key(&key) || !seen.insert(key.clone()) { + continue; + } + wanted.push(key); + } + if wanted.is_empty() { + return; + } + + let bindings = match self.store.find_bindings(&wanted).await { + Ok(rows) => rows, + Err(e) => { + warn!(error = %e, "查询 GitHub 绑定失败，回退为 GitHub 原始信息"); + return; + } + }; + + let mut resolved: Vec<(String, ResolvedIdentity)> = Vec::with_capacity(bindings.len()); + for row in bindings { + let identity = self.hub_identity(&row.username).await; + resolved.push((row.github_login.trim().to_ascii_lowercase(), identity)); + } + for (key, identity) in resolved { + self.cache.insert(key, identity); + } + } + + /// 取某 GitHub login 对应的站点身份；未绑定 hub 用户时保持 GitHub 原样 + fn lookup(&self, login: &str) -> ResolvedIdentity { + let trimmed = login.trim(); + self.cache + .get(&trimmed.to_ascii_lowercase()) + .cloned() + .unwrap_or_else(|| ResolvedIdentity::raw(trimmed)) + } + + /// UserInfo JSON（写库用） + fn get(&self, login: &str) -> serde_json::Value { + self.lookup(login).info + } + + /// 站点用户名（写库用） + fn username(&self, login: &str) -> String { + self.lookup(login).username + } + + /// 用 hub username 查 Casdoor，得到展示名与头像 + async fn hub_identity(&self, username: &str) -> ResolvedIdentity { + let info = match self.casdoor.as_ref() { + None => mapper::hub_user_info(username, username, ""), + Some(casdoor) => match casdoor.get_user(&self.organization, username).await { + Ok(Some(user)) => { + let display = if user.display_name.trim().is_empty() { + user.name.as_str() + } else { + user.display_name.as_str() + }; + mapper::hub_user_info(username, display, user.avatar.trim()) + } + Ok(None) => mapper::hub_user_info(username, username, ""), + Err(e) => { + warn!(username, error = %e, "查询 Casdoor 用户失败，仅替换用户名"); + mapper::hub_user_info(username, username, "") + } + }, + }; + ResolvedIdentity { + username: username.to_string(), + info, + } + } +} + +// ===== migration.user 消费者 ===== + +pub async fn consume_user_loop( + channel: lapin::Channel, + publish_channel: lapin::Channel, + queue: String, + app: Arc, + github_app: Arc, + shutdown: Arc, +) -> Result<()> { + rabbitmq::consume_loop( + channel, + queue, + "ttml-migration-worker", + shutdown, + move |delivery| { + let app = app.clone(); + let github_app = github_app.clone(); + let publish_channel = publish_channel.clone(); + async move { handle_user_message(delivery, &app, &github_app, &publish_channel).await } + }, + ) + .await +} + +async fn handle_user_message( + delivery: lapin::message::Delivery, + app: &Arc, + github_app: &Arc, + publish_channel: &lapin::Channel, +) -> Result<()> { + let msg: MigrationUserMessage = + serde_json::from_slice(&delivery.data).context("解析 migration.user 消息失败")?; + + info!( + task_id = msg.task_id, + username = %msg.username, + github_login = %msg.github_login, + "开始处理迁移任务" + ); + + let store = MigrationStore::new(app.db.clone()); + match run_user_task(&msg, app, github_app).await { + Ok(()) => { + // 自动关闭 PR 功能默认关闭（MIGRATION_CLOSE_PR_ENABLED=true 可重新开启） + if app.cfg.migration.close_pr_enabled { + if let Err(e) = publish_close_pr(publish_channel, &msg).await { + warn!(error = %e, task_id = msg.task_id, "投递 migration.close_pr 失败（可稍后重试）"); + } + } else { + info!(task_id = msg.task_id, "自动关闭 PR 已禁用，跳过 migration.close_pr"); + } + Ok(()) + } + Err(e) => { + let message = format!("{:#}", e); + warn!(error = %message, task_id = msg.task_id, "迁移任务失败"); + if let Err(err) = store.mark_failed(msg.task_id, &message).await { + warn!(error = %err, "标记迁移任务失败状态时出错"); + } + Err(e) + } + } +} + +/// 执行一次用户迁移（支持基于 cursor 的断点续传） +async fn run_user_task( + msg: &MigrationUserMessage, + app: &Arc, + github_app: &Arc, +) -> Result<()> { + let store = MigrationStore::new(app.db.clone()); + let task = store + .get_task(msg.task_id) + .await? + .ok_or_else(|| anyhow::anyhow!("迁移任务不存在: {}", msg.task_id))?; + + let since = if msg.since_pr_number > 0 { + msg.since_pr_number + } else { + app.cfg.migration.start_pr_number + }; + + // 断点续传：cursor 记录上一条已处理的 PR 号，续传时只处理 < cursor 的部分 + let resume_below = task.cursor.as_ref().and_then(|c| c.parse::().ok()); + let mut processed = task.processed_prs; + let mut created = task.created_count; + let mut skipped = task.skipped_count; + let mut failed = task.failed_count; + + store.mark_running(msg.task_id).await?; + + let api = GithubApi::new( + app.http_client_long.clone(), + github_app.clone(), + app.cfg.github.repo.clone(), + app.cfg.github.token.clone(), + app.cfg.migration.concurrency, + ); + + // 1. 发现该用户的 PR：来源 A（直接投稿）+ 来源 B（bot 代提交） + let direct = api + .search_pr_numbers_by_author(&msg.github_login, since) + .await + .context("搜索直接投稿 PR 失败")?; + let bot = api + .search_bot_pr_numbers(&msg.github_login, since) + .await + .context("搜索 bot 代提交 PR 失败")?; + + let direct_count = direct.len(); + let mut numbers = direct; + let mut seen: std::collections::HashSet = numbers.iter().copied().collect(); + for number in bot { + if seen.insert(number) { + numbers.push(number); + } + } + + // 按 PR 号倒序处理，并应用断点续传过滤 + numbers.sort_unstable_by(|a, b| b.cmp(a)); + numbers.dedup(); + if let Some(cursor) = resume_below { + numbers.retain(|n| *n < cursor); + } + + info!( + task_id = msg.task_id, + direct = direct_count, + total = numbers.len(), + resume_below = ?resume_below, + "PR 发现完成" + ); + + store + .set_total(msg.task_id, processed + numbers.len() as i32) + .await?; + + let flush_every = app.cfg.migration.batch_size.max(1); + + // GitHub login → 站点用户信息解析器（整轮任务复用缓存） + let mut resolver = LoginResolver::new(store.clone(), &app.cfg.casdoor); + + for (idx, number) in numbers.iter().enumerate() { + let cursor = number.to_string(); + match process_one_pr(&api, app, &store, msg, *number, &mut resolver).await { + Ok(Outcome::Created) => created += 1, + Ok(Outcome::Skipped) => skipped += 1, + Err(e) => { + failed += 1; + warn!( + task_id = msg.task_id, + pr_number = number, + error = %format!("{:#}", e), + "单条 PR 迁移失败，继续处理" + ); + } + } + processed += 1; + + if (idx + 1) % flush_every == 0 { + store + .update_progress( + msg.task_id, + processed, + created, + skipped, + failed, + Some(&cursor), + ) + .await?; + } + } + + store + .mark_completed(msg.task_id, processed, created, skipped, failed) + .await?; + + info!( + task_id = msg.task_id, + processed, created, skipped, failed, "迁移任务完成" + ); + Ok(()) +} + +/// 投递 migration.close_pr +async fn publish_close_pr(channel: &lapin::Channel, msg: &MigrationUserMessage) -> Result<()> { + let payload = serde_json::json!({ + "task_id": msg.task_id, + "username": msg.username, + "github_login": msg.github_login, + }); + let body = serde_json::to_vec(&payload).context("序列化 migration.close_pr 消息失败")?; + rabbitmq::publish_json( + channel, + MIGRATION_EXCHANGE, + MIGRATION_CLOSE_ROUTING, + &body, + &format!("migration.close_pr:{}", msg.task_id), + ) + .await +} + +/// 单条 PR → 站点投稿 +async fn process_one_pr( + api: &GithubApi, + app: &Arc, + store: &MigrationStore, + msg: &MigrationUserMessage, + number: i64, + resolver: &mut LoginResolver, +) -> Result { + // 2. 去重：命中已迁移记录直接跳过 + if store.has_migrated_pr(number).await? { + return Ok(Outcome::Skipped); + } + + let pr = api.get_pr(number).await?; + + // 定位并下载 TTML + let files = api.list_pr_files(number).await?; + let (path, file_name) = mapper::find_ttml_file(&files) + .ok_or_else(|| anyhow::anyhow!("PR #{} 未找到 raw-lyrics/*.ttml 文件", number))?; + let bytes = api.download_file(&path, &pr.head_sha).await?; + let meta = mapper::parse_ttml_meta(&bytes)?; + + // 审核过程（缺失时降级处理，不阻断迁移） + let timeline = api.list_timeline(number).await.unwrap_or_else(|e| { + warn!(pr_number = number, error = %e, "拉取 timeline 失败，按无 timeline 处理"); + Vec::new() + }); + let reviews = api.list_pr_reviews(number).await.unwrap_or_else(|e| { + warn!(pr_number = number, error = %e, "拉取 reviews 失败，按无 review 处理"); + Vec::new() + }); + let comments = api.list_issue_comments(number).await.unwrap_or_else(|e| { + warn!(pr_number = number, error = %e, "拉取评论失败，按无评论处理"); + Vec::new() + }); + + let mapping = mapper::map_status(&pr, &timeline, &reviews); + + // 3. 上传 TTML：待审核区；approved 额外复制到已通过区 + let bucket = app.cfg.minio.bucket.clone(); + let pending_key = format!("{}{}", PENDING_PREFIX, file_name); + crate::sync::downloader::upload_to_minio( + &app.s3, + &bucket, + &pending_key, + &bytes, + TTML_CONTENT_TYPE, + ) + .await + .with_context(|| format!("上传 {} 到 MinIO 失败", pending_key))?; + + if mapping.status == mapper::STATUS_APPROVED { + let approved_key = format!("{}{}", APPROVED_PREFIX, file_name); + crate::sync::downloader::upload_to_minio( + &app.s3, + &bucket, + &approved_key, + &bytes, + TTML_CONTENT_TYPE, + ) + .await + .with_context(|| format!("上传 {} 到 MinIO 失败", approved_key))?; + } + + // 4. bot 代提交 PR 的归属标注（用于溯源） + let body_author = pr + .body + .as_deref() + .and_then(|body| { + extract_body_authors(body) + .into_iter() + .find(|a| a.eq_ignore_ascii_case(&msg.github_login)) + }) + .unwrap_or_default(); + + let now = chrono::Utc::now(); + let pr_state = if pr.merged { + "merged".to_string() + } else { + pr.state.clone() + }; + let metadata = serde_json::json!({ + "source": "github_migration", + "github_pr_number": pr.number, + "github_pr_title": pr.title, + "github_pr_url": pr.url, + "github_pr_author": pr.author_login, + "github_pr_body_author": body_author, + "github_pr_state": pr_state, + "merged_at": pr.merged_at.map(|t| t.to_rfc3339()), + "labels": pr.labels, + "ttml_author_github": meta.ttml_author_github, + "migrated_at": now.to_rfc3339(), + "data_cutoff_at": now.to_rfc3339(), + }); + + // 5. 审核历史：逐条 review + 最终结论 + let mut logins: Vec = Vec::new(); + for review in &reviews { + let login = review.reviewer(); + if !login.is_empty() { + logins.push(login.to_string()); + } + } + for comment in &comments { + if let Some(user) = comment.user.as_ref() { + logins.push(user.login.clone()); + } + } + logins.push(pr.author_login.clone()); + logins.push(body_author.clone()); + if let Some(login) = mapping.reviewer.as_ref() { + logins.push(login.clone()); + } + if let Some(login) = mapping.closed_by.as_ref() { + logins.push(login.clone()); + } + resolver.warm(&logins).await; + + let mut histories: Vec = Vec::new(); + for review in &reviews { + let reviewer = review.reviewer(); + let submitted_at = review.submitted_at; + if reviewer.is_empty() || submitted_at.is_none() { + continue; + } + let status = if review.state.eq_ignore_ascii_case("APPROVED") { + mapper::STATUS_APPROVED + } else if review.state.eq_ignore_ascii_case("CHANGES_REQUESTED") { + mapper::STATUS_NEED_REVISION + } else { + continue; + }; + histories.push(ReviewHistoryInsert { + reviewer: resolver.username(reviewer), + reviewer_info: resolver.get(reviewer), + status: status.to_string(), + comment: review.body.trim().to_string(), + reviewed_at: submitted_at.expect("submitted_at checked"), + }); + } + if let Some(conclusion) = build_conclusion(&mapping, &pr, &histories, resolver) { + histories.push(conclusion); + } + + // 6. 评论：PR issue 评论 + review 正文 + let mut comments_insert: Vec = comments + .iter() + .filter(|c| !c.body.trim().is_empty()) + .map(|c| { + let login = c.user.as_ref().map(|u| u.login.clone()).unwrap_or_default(); + CommentInsert { + author: resolver.get(&login), + content: c.body.clone(), + created_at: c.created_at.unwrap_or(pr.created_at), + } + }) + .collect(); + for review in &reviews { + let body = review.body.trim(); + let reviewer = review.reviewer(); + if body.is_empty() || reviewer.is_empty() { + continue; + } + comments_insert.push(CommentInsert { + author: resolver.get(reviewer), + content: body.to_string(), + created_at: review.submitted_at.unwrap_or(pr.created_at), + }); + } + + // 7. 文件更新历史（PR 创建即提交该 TTML） + let file_uploader = if body_author.is_empty() { + pr.author_login.clone() + } else { + body_author + }; + let file_history = if file_uploader.is_empty() { + None + } else { + Some(FileHistoryInsert { + uploader: resolver.username(&file_uploader), + uploader_info: resolver.get(&file_uploader), + file_name: file_name.clone(), + uploaded_at: pr.created_at, + }) + }; + + let closed_by = mapping + .closed_by + .as_ref() + .map(|login| resolver.username(login)); + let closed_by_info = mapping.closed_by.as_ref().map(|login| resolver.get(login)); + let reviewer = mapping + .reviewer + .as_ref() + .map(|login| resolver.username(login)); + + let data = SubmissionInsert { + pr_number: pr.number, + username: msg.username.clone(), + title: meta.title, + artist: meta.artist, + album: meta.album, + ncm_id: meta.ncm_id, + qq_id: meta.qq_id, + am_id: meta.am_id, + spotify_id: meta.spotify_id, + file_name, + tags: serde_json::json!(pr.labels), + metadata, + language: meta.language, + languages: serde_json::json!(meta.languages), + status: mapping.status.clone(), + submitter: msg.username.clone(), + submitter_info: msg.submitter_info.to_json(&msg.username), + created_at: pr.created_at, + updated_at: pr.updated_at, + revision_requested_at: mapping.revision_requested_at, + closed_at: mapping.closed_at, + closed_by, + closed_by_info, + reviewer, + reviewed_at: mapping.reviewed_at, + review_comment: mapping.review_comment.clone(), + reviews: histories, + comments: comments_insert, + file_history, + }; + + match store.insert_submission(&data).await? { + Some(_) => Ok(Outcome::Created), + None => Ok(Outcome::Skipped), + } +} + +/// 构造 PR 最终结论对应的审核历史（已有同状态记录时不重复写） +fn build_conclusion( + mapping: &mapper::StatusMapping, + pr: &crate::migration::github_api::PrInfo, + existing: &[ReviewHistoryInsert], + resolver: &LoginResolver, +) -> Option { + if mapping.status == mapper::STATUS_PENDING { + return None; + } + if existing + .iter() + .any(|h| h.status == mapping.status && h.comment == mapping.review_comment.clone().unwrap_or_default()) + { + return None; + } + + let reviewer = mapping + .reviewer + .clone() + .or_else(|| mapping.closed_by.clone()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| pr.author_login.clone()); + if reviewer.is_empty() { + return None; + } + + Some(ReviewHistoryInsert { + reviewer: resolver.username(&reviewer), + reviewer_info: resolver.get(&reviewer), + status: mapping.status.clone(), + comment: mapping.review_comment.clone().unwrap_or_default(), + reviewed_at: mapping + .reviewed_at + .or(mapping.closed_at) + .or(mapping.revision_requested_at) + .unwrap_or(pr.updated_at), + }) +} + +// ===== migration.close_pr 消费者 ===== + +pub async fn consume_close_pr_loop( + channel: lapin::Channel, + queue: String, + app: Arc, + github_app: Arc, + shutdown: Arc, +) -> Result<()> { + rabbitmq::consume_loop( + channel, + queue, + "ttml-migration-close-worker", + shutdown, + move |delivery| { + let app = app.clone(); + let github_app = github_app.clone(); + async move { handle_close_message(delivery, &app, &github_app).await } + }, + ) + .await +} + +async fn handle_close_message( + delivery: lapin::message::Delivery, + app: &Arc, + github_app: &Arc, +) -> Result<()> { + let msg: ClosePrMessage = + serde_json::from_slice(&delivery.data).context("解析 migration.close_pr 消息失败")?; + + let store = MigrationStore::new(app.db.clone()); + let task = store + .get_task(msg.task_id) + .await? + .ok_or_else(|| anyhow::anyhow!("迁移任务不存在: {}", msg.task_id))?; + + // 幂等：已完成则直接跳过 + if task.close_pr_status == "completed" { + info!(task_id = msg.task_id, "关闭 PR 已完成，跳过"); + return Ok(()); + } + + store.mark_close_running(msg.task_id).await?; + + let api = GithubApi::new( + app.http_client_long.clone(), + github_app.clone(), + app.cfg.github.repo.clone(), + app.cfg.github.token.clone(), + app.cfg.migration.concurrency, + ); + + let prs = store + .list_migrated_prs(&msg.username, app.cfg.migration.start_pr_number) + .await?; + + let site_url = app.cfg.migration.site_url.trim_end_matches('/').to_string(); + let mut closed = 0usize; + let mut failed = 0usize; + + for chunk in prs.chunks(CLOSE_BATCH_SIZE) { + for pr in chunk { + match close_one_pr(&api, pr.number, pr.file_name.as_deref(), &site_url).await { + Ok(true) => { + closed += 1; + store.add_closed_count(msg.task_id, 1).await?; + } + Ok(false) => {} + Err(e) => { + failed += 1; + warn!( + task_id = msg.task_id, + pr_number = pr.number, + error = %format!("{:#}", e), + "关闭 PR 失败，继续处理" + ); + } + } + } + tokio::time::sleep(CLOSE_BATCH_INTERVAL).await; + } + + let status = if failed > 0 && closed == 0 { + "failed" + } else { + "completed" + }; + store.set_close_status(msg.task_id, status).await?; + + info!(task_id = msg.task_id, closed, failed, status, "关闭 PR 阶段完成"); + Ok(()) +} + +/// 关闭单个 PR 并评论迁移地址；返回是否实际执行了关闭 +async fn close_one_pr( + api: &GithubApi, + number: i64, + file_name: Option<&str>, + site_url: &str, +) -> Result { + let pr = api.get_pr(number).await?; + if pr.state != "open" { + return Ok(false); + } + + api.close_pr(number).await?; + + let comment = match file_name { + Some(name) if !site_url.is_empty() => format!( + "该投稿已迁移至 amll-hub，请前往 {}/lyric/{} 查看", + site_url, name + ), + _ if !site_url.is_empty() => format!("该投稿已迁移至 amll-hub，请前往 {} 查看", site_url), + _ => "该投稿已迁移至 amll-hub。".to_string(), + }; + if let Err(e) = api.create_issue_comment(number, &comment).await { + warn!(pr_number = number, error = %e, "评论迁移地址失败（PR 已关闭）"); + } + Ok(true) +} diff --git a/worker/src/migration/github_api.rs b/worker/src/migration/github_api.rs new file mode 100644 index 0000000..a20a2b3 --- /dev/null +++ b/worker/src/migration/github_api.rs @@ -0,0 +1,680 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use reqwest::{Client, Method}; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use serde_json::json; +use tokio::sync::Semaphore; +use tracing::{debug, warn}; + +use crate::infra::github_app::GithubAppClient; + +const API_BASE: &str = "https://api.github.com"; +const GRAPHQL_URL: &str = "https://api.github.com/graphql"; +const MAX_RETRY: u32 = 5; +const JSON_RETRY: u32 = 3; +const RATE_LIMIT_FLOOR: i64 = 100; +const SEARCH_RATE_LIMIT_FLOOR: i64 = 1; + +/// 搜索每页条数（GraphQL / REST 上限均为 100） +const SEARCH_PAGE_SIZE: usize = 100; + +#[derive(Debug, Clone, Deserialize)] +pub struct RestUser { + #[serde(default)] + pub login: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct RestLabel { + #[serde(default)] + pub name: String, +} + +/// 归一化后的 PR 信息（来源：GET /repos/{repo}/pulls/{number}） +#[derive(Debug, Clone)] +pub struct PrInfo { + pub number: i64, + /// PR 标题 + pub title: String, + pub body: Option, + /// open / closed + pub state: String, + pub merged: bool, + pub merged_at: Option>, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub closed_at: Option>, + pub url: String, + pub author_login: String, + pub labels: Vec, + pub head_sha: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct PrFile { + #[serde(default)] + pub filename: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct PrReview { + #[serde(default)] + pub user: Option, + #[serde(default)] + pub state: String, + #[serde(default)] + pub body: String, + #[serde(default)] + pub submitted_at: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct IssueComment { + #[serde(default)] + pub user: Option, + #[serde(default)] + pub body: String, + #[serde(default)] + pub created_at: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TimelineEvent { + #[serde(default)] + pub event: String, + #[serde(default)] + pub actor: Option, + #[serde(default)] + pub created_at: Option>, + #[serde(default)] + pub label: Option, +} + +#[derive(Deserialize)] +struct RawPull { + number: i64, + #[serde(default)] + title: String, + #[serde(default)] + body: Option, + #[serde(default)] + state: String, + #[serde(default)] + merged: bool, + #[serde(default)] + merged_at: Option>, + #[serde(default)] + created_at: Option>, + #[serde(default)] + updated_at: Option>, + #[serde(default)] + closed_at: Option>, + #[serde(default)] + html_url: String, + #[serde(default)] + user: Option, + #[serde(default)] + labels: Vec, + #[serde(default)] + head: Option, +} + +#[derive(Deserialize)] +struct RawHead { + #[serde(default)] + sha: String, +} + +#[derive(Deserialize)] +struct SearchResponse { + #[serde(default)] + total_count: i64, + #[serde(default)] + items: Vec, +} + +#[derive(Deserialize)] +struct SearchItem { + number: i64, + #[serde(default)] + body: Option, +} + +#[derive(Deserialize)] +struct GqlResponse { + data: Option, + #[serde(default)] + errors: Option, +} + +#[derive(Deserialize)] +struct GqlSearchData { + search: GqlSearch, +} + +#[derive(Deserialize)] +struct GqlSearch { + #[serde(rename = "pageInfo")] + page_info: GqlPageInfo, + #[serde(default)] + nodes: Vec, +} + +#[derive(Deserialize)] +struct GqlPageInfo { + #[serde(rename = "hasNextPage")] + has_next_page: bool, + #[serde(rename = "endCursor")] + end_cursor: Option, +} + +#[derive(Deserialize)] +struct GqlNode { + #[serde(default)] + number: Option, +} + +const PR_SEARCH_QUERY: &str = r#" +query($q: String!, $after: String) { + search(query: $q, type: ISSUE, first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { number } + } + } +} +"#; + +/// 请求鉴权方式 +#[derive(Clone, Copy, PartialEq, Eq)] +enum Auth { + /// GitHub App 安装令牌 + App, + /// 个人访问令牌（GITHUB_TOKEN） + Pat, +} + +pub struct GithubApi { + http: Client, + app: Arc, + repo: String, + /// 搜索专用 PAT（GITHUB_TOKEN） + pat: String, + /// 请求并发上限（≤10） + semaphore: Arc, +} + +impl GithubApi { + pub fn new( + http: Client, + app: Arc, + repo: impl Into, + pat: impl Into, + concurrency: usize, + ) -> Self { + let limit = concurrency.clamp(1, 10); + Self { + http, + app, + repo: repo.into(), + pat: pat.into(), + semaphore: Arc::new(Semaphore::new(limit)), + } + } + + /// 取搜索用令牌；未配置时直接报错。 + fn search_token(&self) -> Result<&str> { + let token = self.pat.trim(); + if token.is_empty() { + anyhow::bail!( + "未配置 GITHUB_TOKEN：GitHub 在 App 安装令牌下不返回 PR 搜索结果，无法发现投稿，请在 .env 配置 GITHUB_TOKEN 后重试" + ); + } + Ok(token) + } + + // ===== 内部请求 ===== + + /// 发送带鉴权与限流退避的请求 + async fn request( + &self, + method: Method, + url: &str, + accept: Option<&str>, + body: Option, + auth: Auth, + ) -> Result { + let mut attempt: u32 = 0; + loop { + attempt += 1; + let _permit = self + .semaphore + .acquire() + .await + .context("acquire github request permit")?; + let token = match auth { + Auth::App => self.app.installation_token().await?, + Auth::Pat => self.search_token()?.to_string(), + }; + + let mut req = self + .http + .request(method.clone(), url) + .header("Authorization", format!("Bearer {}", token)) + .header("Accept", accept.unwrap_or("application/vnd.github+json")) + .header("X-GitHub-Api-Version", "2022-11-28") + .header("User-Agent", "amll-ttml-worker"); + if let Some(ref payload) = body { + req = req.json(payload); + } + + let resp = req + .send() + .await + .with_context(|| format!("GitHub 请求失败: {} {}", method, url))?; + + // 低配额保护：剩余不足则暂停至 reset。 + let resource = resp + .headers() + .get("x-ratelimit-resource") + .and_then(|v| v.to_str().ok()) + .unwrap_or("core"); + let floor = if resource == "search" { + SEARCH_RATE_LIMIT_FLOOR + } else { + RATE_LIMIT_FLOOR + }; + if let Some(remaining) = header_i64(&resp, "x-ratelimit-remaining") + && remaining < floor + && let Some(reset) = header_i64(&resp, "x-ratelimit-reset") + { + let wait = (reset - chrono::Utc::now().timestamp()).max(1) as u64; + warn!( + resource, + remaining, + wait_secs = wait, + "GitHub 剩余配额不足，暂停至配额重置" + ); + tokio::time::sleep(Duration::from_secs(wait.min(300))).await; + } + + if resp.status().is_success() { + return Ok(resp); + } + + let status = resp.status(); + let retry_after = header_i64(&resp, "retry-after").map(|v| v as u64); + let body_text = resp.text().await.unwrap_or_default(); + + let secondary = body_text.contains("secondary rate limit") + || body_text.contains("API rate limit exceeded"); + let retryable = status.as_u16() == 429 + || (status.as_u16() == 403 && (secondary || retry_after.is_some())) + || status.is_server_error(); + + if retryable && attempt < MAX_RETRY { + let delay = retry_after.unwrap_or(60 * attempt as u64).min(300); + warn!( + status = %status, + attempt, + delay_secs = delay, + "GitHub 请求受限，退避重试" + ); + tokio::time::sleep(Duration::from_secs(delay)).await; + continue; + } + + anyhow::bail!( + "GitHub 请求失败: {} {} status={} body={}", + method, + url, + status, + truncate(&body_text, 500) + ); + } + } + + async fn get_json(&self, path: &str, accept: Option<&str>) -> Result { + self.get_json_with(Auth::App, path, accept).await + } + + async fn get_json_with( + &self, + auth: Auth, + path: &str, + accept: Option<&str>, + ) -> Result { + let url = format!("{}{}", API_BASE, path); + self.request_json(Method::GET, &url, accept, None, auth) + .await + .with_context(|| format!("解析 GitHub 响应失败: {}", url)) + } + + /// 请求并解码 JSON：`request()` 里的重试只覆盖状态码类失败 + async fn request_json( + &self, + method: Method, + url: &str, + accept: Option<&str>, + body: Option, + auth: Auth, + ) -> Result { + let mut last_err: Option = None; + for attempt in 1..=JSON_RETRY { + let resp = self + .request(method.clone(), url, accept, body.clone(), auth) + .await?; + match resp.json::().await { + Ok(parsed) => return Ok(parsed), + Err(e) => { + warn!(url, attempt, error = %e, "读取 GitHub 响应失败，重试"); + last_err = Some(anyhow::Error::new(e)); + } + } + if attempt < JSON_RETRY { + tokio::time::sleep(Duration::from_secs(2 * attempt as u64)).await; + } + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("读取 GitHub 响应失败: {}", url))) + } + + async fn patch_json(&self, path: &str, body: serde_json::Value) -> Result<()> { + let url = format!("{}{}", API_BASE, path); + self.request(Method::PATCH, &url, None, Some(body), Auth::App) + .await?; + Ok(()) + } + + async fn post_json(&self, path: &str, body: serde_json::Value) -> Result<()> { + let url = format!("{}{}", API_BASE, path); + self.request(Method::POST, &url, None, Some(body), Auth::App) + .await?; + Ok(()) + } + + // ===== PR 发现 ===== + pub async fn search_pr_numbers_by_author(&self, login: &str, since: i64) -> Result> { + let q = format!( + "repo:{} type:pr author:{} sort:created-desc", + self.repo, login + ); + let mut numbers = Vec::new(); + let mut after: Option = None; + let mut truncated = false; + + loop { + let variables = json!({ "q": q, "after": after }); + let payload = json!({ "query": PR_SEARCH_QUERY, "variables": variables }); + let url = GRAPHQL_URL.to_string(); + let parsed: GqlResponse = self + .request_json(Method::POST, &url, None, Some(payload), Auth::Pat) + .await + .context("解析 GitHub GraphQL 搜索响应失败")?; + + if let Some(errors) = parsed.errors { + anyhow::bail!("GitHub GraphQL 搜索返回错误: {}", truncate(&errors.to_string(), 300)); + } + let search = parsed + .data + .ok_or_else(|| anyhow::anyhow!("GitHub GraphQL 搜索无数据"))? + .search; + + let node_count = search.nodes.len(); + let mut reached_cutoff = false; + for node in search.nodes { + let Some(number) = node.number else { continue }; + if number < since { + reached_cutoff = true; + break; + } + numbers.push(number); + } + + if reached_cutoff || !search.page_info.has_next_page { + break; + } + if node_count < SEARCH_PAGE_SIZE { + break; + } + match search.page_info.end_cursor { + Some(cursor) => after = Some(cursor), + None => break, + } + // GitHub 搜索最多返回 1000 条，达到上限时记录并停止 + if numbers.len() >= 1000 { + truncated = true; + break; + } + } + + if truncated { + warn!( + login, + count = numbers.len(), + "GraphQL 搜索结果达到 1000 条上限，可能存在遗漏" + ); + } + Ok(numbers) + } + + /// 来源 B：搜索由 github-actions[bot] 代提交、且正文提及该用户的 PR + pub async fn search_bot_pr_numbers(&self, login: &str, since: i64) -> Result> { + let q = format!( + "repo:{} is:pr author:app/github-actions {} in:body", + self.repo, login + ); + let mut numbers = Vec::new(); + let mut page: u32 = 1; + loop { + let path = format!( + "/search/issues?q={}&sort=created&order=desc&per_page={}&page={}", + urlencoding::encode(&q), + SEARCH_PAGE_SIZE, + page + ); + let resp: SearchResponse = self.get_json_with(Auth::Pat, &path, None).await?; + if page == 1 { + debug!(total_count = resp.total_count, "bot 代提交 PR 搜索命中数"); + } + if resp.items.is_empty() { + break; + } + let item_count = resp.items.len(); + let mut reached_cutoff = false; + for item in resp.items { + if item.number < since { + reached_cutoff = true; + break; + } + // 精确校验：正文必须含「歌词作者 @本用户」标注 + let body = item.body.unwrap_or_default(); + if body_mentions_author(&body, login) { + numbers.push(item.number); + } + } + if reached_cutoff || item_count < SEARCH_PAGE_SIZE || page >= 10 { + break; + } + page += 1; + } + Ok(numbers) + } + + // ===== PR 明细 ===== + + pub async fn get_pr(&self, number: i64) -> Result { + let path = format!("/repos/{}/pulls/{}", self.repo, number); + let raw: RawPull = self.get_json(&path, None).await?; + let now = chrono::Utc::now(); + Ok(PrInfo { + number: raw.number, + title: raw.title, + body: raw.body, + state: raw.state, + merged: raw.merged, + merged_at: raw.merged_at, + created_at: raw.created_at.unwrap_or(now), + updated_at: raw.updated_at.unwrap_or(now), + closed_at: raw.closed_at, + url: raw.html_url, + author_login: raw.user.map(|u| u.login).unwrap_or_default(), + labels: raw.labels.into_iter().map(|l| l.name).collect(), + head_sha: raw.head.map(|h| h.sha).unwrap_or_default(), + }) + } + + pub async fn list_pr_files(&self, number: i64) -> Result> { + let path = format!( + "/repos/{}/pulls/{}/files?per_page={}", + self.repo, number, SEARCH_PAGE_SIZE + ); + self.get_json(&path, None).await + } + + pub async fn list_pr_reviews(&self, number: i64) -> Result> { + let path = format!( + "/repos/{}/pulls/{}/reviews?per_page={}", + self.repo, number, SEARCH_PAGE_SIZE + ); + self.get_json(&path, None).await + } + + pub async fn list_issue_comments(&self, number: i64) -> Result> { + let path = format!( + "/repos/{}/issues/{}/comments?per_page={}", + self.repo, number, SEARCH_PAGE_SIZE + ); + self.get_json(&path, None).await + } + + pub async fn list_timeline(&self, number: i64) -> Result> { + let path = format!( + "/repos/{}/issues/{}/timeline?per_page={}", + self.repo, number, SEARCH_PAGE_SIZE + ); + self.get_json(&path, None).await + } + + /// 下载仓库中指定路径、指定 ref 的文件原始内容 + pub async fn download_file(&self, path: &str, git_ref: &str) -> Result> { + let url = format!( + "{}/repos/{}/contents/{}?ref={}", + API_BASE, + self.repo, + path, + urlencoding::encode(git_ref) + ); + let resp = self + .request( + Method::GET, + &url, + Some("application/vnd.github.raw"), + None, + Auth::App, + ) + .await?; + let bytes = resp.bytes().await.context("读取 GitHub 文件内容失败")?; + Ok(bytes.to_vec()) + } + + // ===== 关闭 PR / 评论 ===== + + pub async fn close_pr(&self, number: i64) -> Result<()> { + let path = format!("/repos/{}/pulls/{}", self.repo, number); + self.patch_json(&path, json!({ "state": "closed" })).await + } + + pub async fn create_issue_comment(&self, number: i64, body: &str) -> Result<()> { + let path = format!("/repos/{}/issues/{}/comments", self.repo, number); + self.post_json(&path, json!({ "body": body })).await + } +} + +/// 判断 PR 正文是否以「歌词作者 @login」形式标注了该用户（大小写不敏感精确匹配） +pub fn body_mentions_author(body: &str, login: &str) -> bool { + extract_body_authors(body) + .iter() + .any(|author| author.eq_ignore_ascii_case(login)) +} + +/// 解析 PR 正文中所有「歌词作者 @xxx」标注 +pub fn extract_body_authors(body: &str) -> Vec { + const KEY: &str = "歌词作者"; + let mut authors = Vec::new(); + let mut rest = body; + while let Some(idx) = rest.find(KEY) { + let after = &rest[idx + KEY.len()..]; + let trimmed = after.trim_start(); + if let Some(stripped) = trimmed.strip_prefix('@') { + let login: String = stripped + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '-') + .collect(); + if !login.is_empty() { + authors.push(login); + } + } + // KEY 已被剥离，rest 严格变短，循环必然推进 + rest = after; + } + authors +} + +fn header_i64(resp: &reqwest::Response, name: &str) -> Option { + resp.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.trim().parse::().ok()) +} + +fn truncate(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + s.chars().take(max).collect::() + "..." +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_author_annotation() { + let body = "本 PR 由 bot 代提交\n\n（歌词作者 @Pigeon0v0）"; + assert_eq!(extract_body_authors(body), vec!["Pigeon0v0".to_string()]); + assert!(body_mentions_author(body, "pigeon0v0")); + assert!(!body_mentions_author(body, "Pigeon")); + } + + #[test] + fn annotation_requires_exact_login_boundary() { + let body = "（歌词作者 @abcd）"; + assert!(!body_mentions_author(body, "abc")); + assert!(body_mentions_author(body, "abcd")); + } + + #[test] + fn ignores_mentions_without_keyword() { + let body = "感谢 @apoint123 的贡献"; + assert!(extract_body_authors(body).is_empty()); + assert!(!body_mentions_author(body, "apoint123")); + } + + #[test] + fn supports_hyphenated_login_and_whitespace() { + let body = "（歌词作者 @some-user ）"; + assert_eq!(extract_body_authors(body), vec!["some-user".to_string()]); + } + + #[test] + fn handles_multiple_annotations() { + let body = "（歌词作者 @a1）\n（歌词作者 @b2）"; + assert_eq!( + extract_body_authors(body), + vec!["a1".to_string(), "b2".to_string()] + ); + } +} diff --git a/worker/src/migration/mapper.rs b/worker/src/migration/mapper.rs new file mode 100644 index 0000000..f153b74 --- /dev/null +++ b/worker/src/migration/mapper.rs @@ -0,0 +1,461 @@ +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; + +use crate::migration::github_api::{PrFile, PrInfo, PrReview, TimelineEvent}; + +/// 「待更新」标签名（PR 上代表审核员要求修改） +pub const REVISION_LABEL: &str = "待更新"; + +/// GitHub bot 账号登录名（超时自动关闭的 PR 由它执行） +pub const GITHUB_ACTIONS_BOT: &str = "github-actions[bot]"; + +/// 站点状态常量（对齐 backend/internal/model/submission.go） +pub const STATUS_PENDING: &str = "pending"; +pub const STATUS_NEED_REVISION: &str = "need_revision"; +pub const STATUS_APPROVED: &str = "approved"; +pub const STATUS_REJECTED: &str = "rejected"; +pub const STATUS_CLOSED: &str = "closed"; + +/// GitHub → 站点状态映射结果（见设计文档 §4.3） +#[derive(Debug, Clone, Default)] +pub struct StatusMapping { + pub status: String, + pub reviewer: Option, + pub reviewed_at: Option>, + pub review_comment: Option, + pub closed_at: Option>, + pub closed_by: Option, + pub revision_requested_at: Option>, +} + +/// 按 §4.3 优先级映射站点状态 +pub fn map_status(pr: &PrInfo, timeline: &[TimelineEvent], reviews: &[PrReview]) -> StatusMapping { + let last_review = last_decisive_review(reviews); + let last_approved = reviews + .iter() + .filter(|r| r.state.eq_ignore_ascii_case("APPROVED")) + .max_by_key(|r| r.submitted_at); + + // 优先级 1：已合并，或最后一次 review 结论为 APPROVED → approved + // + // 本仓库的审核流程是「审核员 APPROVED 后直接关闭 PR、不 merge」， + // 只看 merged 会把通过结论误判成「审核员关闭」→ rejected，并显示上一次 + // CHANGES_REQUESTED 的理由，因此必须结合最后一条 review 的结论。 + if pr.merged || last_review.is_some_and(|r| r.state.eq_ignore_ascii_case("APPROVED")) { + return StatusMapping { + status: STATUS_APPROVED.to_string(), + reviewer: last_approved.map(|r| r.reviewer().to_string()), + reviewed_at: last_approved.and_then(|r| r.submitted_at).or(pr.merged_at), + review_comment: last_approved.and_then(|r| non_empty(&r.body)), + ..Default::default() + }; + } + + if pr.state == "closed" { + let closer = timeline + .iter() + .find(|e| e.event == "closed") + .and_then(|e| e.actor.as_ref()) + .map(|a| a.login.clone()); + + // 优先级 2：github-actions[bot] 超时自动关闭 → closed + // 必须先于「作者主动放弃」判断：bot 代提交的 PR 其 author 本身就是 bot， + // bot 关闭动作代表超时而非作者放弃 + if let Some(ref closer_login) = closer + && closer_login.eq_ignore_ascii_case(GITHUB_ACTIONS_BOT) + { + return StatusMapping { + status: STATUS_CLOSED.to_string(), + closed_at: pr.closed_at, + closed_by: Some(closer_login.clone()), + ..Default::default() + }; + } + + // 优先级 3：作者自行关闭（放弃） + if let Some(ref closer_login) = closer + && !closer_login.is_empty() + && closer_login.eq_ignore_ascii_case(&pr.author_login) + { + return StatusMapping { + status: STATUS_CLOSED.to_string(), + closed_at: pr.closed_at, + closed_by: Some(closer_login.clone()), + ..Default::default() + }; + } + + // 优先级 4：审核员关闭（非通过） + let reason = reviews + .iter() + .rev() + .find(|r| r.state.eq_ignore_ascii_case("CHANGES_REQUESTED")) + .and_then(|r| non_empty(&r.body)); + return StatusMapping { + status: STATUS_REJECTED.to_string(), + reviewer: closer.filter(|s| !s.is_empty()), + reviewed_at: pr.closed_at, + review_comment: reason, + ..Default::default() + }; + } + + // 优先级 5：open 且当前带「待更新」标签 → need_revision + let revision_at = timeline + .iter() + .filter(|e| { + e.event == "labeled" && e.label.as_ref().is_some_and(|l| l.name == REVISION_LABEL) + }) + .filter_map(|e| e.created_at) + .next_back(); + + if let Some(revision_at) = revision_at { + // 同步最后一次「要求修改」的审核意见，供详情页展示修改建议 + let last_changes = reviews + .iter() + .rev() + .find(|r| r.state.eq_ignore_ascii_case("CHANGES_REQUESTED")); + return StatusMapping { + status: STATUS_NEED_REVISION.to_string(), + reviewer: last_changes.map(|r| r.reviewer().to_string()), + reviewed_at: last_changes + .and_then(|r| r.submitted_at) + .or(Some(revision_at)), + review_comment: last_changes.and_then(|r| non_empty(&r.body)), + revision_requested_at: Some(revision_at), + ..Default::default() + }; + } + + // 优先级 6：open 无待更新 → pending + StatusMapping { + status: STATUS_PENDING.to_string(), + ..Default::default() + } +} + +impl PrReview { + pub fn reviewer(&self) -> &str { + self.user.as_ref().map(|u| u.login.as_str()).unwrap_or("") + } +} + +/// 时间上最后一条「构成结论」的 review（APPROVED / CHANGES_REQUESTED） +/// +/// GitHub 按提交时间升序返回 reviews；COMMENTED / PENDING 不是结论， +/// 若把它们也算作「最后一条」，一次普通评论就会覆盖掉之前的 APPROVED / CHANGES_REQUESTED。 +fn last_decisive_review(reviews: &[PrReview]) -> Option<&PrReview> { + reviews + .iter() + .filter(|r| { + (r.state.eq_ignore_ascii_case("APPROVED") + || r.state.eq_ignore_ascii_case("CHANGES_REQUESTED")) + && r.submitted_at.is_some() + }) + .max_by_key(|r| r.submitted_at) +} + +/// 从 PR 文件列表定位 `raw-lyrics/*.ttml`，返回 (仓库路径, 文件名) +pub fn find_ttml_file(files: &[PrFile]) -> Option<(String, String)> { + files.iter().find_map(|f| { + let path = f.filename.trim(); + if !path.starts_with("raw-lyrics/") || !path.ends_with(".ttml") { + return None; + } + let name = path.rsplit('/').next()?.to_string(); + if name.is_empty() { + return None; + } + Some((path.to_string(), name)) + }) +} + +/// BCP-47 语言标签 → 站点语言代码（见 model.SubmissionLanguage） +pub fn map_language(raw: Option<&str>) -> (&'static str, Vec) { + let code = raw.unwrap_or("").trim().to_ascii_lowercase(); + let primary = if code.starts_with("zh") { + "zh" + } else if code.starts_with("ja") { + "ja" + } else if code.starts_with("en") { + "en" + } else if code.starts_with("ko") { + "ko" + } else { + "others" + }; + (primary, vec![primary.to_string()]) +} + +/// 解析 TTML，返回站点 submissions 需要的元数据 +pub struct ParsedTtmlMeta { + pub title: String, + pub artist: String, + pub album: String, + pub ncm_id: String, + pub qq_id: String, + pub am_id: String, + pub spotify_id: String, + pub language: String, + pub languages: Vec, + pub ttml_author_github: String, +} + +/// 解析 TTML 字节流（复用 ttml_processor，与 /validate 口径一致） +pub fn parse_ttml_meta(bytes: &[u8]) -> Result { + use ttml_processor::model::PlatformId; + + let text = String::from_utf8_lossy(bytes); + let parsed = ttml_processor::parse_ttml(&text).context("解析 TTML 失败")?; + let meta = parsed.metadata; + + let join = |v: Option>| -> String { + v.map(|items| items.join(" / ")).unwrap_or_default() + }; + + let mut ncm_id = String::new(); + let mut qq_id = String::new(); + let mut am_id = String::new(); + let mut spotify_id = String::new(); + if let Some(ref ids) = meta.platform_ids { + for (key, values) in ids { + let first = values.first().cloned().unwrap_or_default(); + match key { + PlatformId::NcmMusicId => ncm_id = first, + PlatformId::QqMusicId => qq_id = first, + PlatformId::SpotifyId => spotify_id = first, + PlatformId::AppleMusicId => am_id = first, + } + } + } + + let (language, languages) = map_language(meta.language.as_deref()); + + Ok(ParsedTtmlMeta { + title: join(meta.title).chars().take(200).collect(), + artist: join(meta.artist).chars().take(200).collect(), + album: join(meta.album).chars().take(200).collect(), + ncm_id, + qq_id, + am_id, + spotify_id, + language: language.to_string(), + languages, + ttml_author_github: meta + .author_ids + .as_ref() + .and_then(|v| v.first().cloned()) + .unwrap_or_default(), + }) +} + +/// 站点 UserInfo JSON（对齐 model.UserInfo 的 json 字段） +pub fn user_info(login: &str, avatar: Option<&str>) -> serde_json::Value { + serde_json::json!({ + "username": login, + "displayName": login, + "avatar": avatar.unwrap_or(""), + }) +} + +/// 站点 UserInfo JSON（hub 用户：用户名 / 展示名 / 头像均独立指定） +/// +/// 迁移时把 GitHub login 换成 hub 用户信息，展示名与头像取自 Casdoor； +/// 查询不到时 displayName 回退为 username。 +pub fn hub_user_info(username: &str, display_name: &str, avatar: &str) -> serde_json::Value { + let display = display_name.trim(); + serde_json::json!({ + "username": username, + "displayName": if display.is_empty() { username } else { display }, + "avatar": avatar, + }) +} + +fn non_empty(s: &str) -> Option { + let trimmed = s.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::migration::github_api::{RestLabel, RestUser}; + + fn pr(state: &str, merged: bool, author: &str) -> PrInfo { + let now = Utc::now(); + PrInfo { + number: 8000, + title: "测试稿件".to_string(), + body: None, + state: state.to_string(), + merged, + merged_at: if merged { Some(now) } else { None }, + created_at: now, + updated_at: now, + closed_at: if state == "closed" { Some(now) } else { None }, + url: "u".to_string(), + author_login: author.to_string(), + labels: Vec::new(), + head_sha: "sha".to_string(), + } + } + + fn closed_event(actor: &str) -> TimelineEvent { + TimelineEvent { + event: "closed".to_string(), + actor: Some(RestUser { + login: actor.to_string(), + }), + created_at: Some(Utc::now()), + label: None, + } + } + + fn review(login: &str, state: &str, body: &str) -> PrReview { + PrReview { + user: Some(RestUser { + login: login.to_string(), + }), + state: state.to_string(), + body: body.to_string(), + submitted_at: Some(Utc::now()), + } + } + + #[test] + fn merged_maps_to_approved() { + let reviews = vec![review("reviewer1", "APPROVED", "通过")]; + let m = map_status(&pr("closed", true, "alice"), &[], &reviews); + assert_eq!(m.status, STATUS_APPROVED); + assert_eq!(m.reviewer.as_deref(), Some("reviewer1")); + assert_eq!(m.review_comment.as_deref(), Some("通过")); + } + + #[test] + fn approved_then_closed_without_merge_maps_to_approved() { + // 本仓库流程：审核员 APPROVED 后直接关闭、不 merge + let mut changes = review("ASyunagi", "CHANGES_REQUESTED", "需要更改"); + changes.submitted_at = Some(Utc::now() - chrono::Duration::hours(1)); + let approved = review("ASyunagi", "APPROVED", "通过"); + let m = map_status( + &pr("closed", false, "alice"), + &[closed_event("ASyunagi")], + &[changes, approved], + ); + assert_eq!(m.status, STATUS_APPROVED); + assert_eq!(m.reviewer.as_deref(), Some("ASyunagi")); + assert_eq!(m.review_comment.as_deref(), Some("通过")); + } + + #[test] + fn trailing_comment_review_does_not_override_approval() { + let approved = review("reviewer1", "APPROVED", "通过"); + let mut commented = review("someone", "COMMENTED", "随便说说"); + commented.submitted_at = Some(Utc::now() + chrono::Duration::seconds(1)); + let m = map_status(&pr("open", false, "alice"), &[], &[approved, commented]); + assert_eq!(m.status, STATUS_APPROVED); + } + + #[test] + fn author_closed_maps_to_closed() { + let m = map_status(&pr("closed", false, "alice"), &[closed_event("alice")], &[]); + assert_eq!(m.status, STATUS_CLOSED); + assert_eq!(m.closed_by.as_deref(), Some("alice")); + } + + #[test] + fn reviewer_closed_maps_to_rejected() { + let m = map_status( + &pr("closed", false, "alice"), + &[closed_event("reviewer1")], + &[], + ); + assert_eq!(m.status, STATUS_REJECTED); + assert_eq!(m.reviewer.as_deref(), Some("reviewer1")); + } + + #[test] + fn bot_closed_maps_to_closed() { + let m = map_status( + &pr("closed", false, "github-actions[bot]"), + &[closed_event(GITHUB_ACTIONS_BOT)], + &[], + ); + assert_eq!(m.status, STATUS_CLOSED); + assert_eq!(m.closed_by.as_deref(), Some(GITHUB_ACTIONS_BOT)); + } + + #[test] + fn open_with_revision_label_maps_to_need_revision_with_comment() { + let label = TimelineEvent { + event: "labeled".to_string(), + actor: Some(RestUser { + login: "ASyunagi".to_string(), + }), + created_at: Some(Utc::now()), + label: Some(RestLabel { + name: REVISION_LABEL.to_string(), + }), + }; + let m = map_status( + &pr("open", false, "github-actions[bot]"), + &[label], + &[review("ASyunagi", "CHANGES_REQUESTED", "第 14 行结束时间需延后")], + ); + assert_eq!(m.status, STATUS_NEED_REVISION); + assert_eq!(m.reviewer.as_deref(), Some("ASyunagi")); + assert_eq!(m.review_comment.as_deref(), Some("第 14 行结束时间需延后")); + assert!(m.revision_requested_at.is_some()); + } + + #[test] + fn open_with_label_maps_to_need_revision() { + let timeline = vec![TimelineEvent { + event: "labeled".to_string(), + actor: None, + created_at: Some(Utc::now()), + label: Some(crate::migration::github_api::RestLabel { + name: REVISION_LABEL.to_string(), + }), + }]; + let m = map_status(&pr("open", false, "alice"), &timeline, &[]); + assert_eq!(m.status, STATUS_NEED_REVISION); + assert!(m.revision_requested_at.is_some()); + } + + #[test] + fn open_without_label_maps_to_pending() { + let m = map_status(&pr("open", false, "alice"), &[], &[]); + assert_eq!(m.status, STATUS_PENDING); + } + + #[test] + fn finds_ttml_in_raw_lyrics() { + let files = vec![ + PrFile { + filename: "metadata/index.jsonl".to_string(), + }, + PrFile { + filename: "raw-lyrics/1700000000000-1-abcdefgh.ttml".to_string(), + }, + ]; + let (path, name) = find_ttml_file(&files).unwrap(); + assert_eq!(path, "raw-lyrics/1700000000000-1-abcdefgh.ttml"); + assert_eq!(name, "1700000000000-1-abcdefgh.ttml"); + assert!(find_ttml_file(&[]).is_none()); + } + + #[test] + fn maps_language_codes() { + assert_eq!(map_language(Some("zh-Hans")).0, "zh"); + assert_eq!(map_language(Some("ja")).0, "ja"); + assert_eq!(map_language(Some("en-US")).0, "en"); + assert_eq!(map_language(Some("ko")).0, "ko"); + assert_eq!(map_language(None).0, "others"); + assert_eq!(map_language(Some("fr")).0, "others"); + } +} diff --git a/worker/src/migration/mod.rs b/worker/src/migration/mod.rs new file mode 100644 index 0000000..62d2258 --- /dev/null +++ b/worker/src/migration/mod.rs @@ -0,0 +1,4 @@ +pub mod consumer; +pub mod github_api; +pub mod mapper; +pub mod store; diff --git a/worker/src/migration/store.rs b/worker/src/migration/store.rs new file mode 100644 index 0000000..01052be --- /dev/null +++ b/worker/src/migration/store.rs @@ -0,0 +1,454 @@ +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use sea_orm::{ + ConnectionTrait, DatabaseBackend, DatabaseConnection, FromQueryResult, Statement, + TransactionTrait, Value, +}; + +/// 迁移任务当前状态（读取自 github_migration_tasks） +#[derive(Debug, Clone, FromQueryResult)] +pub struct TaskRow { + pub processed_prs: i32, + pub created_count: i32, + pub skipped_count: i32, + pub failed_count: i32, + pub cursor: Option, + pub close_pr_status: String, +} + +/// 已迁移 PR（关闭 PR 阶段使用） +#[derive(Debug, Clone, FromQueryResult)] +pub struct MigratedPrRow { + pub number: i64, + pub file_name: Option, +} + +/// GitHub 绑定记录（github_login → 站点 username） +#[derive(Debug, Clone, FromQueryResult)] +pub struct GithubBindingRow { + pub github_login: String, + pub username: String, +} + +/// 审核历史写入项 +pub struct ReviewHistoryInsert { + pub reviewer: String, + pub reviewer_info: serde_json::Value, + pub status: String, + pub comment: String, + pub reviewed_at: DateTime, +} + +/// 评论写入项 +pub struct CommentInsert { + pub author: serde_json::Value, + pub content: String, + pub created_at: DateTime, +} + +/// 文件更新历史写入项 +pub struct FileHistoryInsert { + pub uploader: String, + pub uploader_info: serde_json::Value, + pub file_name: String, + pub uploaded_at: DateTime, +} + +/// 单条 PR → 站点投稿的完整写入数据 +pub struct SubmissionInsert { + pub pr_number: i64, + pub username: String, + pub title: String, + pub artist: String, + pub album: String, + pub ncm_id: String, + pub qq_id: String, + pub am_id: String, + pub spotify_id: String, + pub file_name: String, + pub tags: serde_json::Value, + pub metadata: serde_json::Value, + pub language: String, + pub languages: serde_json::Value, + pub status: String, + pub submitter: String, + pub submitter_info: serde_json::Value, + pub created_at: DateTime, + pub updated_at: DateTime, + pub revision_requested_at: Option>, + pub closed_at: Option>, + pub closed_by: Option, + pub closed_by_info: Option, + pub reviewer: Option, + pub reviewed_at: Option>, + pub review_comment: Option, + pub reviews: Vec, + pub comments: Vec, + pub file_history: Option, +} + +/// 迁移相关的数据库读写 +#[derive(Clone)] +pub struct MigrationStore { + db: DatabaseConnection, +} + +impl MigrationStore { + pub fn new(db: DatabaseConnection) -> Self { + Self { db } + } + + // ===== 任务进度 ===== + + pub async fn get_task(&self, task_id: i64) -> Result> { + let sql = r#"SELECT processed_prs, created_count, skipped_count, failed_count, + cursor, close_pr_status + FROM github_migration_tasks WHERE id = $1"#; + Ok(TaskRow::find_by_statement(stmt(sql, vec![task_id.into()])) + .one(&self.db) + .await?) + } + + pub async fn mark_running(&self, task_id: i64) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET status = 'running', error = NULL WHERE id = $1", + vec![task_id.into()], + ) + .await + } + + pub async fn set_total(&self, task_id: i64, total: i32) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET total_prs = $2 WHERE id = $1", + vec![task_id.into(), total.into()], + ) + .await + } + + /// 刷新进度与断点游标（每处理 N 条调用一次） + pub async fn update_progress( + &self, + task_id: i64, + processed: i32, + created: i32, + skipped: i32, + failed: i32, + cursor: Option<&str>, + ) -> Result<()> { + let sql = r#"UPDATE github_migration_tasks + SET processed_prs = $2, created_count = $3, skipped_count = $4, + failed_count = $5, cursor = $6 + WHERE id = $1"#; + self.exec( + sql, + vec![ + task_id.into(), + processed.into(), + created.into(), + skipped.into(), + failed.into(), + cursor.map(|s| s.to_string()).into(), + ], + ) + .await + } + + pub async fn mark_completed( + &self, + task_id: i64, + processed: i32, + created: i32, + skipped: i32, + failed: i32, + ) -> Result<()> { + let sql = r#"UPDATE github_migration_tasks + SET status = 'completed', processed_prs = $2, created_count = $3, + skipped_count = $4, failed_count = $5, completed_at = NOW() + WHERE id = $1"#; + self.exec( + sql, + vec![ + task_id.into(), + processed.into(), + created.into(), + skipped.into(), + failed.into(), + ], + ) + .await + } + + pub async fn mark_failed(&self, task_id: i64, error: &str) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET status = 'failed', error = $2 WHERE id = $1", + vec![task_id.into(), error.to_string().into()], + ) + .await + } + + pub async fn mark_close_running(&self, task_id: i64) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET close_pr_status = 'running' WHERE id = $1", + vec![task_id.into()], + ) + .await + } + + pub async fn set_close_status(&self, task_id: i64, status: &str) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET close_pr_status = $2 WHERE id = $1", + vec![task_id.into(), status.to_string().into()], + ) + .await + } + + pub async fn add_closed_count(&self, task_id: i64, delta: i32) -> Result<()> { + self.exec( + "UPDATE github_migration_tasks SET closed_pr_count = closed_pr_count + $2 WHERE id = $1", + vec![task_id.into(), delta.into()], + ) + .await + } + + // ===== 已迁移 PR ===== + + pub async fn has_migrated_pr(&self, pr_number: i64) -> Result { + let sql = "SELECT 1 AS number FROM github_migrated_prs WHERE pr_number = $1"; + let row = MigratedPrRow::find_by_statement(stmt(sql, vec![pr_number.into()])) + .one(&self.db) + .await?; + Ok(row.is_some()) + } + + /// 查询该用户已迁移的 PR（附带稿件文件名，用于关闭 PR 时的迁移地址评论） + pub async fn list_migrated_prs(&self, username: &str, since: i64) -> Result> { + let sql = r#"SELECT g.pr_number AS number, s.file_name AS file_name + FROM github_migrated_prs g + LEFT JOIN submissions s ON s.id = g.submission_id + WHERE g.username = $1 AND g.pr_number >= $2 + ORDER BY g.pr_number DESC"#; + Ok( + MigratedPrRow::find_by_statement(stmt( + sql, + vec![username.to_string().into(), since.into()], + )) + .all(&self.db) + .await?, + ) + } + + /// 按 github_login（大小写不敏感）批量查询绑定关系 + pub async fn find_bindings(&self, github_logins: &[String]) -> Result> { + if github_logins.is_empty() { + return Ok(Vec::new()); + } + let placeholders: Vec = (1..=github_logins.len()).map(|i| format!("${}", i)).collect(); + let sql = format!( + "SELECT github_login, username FROM user_github_bindings WHERE lower(github_login) IN ({})", + placeholders.join(", ") + ); + let params: Vec = github_logins + .iter() + .map(|login| login.to_ascii_lowercase().into()) + .collect(); + Ok( + GithubBindingRow::find_by_statement(stmt(&sql, params)) + .all(&self.db) + .await?, + ) + } + + // ===== 写入稿件 ===== + + /// 在单个事务内写入 submissions / 审核历史 / 评论 / 文件历史 / 已迁移 PR 记录 + /// + /// 返回 `Ok(None)` 表示该 PR 已被其他任务迁移（唯一约束冲突），本次跳过。 + pub async fn insert_submission(&self, data: &SubmissionInsert) -> Result> { + let txn = self.db.begin().await?; + + let submission_id = insert_submission_row(&txn, data).await?; + + // 全局唯一兜底：并发下若已被迁移，回滚整条事务 + let inserted = insert_migrated_pr(&txn, data.pr_number, &data.username, submission_id).await?; + if !inserted { + txn.rollback().await.context("rollback duplicated migration")?; + return Ok(None); + } + + for review in &data.reviews { + insert_review_history(&txn, submission_id, review).await?; + } + for comment in &data.comments { + insert_comment(&txn, submission_id, comment).await?; + } + if let Some(file) = &data.file_history { + insert_file_history(&txn, submission_id, file).await?; + } + + txn.commit().await.context("commit migrated submission")?; + Ok(Some(submission_id)) + } + + async fn exec(&self, sql: &str, params: Vec) -> Result<()> { + self.db + .execute(stmt(sql, params)) + .await + .with_context(|| format!("执行 SQL 失败: {}", first_line(sql)))?; + Ok(()) + } +} + +async fn insert_submission_row( + conn: &C, + data: &SubmissionInsert, +) -> Result { + let sql = r#"INSERT INTO submissions ( + title, artist, album, ncm_id, qq_id, am_id, spotify_id, file_name, + notes, tags, metadata, language, languages, + status, submitter, submitter_info, provider, + created_at, updated_at, + revision_requested_at, closed_at, closed_by, closed_by_info, + reviewer, reviewed_at, review_comment + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, + '', $9::jsonb, $10::jsonb, $11, $12::jsonb, + $13, $14, $15::jsonb, 'github_migration', + $16, $17, + $18, $19, $20, $21::jsonb, + $22, $23, $24 + ) RETURNING id"#; + + let params: Vec = vec![ + data.title.clone().into(), + data.artist.clone().into(), + data.album.clone().into(), + data.ncm_id.clone().into(), + data.qq_id.clone().into(), + data.am_id.clone().into(), + data.spotify_id.clone().into(), + data.file_name.clone().into(), + data.tags.clone().into(), + data.metadata.clone().into(), + data.language.clone().into(), + data.languages.clone().into(), + data.status.clone().into(), + data.submitter.clone().into(), + data.submitter_info.clone().into(), + data.created_at.into(), + data.updated_at.into(), + data.revision_requested_at.into(), + data.closed_at.into(), + data.closed_by.clone().into(), + data.closed_by_info.clone().into(), + data.reviewer.clone().into(), + data.reviewed_at.into(), + data.review_comment.clone().into(), + ]; + + let row = conn + .query_one(stmt(sql, params)) + .await + .context("插入 submissions 失败")? + .ok_or_else(|| anyhow::anyhow!("插入 submissions 未返回 id"))?; + row.try_get::("", "id") + .context("读取新生成的 submission id 失败") +} + +async fn insert_migrated_pr( + conn: &C, + pr_number: i64, + username: &str, + submission_id: i64, +) -> Result { + let sql = r#"INSERT INTO github_migrated_prs (pr_number, username, submission_id) + VALUES ($1, $2, $3) + ON CONFLICT (pr_number) DO NOTHING"#; + let res = conn + .execute(stmt( + sql, + vec![ + pr_number.into(), + username.to_string().into(), + submission_id.into(), + ], + )) + .await + .context("写入 github_migrated_prs 失败")?; + Ok(res.rows_affected() > 0) +} + +async fn insert_review_history( + conn: &C, + submission_id: i64, + review: &ReviewHistoryInsert, +) -> Result<()> { + let sql = r#"INSERT INTO submission_review_history + (submission_id, reviewer, reviewer_info, status, comment, reviewed_at) + VALUES ($1, $2, $3::jsonb, $4, $5, $6)"#; + conn.execute(stmt( + sql, + vec![ + submission_id.into(), + review.reviewer.clone().into(), + review.reviewer_info.clone().into(), + review.status.clone().into(), + review.comment.clone().into(), + review.reviewed_at.into(), + ], + )) + .await + .context("写入 submission_review_history 失败")?; + Ok(()) +} + +async fn insert_comment( + conn: &C, + submission_id: i64, + comment: &CommentInsert, +) -> Result<()> { + let sql = r#"INSERT INTO submission_comments (submission_id, author, content, created_at) + VALUES ($1, $2::jsonb, $3, $4)"#; + conn.execute(stmt( + sql, + vec![ + submission_id.into(), + comment.author.clone().into(), + comment.content.clone().into(), + comment.created_at.into(), + ], + )) + .await + .context("写入 submission_comments 失败")?; + Ok(()) +} + +async fn insert_file_history( + conn: &C, + submission_id: i64, + file: &FileHistoryInsert, +) -> Result<()> { + let sql = r#"INSERT INTO submission_file_history + (submission_id, uploader, uploader_info, file_name, uploaded_at) + VALUES ($1, $2, $3::jsonb, $4, $5)"#; + conn.execute(stmt( + sql, + vec![ + submission_id.into(), + file.uploader.clone().into(), + file.uploader_info.clone().into(), + file.file_name.clone().into(), + file.uploaded_at.into(), + ], + )) + .await + .context("写入 submission_file_history 失败")?; + Ok(()) +} + +fn stmt(sql: &str, params: Vec) -> Statement { + Statement::from_sql_and_values(DatabaseBackend::Postgres, sql, params) +} + +fn first_line(sql: &str) -> &str { + sql.lines().next().unwrap_or("").trim() +}