diff --git a/backend/pom.xml b/backend/pom.xml index bf0c898..a392a88 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -177,6 +177,12 @@ springdoc-openapi-starter-webmvc-ui 2.3.0 + + + com.tencentcloudapi + tencentcloud-speech-sdk-java + 1.0.67 + diff --git a/backend/src/main/java/com/imeeting/common/RedisKeys.java b/backend/src/main/java/com/imeeting/common/RedisKeys.java index 4cb6545..9faf822 100644 --- a/backend/src/main/java/com/imeeting/common/RedisKeys.java +++ b/backend/src/main/java/com/imeeting/common/RedisKeys.java @@ -71,6 +71,10 @@ public final class RedisKeys { return "biz:meeting:realtime:state:" + meetingId; } + public static String realtimeMeetingTranscriptCacheKey(Long meetingId) { + return "biz:meeting:realtime:transcript-cache:" + meetingId; + } + public static String realtimeMeetingResumeTimeoutKey(Long meetingId) { return realtimeMeetingResumeTimeoutPrefix() + meetingId; } diff --git a/backend/src/main/java/com/imeeting/common/SysParamKeys.java b/backend/src/main/java/com/imeeting/common/SysParamKeys.java index 6c6cb55..fcdc4d0 100644 --- a/backend/src/main/java/com/imeeting/common/SysParamKeys.java +++ b/backend/src/main/java/com/imeeting/common/SysParamKeys.java @@ -10,6 +10,10 @@ public final class SysParamKeys { public static final String CAPTCHA_ENABLED = "security.captcha.enabled"; /** AI 会议总结使用的系统提示词。 */ public static final String MEETING_SUMMARY_SYSTEM_PROMPT = "meeting.summary.system_prompt"; + /** 是否启用 AI 目录。 */ + public static final String MEETING_AI_CATALOG_ENABLED = "meeting.ai_catalog.enabled"; + /** 会议总结派发模式：PARALLEL / SERIAL。 */ + public static final String MEETING_SUMMARY_DISPATCH_MODE = "meeting.summary.dispatch_mode"; /** 离线会议音频上传大小上限，单位 MB。 */ public static final String MEETING_OFFLINE_AUDIO_MAX_SIZE_MB = "meeting.offline_audio.max_size_mb"; /** 是否允许创建离线会议。 */ diff --git a/backend/src/main/java/com/imeeting/config/MeetingAsyncExecutorConfig.java b/backend/src/main/java/com/imeeting/config/MeetingAsyncExecutorConfig.java index eeb8fbe..9aa9342 100644 --- a/backend/src/main/java/com/imeeting/config/MeetingAsyncExecutorConfig.java +++ b/backend/src/main/java/com/imeeting/config/MeetingAsyncExecutorConfig.java @@ -36,4 +36,16 @@ public class MeetingAsyncExecutorConfig { executor.initialize(); return executor; } + + @Bean("chunkMergeExecutor") + public Executor chunkMergeExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(4); + executor.setQueueCapacity(10); + executor.setThreadNamePrefix("imeeting-chunk-merge-"); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + executor.initialize(); + return executor; + } } diff --git a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingChunkUploadController.java b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingChunkUploadController.java index e48ac49..4db3e38 100644 --- a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingChunkUploadController.java +++ b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingChunkUploadController.java @@ -49,7 +49,7 @@ public class AndroidMeetingChunkUploadController { "meetingId", meetingId, "chunkIndex", chunkIndex, "chunkFile", chunkFile); - AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); + AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request,true); androidChunkUploadService.saveChunk(meetingId, chunkIndex, chunkFile, authContext); return ApiResponse.ok(true); } @@ -71,6 +71,7 @@ public class AndroidMeetingChunkUploadController { "meetingId", meetingId, "totalChunks", totalChunks); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); - return ApiResponse.ok(androidChunkUploadService.completeUpload(meetingId, totalChunks, authContext)); + androidChunkUploadService.completeUploadAsync(meetingId, totalChunks, authContext); + return ApiResponse.ok(new LegacyUploadAudioResponse(meetingId, null, "后台合并上传中")); } } diff --git a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingController.java b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingController.java index 5729293..0519e41 100644 --- a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingController.java +++ b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingController.java @@ -56,6 +56,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.util.StopWatch; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -79,6 +80,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @Tag(name = "Android会议接口") @@ -97,7 +99,7 @@ public class AndroidMeetingController { private String h5BaseUrl; private final AndroidAuthService androidAuthService; - private final AndroidMeetingPushService androidMeetingPushService; + private final AndroidMeetingPushService androidMeetingPushService; private final AndroidChunkUploadService androidChunkUploadService; private final LegacyMeetingAdapterService legacyMeetingAdapterService; private final MeetingQueryService meetingQueryService; @@ -148,7 +150,7 @@ public class AndroidMeetingController { this.paramService = paramService; this.dictItemService = dictItemService; this.meetingUnifiedStatusService = meetingUnifiedStatusService; - this.androidMeetingPushService = androidMeetingPushService; + this.androidMeetingPushService = androidMeetingPushService; } @Operation(summary = "创建Android离线会议") @@ -238,22 +240,24 @@ public class AndroidMeetingController { AndroidRequestLogHelper.logRequest(log, "Android会议", "结束离线会议录音阶段", "meetingId", meetingId, "request", command); - AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); + AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request, true); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); - MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser); - LegacyUploadAudioResponse uploadResult = null; + requireOperableOfflineMeeting(meetingId, authContext, loginUser); + MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId); + LegacyUploadAudioResponse uploadResult = new LegacyUploadAudioResponse(); if (isUploadFinishedStage(command)) { - uploadResult = androidChunkUploadService.completeUpload( + androidChunkUploadService.completeUploadAsync( meeting.getId(), - command == null ? null : command.getTotalChunks(), + command.getTotalChunks(), authContext ); - if (uploadResult == null || uploadResult.getAudioUrl() == null || uploadResult.getAudioUrl().isBlank()) { - throw new RuntimeException("分片上传完成后未生成 audio_url"); - } +// if (uploadResult == null) { +// throw new RuntimeException("分片上传完成后未生成结果"); +// } + uploadResult.setMeetingId(meetingId); } meetingCommandService.finishOfflineMeeting(meeting.getId(), command == null ? null : command.getFinishStage()); - return ApiResponse.ok(uploadResult != null ? uploadResult : true); + return ApiResponse.ok(uploadResult); } @Operation(summary = "分页查询Android会议") @@ -292,7 +296,7 @@ public class AndroidMeetingController { } - @Operation(summary = "查询Android会议统一状态") + @Operation(summary = "查询Android会议统一状态") @ApiResponses({ @io.swagger.v3.oas.annotations.responses.ApiResponse( responseCode = "200", @@ -310,13 +314,14 @@ public class AndroidMeetingController { "request", command); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); - MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser); + requireOperableOfflineMeeting(meetingId, authContext, loginUser); + MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId,false); UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId); boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript()); boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary()); List transcripts = includeTranscript ? meetingQueryService.getTranscripts(meetingId) : null; - String summaryContent = includeSummary ? meetingQueryService.getDetailIgnoreTenant(meetingId).getSummaryContent() : null; - return ApiResponse.ok(AndroidUnifiedMeetingStatusResponse.builder() + String summaryContent = includeSummary ? meeting.getSummaryContent() : null; + AndroidUnifiedMeetingStatusResponse build = AndroidUnifiedMeetingStatusResponse.builder() .meetingId(meetingId) .status(status) .meeting(meeting) @@ -324,7 +329,9 @@ public class AndroidMeetingController { .transcripts(transcripts) .includesSummary(includeSummary) .summaryContent(summaryContent) - .build()); + .build(); + log.info("[{}]{}.返回数据:[{}]", "Android会议", "查询会议统一状态", build); + return ApiResponse.ok(build); } @Operation(summary = "重试 Android 会议 ASR 识别") @@ -343,7 +350,7 @@ public class AndroidMeetingController { LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); requireOperableOfflineMeeting(meetingId, authContext, loginUser); meetingCommandService.retryTranscription(meetingId); - androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode()); + androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode()); return ApiResponse.ok(true); } @@ -364,17 +371,18 @@ public class AndroidMeetingController { LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); requireOperableOfflineMeeting(meetingId, authContext, loginUser); meetingCommandService.retrySummary(meetingId); - androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode()); - return ApiResponse.ok(true); + androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode()); + return ApiResponse.ok(true); } - @Operation(summary = "更新Android会议访问密码") - @ApiResponses({ - @io.swagger.v3.oas.annotations.responses.ApiResponse( - responseCode = "200", - description = "返回更新后的会议访问密码，传空时表示清空访问密码", - content = @Content(schema = @Schema(implementation = String.class)) - ) - }) + + @Operation(summary = "更新Android会议访问密码") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse( + responseCode = "200", + description = "返回更新后的会议访问密码，传空时表示清空访问密码", + content = @Content(schema = @Schema(implementation = String.class)) + ) + }) @PutMapping("/{meetingId}/access-password") @Log(value = "修改Android会议访问密码", type = "Android会议管理") public ApiResponse updateAccessPassword(HttpServletRequest request, @@ -390,9 +398,9 @@ public class AndroidMeetingController { return ApiResponse.error("仅会议创建人可设置访问密码"); } String password = normalizePassword(command == null ? null : command.getPassword()); - meetingService.update(new LambdaUpdateWrapper() - .eq(Meeting::getId,meeting.getId()) - .set(Meeting::getAccessPassword, password)); + meetingService.update(new LambdaUpdateWrapper() + .eq(Meeting::getId, meeting.getId()) + .set(Meeting::getAccessPassword, password)); return ApiResponse.ok(password); } @@ -415,54 +423,55 @@ public class AndroidMeetingController { meetingCommandService.deleteMeeting(meetingId); return ApiResponse.ok(true); } - @GetMapping("/config") - @Log(value = "获取会议配置", type = "Android会议管理") - @Operation(summary = "获取会议配置") - @Anonymous - public ApiResponse config(HttpServletRequest request) { - AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口"); - AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); - LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext); - Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId(); - Long userId = loginUser != null ? loginUser.getUserId() : null; - boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin()); - boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin()); - AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo(); - PageResult> promptTemplateList = promptTemplateService.pageTemplates( - 1, - 1000, - null, - null, - tenantId, - userId, - isPlatformAdmin, - isTenantAdmin - ); - List enabledTemplates = promptTemplateList.getRecords() == null - ? List.of() - : promptTemplateList.getRecords().stream() - .filter(item -> Integer.valueOf(1).equals(item.getStatus())) - .toList(); - resultVo.setTemplateList(enabledTemplates); - PageResult> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId); - List enabledModels = modelList.getRecords() == null - ? List.of() - : modelList.getRecords().stream() - .filter(item -> Integer.valueOf(1).equals(item.getStatus())) - .toList(); - resultVo.setModelsList(enabledModels); - resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail")); - resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION,"30"))); - resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10"))); - resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION,String.valueOf(60*4)))); - BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99")); - bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP); - resultVo.setPacketLossRate(bigDecimal ); - resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false"))); - resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60"))); - return ApiResponse.ok(resultVo); - } + @GetMapping("/config") + @Log(value = "获取会议配置", type = "Android会议管理") + @Operation(summary = "获取会议配置") + @Anonymous + public ApiResponse config(HttpServletRequest request) { + AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口"); + AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); + LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext); + Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId(); + Long userId = loginUser != null ? loginUser.getUserId() : null; + boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin()); + boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin()); + AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo(); + PageResult> promptTemplateList = promptTemplateService.pageTemplates( + 1, + 1000, + null, + null, + tenantId, + userId, + isPlatformAdmin, + isTenantAdmin + ); + List enabledTemplates = promptTemplateList.getRecords() == null + ? List.of() + : promptTemplateList.getRecords().stream() + .filter(item -> Integer.valueOf(1).equals(item.getStatus())) + .toList(); + resultVo.setTemplateList(enabledTemplates); + PageResult> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId); + List enabledModels = modelList.getRecords() == null + ? List.of() + : modelList.getRecords().stream() + .filter(item -> Integer.valueOf(1).equals(item.getStatus())) + .toList(); + resultVo.setModelsList(enabledModels); + resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail")); + resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION, "30"))); + resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10"))); + resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, String.valueOf(60 * 4)))); + BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99")); + bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP); + resultVo.setPacketLossRate(bigDecimal); + resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false"))); + resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60"))); + + return ApiResponse.ok(resultVo); + } private void resolvePublicDeviceTenantId(HttpServletRequest request, AndroidOfflineMeetingCreateCommand command, @@ -545,29 +554,14 @@ public class AndroidMeetingController { return ChronoUnit.DAYS.between(LocalDate.now(), meetingTime.toLocalDate()); } - private MeetingVO requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) { - MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId); - if (meeting == null) { - throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(), "会议不存在"); - } + private Meeting requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) { + Meeting meeting = meetingAccessService.requireMeetingIgnoreTenant(meetingId); if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) { throw new RuntimeException("当前会议不是离线会议"); } if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) { throw new RuntimeException("设备ID不能为空"); } -// if (meeting.getSourceDeviceCode() == null || !meeting.getSourceDeviceCode().equals(authContext.getDeviceId())) { -// throw new RuntimeException("当前会议不属于该设备"); -// } -// if (authContext.isAnonymous()) { -// if (!MeetingConstants.DEVICE_MODE_PUBLIC.equals(meeting.getSourceDeviceMode())) { -// throw new RuntimeException("当前会议不是公有设备会议"); -// } -// return meeting; -// } -// if (loginUser == null || !Objects.equals(meeting.getCreatorId(), loginUser.getUserId())) { -// throw new RuntimeException("仅会议创建人可操作当前会议"); -// } return meeting; } diff --git a/backend/src/main/java/com/imeeting/controller/biz/AiModelController.java b/backend/src/main/java/com/imeeting/controller/biz/AiModelController.java index 9bd38a1..71af7c6 100644 --- a/backend/src/main/java/com/imeeting/controller/biz/AiModelController.java +++ b/backend/src/main/java/com/imeeting/controller/biz/AiModelController.java @@ -3,6 +3,7 @@ package com.imeeting.controller.biz; import com.imeeting.dto.biz.AiLocalProfileVO; import com.imeeting.dto.biz.AiModelDTO; import com.imeeting.dto.biz.AiModelVO; +import com.imeeting.enums.ModelProviderEnum; import com.imeeting.service.biz.AiModelService; import com.unisbase.common.ApiResponse; import com.unisbase.common.annotation.Log; @@ -121,7 +122,7 @@ public class AiModelController { @PostMapping("/llm-connectivity-test") @PreAuthorize("isAuthenticated()") public ApiResponse testLlmConnectivity(@RequestBody AiModelDTO dto) { - if ("custom".equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) { + if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) { return ApiResponse.error("基础地址不能为空"); } if (dto.getModelCode() == null || dto.getModelCode().isBlank()) { diff --git a/backend/src/main/java/com/imeeting/controller/biz/MeetingController.java b/backend/src/main/java/com/imeeting/controller/biz/MeetingController.java index 8ebcdaa..ffdbc8e 100644 --- a/backend/src/main/java/com/imeeting/controller/biz/MeetingController.java +++ b/backend/src/main/java/com/imeeting/controller/biz/MeetingController.java @@ -212,6 +212,7 @@ public class MeetingController { MeetingCreateConfigVO vo = new MeetingCreateConfigVO(); vo.setOfflineEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_OFFLINE_ENABLED, true)); vo.setRealtimeEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_REALTIME_ENABLED, true)); + vo.setAiCatalogEnabled(resolveBooleanParam(SysParamKeys.MEETING_AI_CATALOG_ENABLED, false)); vo.setOfflineAudioMaxSizeMb(resolveLongParam(SysParamKeys.MEETING_OFFLINE_AUDIO_MAX_SIZE_MB, 1024L)); return ApiResponse.ok(vo); } @@ -398,17 +399,6 @@ public class MeetingController { return ApiResponse.ok(result); } - @Operation(summary = "追加实时转写片段") - @PostMapping("/{id}/realtime/transcripts") - @PreAuthorize("isAuthenticated()") - public ApiResponse appendRealtimeTranscripts(@PathVariable Long id, @RequestBody List items) { - LoginUser loginUser = currentLoginUser(); - Meeting meeting = meetingAccessService.requireMeeting(id); - meetingAccessService.assertCanControlRealtimeMeeting(meeting, loginUser, MeetingConstants.SOURCE_WEB); - meetingCommandService.appendRealtimeTranscripts(id, items); - return ApiResponse.ok(true); - } - @Operation(summary = "暂停实时会议") @PostMapping("/{id}/realtime/pause") @PreAuthorize("isAuthenticated()") diff --git a/backend/src/main/java/com/imeeting/dto/android/AndroidUnifiedMeetingStatusResponse.java b/backend/src/main/java/com/imeeting/dto/android/AndroidUnifiedMeetingStatusResponse.java index 839befc..04f7ae4 100644 --- a/backend/src/main/java/com/imeeting/dto/android/AndroidUnifiedMeetingStatusResponse.java +++ b/backend/src/main/java/com/imeeting/dto/android/AndroidUnifiedMeetingStatusResponse.java @@ -6,12 +6,14 @@ import com.imeeting.dto.biz.UnifiedMeetingStatusVO; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Builder; import lombok.Data; +import lombok.ToString; import java.util.List; @Data @Builder @Schema(description = "Android 统一会议状态响应") +@ToString public class AndroidUnifiedMeetingStatusResponse { @Schema(description = "会议ID") private Long meetingId; diff --git a/backend/src/main/java/com/imeeting/dto/android/legacy/LegacyUploadAudioResponse.java b/backend/src/main/java/com/imeeting/dto/android/legacy/LegacyUploadAudioResponse.java index 30f506c..5b48574 100644 --- a/backend/src/main/java/com/imeeting/dto/android/legacy/LegacyUploadAudioResponse.java +++ b/backend/src/main/java/com/imeeting/dto/android/legacy/LegacyUploadAudioResponse.java @@ -18,4 +18,12 @@ public class LegacyUploadAudioResponse { @JsonProperty("audio_url") @Schema(description = "上传后的音频访问地址") private String audioUrl; + + @JsonProperty("message") + @Schema(description = "结果提示") + private String message; + + public LegacyUploadAudioResponse(Long meetingId, String audioUrl) { + this(meetingId, audioUrl, null); + } } diff --git a/backend/src/main/java/com/imeeting/dto/biz/MeetingCreateConfigVO.java b/backend/src/main/java/com/imeeting/dto/biz/MeetingCreateConfigVO.java index cf7d4f5..fa8ef22 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/MeetingCreateConfigVO.java +++ b/backend/src/main/java/com/imeeting/dto/biz/MeetingCreateConfigVO.java @@ -13,6 +13,9 @@ public class MeetingCreateConfigVO { @Schema(description = "是否启用实时会议") private Boolean realtimeEnabled; + @Schema(description = "是否启用 AI 目录") + private Boolean aiCatalogEnabled; + @Schema(description = "离线音频上传大小上限，单位 MB") private Long offlineAudioMaxSizeMb; } diff --git a/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java b/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java index 66c89c5..e6426b9 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java +++ b/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java @@ -75,6 +75,9 @@ public class MeetingVO { @Schema(description = "总结模板ID") private Long promptId; + @Schema(description = "是否启用 AI 目录") + private Boolean aiCatalogEnabled; + @Schema(description = "音频保存状态") private String audioSaveStatus; diff --git a/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingResumeConfig.java b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingResumeConfig.java index 7906eba..ab46c51 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingResumeConfig.java +++ b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingResumeConfig.java @@ -27,4 +27,6 @@ public class RealtimeMeetingResumeConfig { private Boolean saveAudio; @Schema(description = "热词列表") private List> hotwords; + @Schema(description = "腾讯说话人上下文 ID") + private String speakerContextId; } diff --git a/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheItem.java b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheItem.java new file mode 100644 index 0000000..f1941d9 --- /dev/null +++ b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheItem.java @@ -0,0 +1,22 @@ +package com.imeeting.dto.biz; + +import lombok.Data; + +@Data +public class RealtimeMeetingTranscriptCacheItem { + private String sentenceKey; + private String sentenceGroupKey; + private Integer sentenceId; + private Integer sentenceType; + private String speakerId; + private String speakerName; + private String userId; + private Integer startTime; + private Integer endTime; + private String content; + private Integer sortOrder; + private Boolean finalResult; + private Long transcriptId; + private Long firstReceivedAt; + private Long updatedAt; +} diff --git a/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheState.java b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheState.java new file mode 100644 index 0000000..8beaf3a --- /dev/null +++ b/backend/src/main/java/com/imeeting/dto/biz/RealtimeMeetingTranscriptCacheState.java @@ -0,0 +1,15 @@ +package com.imeeting.dto.biz; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class RealtimeMeetingTranscriptCacheState { + private Long meetingId; + private Integer nextSortOrder; + private Integer nextLegacySequence; + private List items = new ArrayList<>(); + private Long updatedAt; +} diff --git a/backend/src/main/java/com/imeeting/dto/biz/RealtimeSocketSessionData.java b/backend/src/main/java/com/imeeting/dto/biz/RealtimeSocketSessionData.java index c9f6ead..6bd122f 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/RealtimeSocketSessionData.java +++ b/backend/src/main/java/com/imeeting/dto/biz/RealtimeSocketSessionData.java @@ -2,11 +2,16 @@ package com.imeeting.dto.biz; import lombok.Data; +import java.util.Map; + @Data public class RealtimeSocketSessionData { private Long meetingId; private Long userId; private Long tenantId; private Long asrModelId; + private String provider; private String targetWsUrl; + private String modelCode; + private Map mediaConfig; } diff --git a/backend/src/main/java/com/imeeting/enums/ModelProviderEnum.java b/backend/src/main/java/com/imeeting/enums/ModelProviderEnum.java new file mode 100644 index 0000000..5d241bd --- /dev/null +++ b/backend/src/main/java/com/imeeting/enums/ModelProviderEnum.java @@ -0,0 +1,17 @@ +package com.imeeting.enums; + +import lombok.Getter; + +@Getter +public enum ModelProviderEnum { + LOCAL("local", "本地"), + TENCENT("tencent", "腾讯云"); + + private final String code; + private final String description; + + ModelProviderEnum(String code, String description) { + this.code = code; + this.description = description; + } +} diff --git a/backend/src/main/java/com/imeeting/grpc/push/AndroidPushGrpcService.java b/backend/src/main/java/com/imeeting/grpc/push/AndroidPushGrpcService.java index 9102f0a..9451917 100644 --- a/backend/src/main/java/com/imeeting/grpc/push/AndroidPushGrpcService.java +++ b/backend/src/main/java/com/imeeting/grpc/push/AndroidPushGrpcService.java @@ -254,7 +254,20 @@ public class AndroidPushGrpcService extends PushServiceGrpc.PushServiceImplBase return switch (platform) { case IOS -> "ios"; case ANDROID -> "android"; - case PLATFORM_UNKNOWN, UNRECOGNIZED -> "android"; + case PLATFORM_UNKNOWN,UNRECOGNIZED -> "android"; + case HARMONY_MOBILE ->"harmony_mobile"; + + // Desktop + case WINDOWS ->"windows"; + case MACOS->"macos"; + case LINUX -> "linux"; + + // Linux发行版（可选） + case KYLIN ->"kylin"; + case UOS ->"uos"; + + // Harmony PC + case HARMONY_PC ->"harmony_pc"; }; } } diff --git a/backend/src/main/java/com/imeeting/service/android/AndroidChunkUploadService.java b/backend/src/main/java/com/imeeting/service/android/AndroidChunkUploadService.java index 6917bd9..807f7df 100644 --- a/backend/src/main/java/com/imeeting/service/android/AndroidChunkUploadService.java +++ b/backend/src/main/java/com/imeeting/service/android/AndroidChunkUploadService.java @@ -15,4 +15,12 @@ public interface AndroidChunkUploadService { LegacyUploadAudioResponse completeUpload(Long meetingId, Integer totalChunks, AndroidAuthContext authContext) throws IOException; + + /** + * 异步执行分片合并 + 音频上传 + 触发离线处理。 + * 不阻塞调用线程（Tomcat），错误通过 failOfflineTranscription 回写会议状态。 + */ + void completeUploadAsync(Long meetingId, + Integer totalChunks, + AndroidAuthContext authContext); } diff --git a/backend/src/main/java/com/imeeting/service/android/impl/AndroidChunkUploadServiceImpl.java b/backend/src/main/java/com/imeeting/service/android/impl/AndroidChunkUploadServiceImpl.java index 6e6de3f..ffe23b2 100644 --- a/backend/src/main/java/com/imeeting/service/android/impl/AndroidChunkUploadServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/android/impl/AndroidChunkUploadServiceImpl.java @@ -5,14 +5,16 @@ import com.imeeting.dto.android.AndroidChunkUploadSessionState; import com.imeeting.dto.android.legacy.LegacyUploadAudioResponse; import com.imeeting.service.android.AndroidChunkUploadService; import com.imeeting.service.android.legacy.LegacyMeetingAdapterService; +import com.imeeting.service.biz.MeetingCommandService; +import com.imeeting.service.biz.MeetingQueryService; +import com.imeeting.support.TaskSecurityContextRunner; import com.imeeting.support.redis.AndroidChunkUploadSessionCache; -import com.unisbase.security.LoginUser; -import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -30,12 +32,16 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; @Service -@RequiredArgsConstructor +@Slf4j public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService { - private static final Pattern CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$"); - + private static final Pattern LEGACY_CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$"); + private static final Pattern CHUNK_DIR_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)$"); + private static final String CHUNK_ROOT_DIR = "chunks"; + private final TaskSecurityContextRunner taskSecurityContextRunner; private final AndroidChunkUploadSessionCache sessionCache; private final LegacyMeetingAdapterService legacyMeetingAdapterService; + private final MeetingCommandService meetingCommandService; + private final java.util.concurrent.Executor chunkMergeExecutor; @Value("${unisbase.app.upload-path}") private String uploadPath; @@ -43,6 +49,18 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService @Value("${imeeting.audio.ffmpeg-path:ffmpeg}") private String ffmpegPath; + public AndroidChunkUploadServiceImpl(AndroidChunkUploadSessionCache sessionCache, + LegacyMeetingAdapterService legacyMeetingAdapterService, + MeetingCommandService meetingCommandService, + @Qualifier("chunkMergeExecutor") java.util.concurrent.Executor chunkMergeExecutor, + TaskSecurityContextRunner taskSecurityContextRunner) { + this.sessionCache = sessionCache; + this.legacyMeetingAdapterService = legacyMeetingAdapterService; + this.meetingCommandService = meetingCommandService; + this.chunkMergeExecutor = chunkMergeExecutor; + this.taskSecurityContextRunner = taskSecurityContextRunner; + } + @Override public void saveChunk(Long meetingId, Integer chunkIndex, @@ -54,7 +72,7 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService if (chunkIndex == null || chunkIndex < 0) { throw new RuntimeException("分片参数无效"); } - if (chunkFile == null) { + if (chunkFile == null) { throw new RuntimeException("chunk_file不能为空"); } @@ -63,19 +81,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService throw new RuntimeException("分片上传会话与当前设备或会议不匹配"); } - String chunkFileName = buildStoredChunkFileName(chunkIndex, chunkFile.getOriginalFilename()); - String previousFileName = state.getChunkFileNames().get(chunkIndex); - Path meetingDir = sessionDir(meetingId); - Files.createDirectories(meetingDir); + String originalFileName = resolveOriginalFileName(chunkFile.getOriginalFilename()); + Path chunkFilePath = resolveChunkFilePath(meetingId, chunkIndex, originalFileName); + Path chunkDir = chunkFilePath.getParent(); + clearChunkDirectory(chunkDir); + Files.createDirectories(chunkDir); - if (previousFileName != null && !previousFileName.equals(chunkFileName)) { - deleteQuietly(meetingDir.resolve(previousFileName)); + Files.write(chunkFilePath, chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + + String previousFileName = state.getChunkFileNames().put(chunkIndex, originalFileName); + if (previousFileName != null && !previousFileName.equals(originalFileName)) { state.getUploadedChunkFileNames().remove(previousFileName); } - - Files.write(meetingDir.resolve(chunkFileName), chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - state.getUploadedChunkFileNames().add(chunkFileName); - state.getChunkFileNames().put(chunkIndex, chunkFileName); + state.getUploadedChunkFileNames().add(originalFileName); state.getReceivedChunks().add(chunkIndex); saveState(meetingId, state); } @@ -91,53 +109,70 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService throw new RuntimeException("total_chunks不能为空且必须大于0"); } - AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext); + AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext); if (!Objects.equals(state.getMeetingId(), meetingId) || !Objects.equals(state.getDeviceId(), authContext.getDeviceId())) { throw new RuntimeException("分片上传会话与当前设备或会议不匹配"); } - Path meetingDir = sessionDir(meetingId); - Files.createDirectories(meetingDir); + Path meetingDir = sessionDir(meetingId); + Files.createDirectories(meetingDir); state.setTotalChunks(totalChunks); - List orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks); + List orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks); saveState(meetingId, state); - Path mergedFile = mergeChunks(state, orderedChunkPaths); + Path mergedFile = mergeChunks(state, orderedChunkPaths); + if (mergedFile == null) { + meetingCommandService.failOfflineTranscription(meetingId, "安卓上传文件为空"); + cleanup(meetingId); + return new LegacyUploadAudioResponse(meetingId, null, "无可合并音频"); + } + MultipartFile mergedMultipart = new LocalMultipartFile( - buildMergedOriginalFilename(state, mergedFile), + resolveMergedOriginalFilename(state, orderedChunkPaths, mergedFile), state.getContentType(), - Files.readAllBytes(mergedFile) + mergedFile ); - LegacyUploadAudioResponse response; - if (authContext.isAnonymous()) { - response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice( - meetingId, - null, - null, - false, - mergedMultipart, - authContext - ); - } else { - LoginUser loginUser = toLoginUser(authContext); - response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcess( - meetingId, - null, - null, - false, - mergedMultipart, - authContext, - loginUser - ); - } + LegacyUploadAudioResponse response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice( + meetingId, + null, + null, + false, + mergedMultipart, + authContext + ); if (response != null) { cleanup(meetingId); } return response; } + @Override + public void completeUploadAsync(Long meetingId, + Integer totalChunks, + AndroidAuthContext authContext) { + if (meetingId == null) { + throw new RuntimeException("meeting_id不能为空"); + } + if (totalChunks == null || totalChunks <= 0) { + throw new RuntimeException("total_chunks不能为空且必须大于0"); + } + + chunkMergeExecutor.execute( ()->taskSecurityContextRunner.runAsTenantUser( authContext.getTenantId(), authContext.getUserId(), () -> { + try { + completeUpload(meetingId, totalChunks, authContext); + } catch (Exception ex) { + log.error("[分片合并] 会议{}异步合并上传失败: {}", meetingId, ex.getMessage(), ex); + try { + meetingCommandService.failOfflineTranscription(meetingId, "音频合并上传失败: " + ex.getMessage()); + } catch (Exception inner) { + log.error("[分片合并] 会议{}标记失败状态异常: {}", meetingId, inner.getMessage(), inner); + } + } + })); + } + private AndroidChunkUploadSessionState getOrCreateState(Long meetingId, MultipartFile chunkFile, AndroidAuthContext authContext) { @@ -148,155 +183,170 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService AndroidChunkUploadSessionState state = new AndroidChunkUploadSessionState(); state.setMeetingId(meetingId); state.setDeviceId(authContext.getDeviceId()); - state.setFileName(normalizeChunkSourceFileName(chunkFile.getOriginalFilename())); + state.setFileName(resolveOriginalFileName(chunkFile.getOriginalFilename())); state.setContentType(chunkFile.getContentType()); saveState(meetingId, state); return state; } - private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) { - AndroidChunkUploadSessionState state = getState(meetingId); - if (state != null) { - return state; - } - AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState(); - rebuiltState.setMeetingId(meetingId); - rebuiltState.setDeviceId(authContext.getDeviceId()); - return rebuiltState; - } - - private List rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state, - Path meetingDir, - int totalChunks) throws IOException { - Map validatedChunkFiles = scanChunkFiles(meetingDir, true); - Map mergeChunkFiles = scanChunkFiles(meetingDir, false); - state.getReceivedChunks().clear(); - state.getUploadedChunkFileNames().clear(); - state.getChunkFileNames().clear(); - - for (Map.Entry entry : validatedChunkFiles.entrySet()) { - Integer chunkIndex = entry.getKey(); - Path stateChunkPath = mergeChunkFiles.getOrDefault(chunkIndex, entry.getValue()); - String fileName = stateChunkPath.getFileName().toString(); - state.getReceivedChunks().add(chunkIndex); - state.getUploadedChunkFileNames().add(fileName); - state.getChunkFileNames().put(chunkIndex, fileName); + private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) { + AndroidChunkUploadSessionState state = getState(meetingId); + if (state != null) { + return state; + } + AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState(); + rebuiltState.setMeetingId(meetingId); + rebuiltState.setDeviceId(authContext.getDeviceId()); + return rebuiltState; } - List orderedChunkPaths = new ArrayList<>(totalChunks); - for (int i = 0; i < totalChunks; i++) { - Path validatedChunkPath = validatedChunkFiles.get(i); - if (validatedChunkPath == null) { - throw new RuntimeException("分片未上传完整"); - } - Path chunkPath = mergeChunkFiles.get(i); - if (chunkPath != null) { - orderedChunkPaths.add(chunkPath); - } - } - return orderedChunkPaths; - } + private List rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state, + Path meetingDir, + int totalChunks) throws IOException { + Map chunkFiles = scanChunkFiles(meetingDir, true); + state.getReceivedChunks().clear(); + state.getUploadedChunkFileNames().clear(); + state.getChunkFileNames().clear(); - private Map scanChunkFiles(Path meetingDir, boolean includePending) throws IOException { - Map chunkFiles = new TreeMap<>(); - if (!Files.exists(meetingDir)) { - return chunkFiles; - } - try (var paths = Files.list(meetingDir)) { - paths.filter(Files::isRegularFile) - .filter(path -> isUsableChunkFile(path, includePending)) - .forEach(path -> { - Integer chunkIndex = parseChunkIndex(path.getFileName().toString(), includePending); - if (chunkIndex == null) { - return; - } - Path existing = chunkFiles.get(chunkIndex); - if (existing == null || isPreferredChunkFile(path, existing)) { - chunkFiles.put(chunkIndex, path); - } - }); - } - return chunkFiles; - } + for (Map.Entry entry : chunkFiles.entrySet()) { + Integer chunkIndex = entry.getKey(); + Path chunkPath = entry.getValue(); + String fileName = chunkPath.getFileName().toString(); + state.getReceivedChunks().add(chunkIndex); + state.getUploadedChunkFileNames().add(fileName); + state.getChunkFileNames().put(chunkIndex, fileName); + } - private boolean isUsableChunkFile(Path path, boolean includePending) { - if (path == null || path.getFileName() == null) { - return false; + List orderedChunkPaths = new ArrayList<>(totalChunks); + for (int i = 0; i < totalChunks; i++) { + Path chunkPath = chunkFiles.get(i); + if (chunkPath == null) { + throw new RuntimeException("分片未上传完整"); + } + orderedChunkPaths.add(chunkPath); + } + return orderedChunkPaths; } - String fileName = normalizeChunkStorageFileName(path.getFileName().toString(), includePending); - if (fileName == null) { - return false; - } - return CHUNK_FILE_NAME_PATTERN.matcher(fileName).matches(); - } - private Integer parseChunkIndex(String fileName, boolean includePending) { - String normalizedFileName = normalizeChunkStorageFileName(fileName, includePending); - if (normalizedFileName == null) { - return null; - } - Matcher matcher = CHUNK_FILE_NAME_PATTERN.matcher(normalizedFileName); - if (!matcher.matches()) { - return null; - } - return Integer.parseInt(matcher.group(1)); - } + private Map scanChunkFiles(Path meetingDir, boolean includePending) throws IOException { + Map chunkFiles = new TreeMap<>(); + if (!Files.exists(meetingDir)) { + return chunkFiles; + } - private boolean isPreferredChunkFile(Path candidate, Path existing) { - boolean candidatePending = isPendingChunkFile(candidate); - boolean existingPending = isPendingChunkFile(existing); - if (candidatePending != existingPending) { - return !candidatePending; - } - try { - return Files.getLastModifiedTime(candidate).compareTo(Files.getLastModifiedTime(existing)) >= 0; - } catch (IOException ex) { - return candidate.getFileName().toString().compareTo(existing.getFileName().toString()) >= 0; - } - } + Path chunkRoot = meetingDir.resolve(CHUNK_ROOT_DIR); + if (Files.exists(chunkRoot)) { + try (var dirs = Files.list(chunkRoot)) { + dirs.filter(Files::isDirectory).forEach(dir -> { + Integer chunkIndex = parseChunkDirIndex(dir.getFileName().toString()); + if (chunkIndex == null || chunkFiles.containsKey(chunkIndex)) { + return; + } + Path chunkPath = pickChunkFile(dir, includePending); + if (chunkPath != null) { + chunkFiles.put(chunkIndex, chunkPath); + } + }); + } + if (!chunkFiles.isEmpty()) { + return chunkFiles; + } + } - private String normalizeChunkStorageFileName(String fileName, boolean includePending) { - if (fileName == null) { - return null; + try (var paths = Files.list(meetingDir)) { + paths.filter(Files::isRegularFile) + .forEach(path -> { + Integer chunkIndex = parseLegacyChunkIndex(path.getFileName().toString()); + if (chunkIndex != null) { + chunkFiles.put(chunkIndex, path); + } + }); + } + return chunkFiles; } - if (fileName.endsWith(".pending")) { - return includePending ? fileName.substring(0, fileName.length() - ".pending".length()) : null; + + private Integer parseChunkDirIndex(String directoryName) { + if (directoryName == null) { + return null; + } + Matcher matcher = CHUNK_DIR_NAME_PATTERN.matcher(directoryName); + if (!matcher.matches()) { + return null; + } + return Integer.parseInt(matcher.group(1)); } - return fileName; - } - private boolean isPendingChunkFile(Path path) { - return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending"); - } + private Integer parseLegacyChunkIndex(String fileName) { + if (fileName == null) { + return null; + } + Matcher matcher = LEGACY_CHUNK_FILE_NAME_PATTERN.matcher(fileName); + if (!matcher.matches()) { + return null; + } + return Integer.parseInt(matcher.group(1)); + } - private Path mergeChunks(AndroidChunkUploadSessionState state, List chunkPaths) throws IOException { + private Path pickChunkFile(Path chunkDir, boolean includePending) { + if (chunkDir == null || !Files.isDirectory(chunkDir)) { + return null; + } + try (var files = Files.list(chunkDir)) { + Path preferred = files + .filter(Files::isRegularFile) + .filter(path -> includePending || !isPendingChunkFile(path)) + .findFirst() + .orElse(null); + if (preferred != null) { + return preferred; + } + } catch (IOException ex) { + return null; + } + try (var files = Files.list(chunkDir)) { + return files.filter(Files::isRegularFile).findFirst().orElse(null); + } catch (IOException ex) { + return null; + } + } + + private Path mergeChunks(AndroidChunkUploadSessionState state, List chunkPaths) throws IOException { + List mergeableChunkPaths = filterMergeableChunkPaths(chunkPaths); Path meetingDir = sessionDir(state.getMeetingId()); - String mergedExtension = resolveMergedExtension(state, chunkPaths); + String mergedExtension = resolveMergedExtension(state, mergeableChunkPaths); Path mergedOutput = meetingDir.resolve("merged" + mergedExtension); Path concatList = meetingDir.resolve("concat-inputs.txt"); Files.deleteIfExists(mergedOutput); - if (chunkPaths.isEmpty() || allChunkFilesEmpty(chunkPaths)) { - Files.write(mergedOutput, new byte[0], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - return mergedOutput; - } + if (mergeableChunkPaths.isEmpty() || allChunkFilesEmpty(mergeableChunkPaths)) { + return null; + } - if (chunkPaths.size() == 1) { - return chunkPaths.get(0); - } + if (mergeableChunkPaths.size() == 1) { + return mergeableChunkPaths.get(0); + } - writeConcatListFile(concatList, chunkPaths); + writeConcatListFile(concatList, mergeableChunkPaths); executeFfmpegConcat(concatList, mergedOutput); return mergedOutput; } - private boolean allChunkFilesEmpty(List chunkPaths) throws IOException { - for (Path chunkPath : chunkPaths) { - if (Files.size(chunkPath) > 0) { - return false; - } + private boolean allChunkFilesEmpty(List chunkPaths) throws IOException { + for (Path chunkPath : chunkPaths) { + if (Files.size(chunkPath) > 0) { + return false; + } + } + return true; } - return true; + + private List filterMergeableChunkPaths(List chunkPaths) { + if (chunkPaths == null || chunkPaths.isEmpty()) { + return List.of(); + } + return chunkPaths.stream() + .filter(path -> path != null && !isPendingChunkFile(path)) + .toList(); } private AndroidChunkUploadSessionState getState(Long meetingId) { @@ -329,22 +379,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService return Paths.get(normalizedBasePath, "android-chunks", String.valueOf(meetingId)); } - private String buildStoredChunkFileName(Integer chunkIndex, String originalFileName) { - String normalizedSourceName = normalizeChunkSourceFileName(originalFileName); - int extensionIndex = normalizedSourceName.lastIndexOf('.'); - String extension = extensionIndex >= 0 ? normalizedSourceName.substring(extensionIndex) : ""; - String safeExtension = extension.isBlank() ? ".bin" : extension; - return "chunk-" + (chunkIndex == null ? "unknown" : chunkIndex) + safeExtension; + private Path resolveChunkFilePath(Long meetingId, Integer chunkIndex, String originalFileName) { + return sessionDir(meetingId) + .resolve(CHUNK_ROOT_DIR) + .resolve("chunk-" + chunkIndex) + .resolve(originalFileName); } - private String buildMergedOriginalFilename(AndroidChunkUploadSessionState state, Path mergedFile) { - if (state.getFileName() != null && !state.getFileName().isBlank()) { - String normalizedSourceName = normalizeChunkSourceFileName(state.getFileName()); - int extensionIndex = normalizedSourceName.lastIndexOf('.'); - if (extensionIndex >= 0) { - return "merged" + normalizedSourceName.substring(extensionIndex); - } - return normalizedSourceName; + private String resolveMergedOriginalFilename(AndroidChunkUploadSessionState state, List chunkPaths, Path mergedFile) { + if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) { + return state.getFileName(); + } + if (chunkPaths != null && !chunkPaths.isEmpty() && chunkPaths.get(0).getFileName() != null) { + return chunkPaths.get(0).getFileName().toString(); } return mergedFile == null || mergedFile.getFileName() == null ? "meeting-audio.bin" : mergedFile.getFileName().toString(); } @@ -353,41 +400,56 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService if (chunkPath == null || chunkPath.getFileName() == null) { return ".bin"; } - String fileName = normalizeChunkSourceFileName(chunkPath.getFileName().toString()); + String fileName = chunkPath.getFileName().toString(); int extensionIndex = fileName.lastIndexOf('.'); return extensionIndex >= 0 ? fileName.substring(extensionIndex) : ".bin"; } - private String resolveMergedExtension(AndroidChunkUploadSessionState state, List chunkPaths) { - if (chunkPaths != null && !chunkPaths.isEmpty()) { - return resolveMergedExtension(chunkPaths.get(0)); - } - if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) { - int extensionIndex = state.getFileName().lastIndexOf('.'); - if (extensionIndex >= 0) { - return state.getFileName().substring(extensionIndex); - } - } - return ".bin"; + private String resolveMergedExtension(AndroidChunkUploadSessionState state, List chunkPaths) { + if (chunkPaths != null && !chunkPaths.isEmpty()) { + return resolveMergedExtension(chunkPaths.get(0)); + } + if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) { + int extensionIndex = state.getFileName().lastIndexOf('.'); + if (extensionIndex >= 0) { + return state.getFileName().substring(extensionIndex); + } + } + return ".bin"; } - private String normalizeChunkSourceFileName(String fileName) { - if (fileName == null) { - return ""; + private boolean isPendingChunkFile(Path path) { + return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending"); + } + + private String resolveOriginalFileName(String originalFileName) { + if (originalFileName == null || originalFileName.trim().isEmpty()) { + throw new RuntimeException("chunk_file原始文件名不能为空"); } - String normalized = Paths.get(fileName.trim()).getFileName().toString(); - if (normalized.endsWith(".pending")) { - return normalized.substring(0, normalized.length() - ".pending".length()); + if (originalFileName.contains("/") || originalFileName.contains("\\")) { + throw new RuntimeException("chunk_file文件名不合法"); + } + return originalFileName; + } + + private void clearChunkDirectory(Path chunkDir) throws IOException { + if (chunkDir == null || !Files.exists(chunkDir)) { + return; + } + try (var paths = Files.walk(chunkDir)) { + paths.sorted((left, right) -> right.compareTo(left)).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); } - return normalized; } private void writeConcatListFile(Path concatList, List chunkPaths) throws IOException { List lines = new ArrayList<>(chunkPaths.size()); for (Path chunkPath : chunkPaths) { - if (chunkPath.getFileName() != null && chunkPath.getFileName().toString().endsWith(".pending")) { - continue; - } String normalizedPath = chunkPath.toAbsolutePath().toString().replace("'", "'\\''"); lines.add("file '" + normalizedPath + "'"); } @@ -424,46 +486,20 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService if (process.exitValue() != 0) { throw new IOException("音频分片合并失败: " + new String(output, StandardCharsets.UTF_8)); } - if (!Files.exists(mergedOutput)) { + if (!Files.exists(mergedOutput)) { throw new IOException("音频分片合并结果为空"); } } - private void deleteQuietly(Path path) { - if (path == null) { - return; - } - try { - Files.deleteIfExists(path); - } catch (IOException ignored) { - } - } - - private LoginUser toLoginUser(AndroidAuthContext authContext) { - if (authContext == null || authContext.isAnonymous() || authContext.getUserId() == null || authContext.getTenantId() == null) { - throw new RuntimeException("安卓用户未登录或认证无效"); - } - LoginUser loginUser = new LoginUser( - authContext.getUserId(), - authContext.getTenantId(), - authContext.getUsername(), - authContext.getPlatformAdmin(), - authContext.getTenantAdmin(), - authContext.getPermissions() - ); - loginUser.setDisplayName(authContext.getDisplayName()); - return loginUser; - } - private static final class LocalMultipartFile implements MultipartFile { private final String originalFilename; private final String contentType; - private final byte[] bytes; + private final Path filePath; - private LocalMultipartFile(String originalFilename, String contentType, byte[] bytes) { + private LocalMultipartFile(String originalFilename, String contentType, Path filePath) { this.originalFilename = originalFilename; this.contentType = contentType; - this.bytes = bytes == null ? new byte[0] : bytes; + this.filePath = filePath; } @Override @@ -483,27 +519,35 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService @Override public boolean isEmpty() { - return bytes.length == 0; + try { + return Files.size(filePath) == 0; + } catch (IOException ex) { + return true; + } } @Override public long getSize() { - return bytes.length; + try { + return Files.size(filePath); + } catch (IOException ex) { + return 0; + } } @Override - public byte[] getBytes() { - return bytes; + public byte[] getBytes() throws IOException { + return Files.readAllBytes(filePath); } @Override - public InputStream getInputStream() { - return new ByteArrayInputStream(bytes); + public InputStream getInputStream() throws IOException { + return Files.newInputStream(filePath); } @Override public void transferTo(java.io.File dest) throws IOException, IllegalStateException { - Files.write(dest.toPath(), bytes); + Files.copy(filePath, dest.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); } } } diff --git a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java index 40779e9..9d715d2 100644 --- a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @Service @@ -289,6 +290,7 @@ public class LegacyMeetingAdapterServiceImpl implements LegacyMeetingAdapterServ String relocatedUrl = meetingDomainSupport.relocateAudioUrl(meetingId, stagingUrl); taskSecurityContextRunner.runAsTenantUser(meeting.getTenantId(), meeting.getCreatorId(), () -> { meetingDomainSupport.applyMeetingAudioMetadata(meeting, relocatedUrl); + meetingDomainSupport.prewarmPlaybackAudioAfterCommit(relocatedUrl); meeting.setSummaryModelId(profile.getResolvedSummaryModelId()); meeting.setPromptId(profile.getResolvedPromptId()); meeting.setAudioSaveStatus(RealtimeMeetingAudioStorageService.STATUS_SUCCESS); diff --git a/backend/src/main/java/com/imeeting/service/biz/MeetingCommandService.java b/backend/src/main/java/com/imeeting/service/biz/MeetingCommandService.java index 124b3c3..f781e87 100644 --- a/backend/src/main/java/com/imeeting/service/biz/MeetingCommandService.java +++ b/backend/src/main/java/com/imeeting/service/biz/MeetingCommandService.java @@ -28,14 +28,14 @@ public interface MeetingCommandService { void deleteMeeting(Long id); - void appendRealtimeTranscripts(Long meetingId, List items); - void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult); void completeRealtimeMeeting(Long meetingId, String audioUrl, boolean overwriteAudio); void finishOfflineMeeting(Long meetingId, String finishStage); + void failOfflineTranscription(Long meetingId, String failureMessage); + void updateSpeakerInfo(Long meetingId, String speakerId, String newName, String label); void updateMeetingTranscript(UpdateMeetingTranscriptCommand command); diff --git a/backend/src/main/java/com/imeeting/service/biz/MeetingQueryService.java b/backend/src/main/java/com/imeeting/service/biz/MeetingQueryService.java index 8d494c7..f883d44 100644 --- a/backend/src/main/java/com/imeeting/service/biz/MeetingQueryService.java +++ b/backend/src/main/java/com/imeeting/service/biz/MeetingQueryService.java @@ -16,7 +16,10 @@ public interface MeetingQueryService { MeetingVO getDetail(Long id); - MeetingVO getDetailIgnoreTenant(Long id); + default MeetingVO getDetailIgnoreTenant(Long id){ + return getDetailIgnoreTenant(id,true); + }; + MeetingVO getDetailIgnoreTenant(Long id,Boolean includeAudio); List getTranscripts(Long meetingId); diff --git a/backend/src/main/java/com/imeeting/service/biz/RealtimeMeetingSessionStateService.java b/backend/src/main/java/com/imeeting/service/biz/RealtimeMeetingSessionStateService.java index 7f4c273..77f74bb 100644 --- a/backend/src/main/java/com/imeeting/service/biz/RealtimeMeetingSessionStateService.java +++ b/backend/src/main/java/com/imeeting/service/biz/RealtimeMeetingSessionStateService.java @@ -11,6 +11,8 @@ public interface RealtimeMeetingSessionStateService { void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig); + void rememberSpeakerContext(Long meetingId, String speakerContextId); + void assertCanOpenSession(Long meetingId); boolean activate(Long meetingId, String connectionId); @@ -25,6 +27,8 @@ public interface RealtimeMeetingSessionStateService { void refreshAfterTranscript(Long meetingId); + void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount); + boolean markCompletingIfResumeExpired(Long meetingId); void expireEmptySession(Long meetingId); diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/AiModelServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/AiModelServiceImpl.java index 1f552a2..36e31a6 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/AiModelServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/AiModelServiceImpl.java @@ -11,6 +11,7 @@ import com.imeeting.dto.biz.AiLocalProfileVO; import com.imeeting.dto.biz.AiModelVO; import com.imeeting.entity.biz.AsrModel; import com.imeeting.entity.biz.LlmModel; +import com.imeeting.enums.ModelProviderEnum; import com.imeeting.mapper.biz.AsrModelMapper; import com.imeeting.mapper.biz.LlmModelMapper; import com.imeeting.service.biz.AiModelService; @@ -44,6 +45,10 @@ public class AiModelServiceImpl implements AiModelService { private static final String TYPE_ASR = "ASR"; private static final String TYPE_LLM = "LLM"; + private static final String TENCENT_PROVIDER = "tencent"; + private static final String MEDIA_TENCENT_APP_ID = "tencentAppId"; + private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId"; + private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey"; private static final int DEFAULT_SORT_ORDER = 0; private static final String DEFAULT_LLM_API_PATH = "/v1/chat/completions"; private static final String DEFAULT_ANTHROPIC_API_PATH = "/messages"; @@ -169,7 +174,7 @@ public class AiModelServiceImpl implements AiModelService { if (resolvedBaseUrl == null || resolvedBaseUrl.isBlank()) { return Collections.emptyList(); } - if ("custom".equals(providerKey)) { + if (ModelProviderEnum.LOCAL.getCode().equals(providerKey)) { return fetchLocalProfile(resolvedBaseUrl, apiKey).getAsrModels(); } String targetUrl = resolveModelListUrl(providerKey, resolvedBaseUrl, apiKey); @@ -464,7 +469,7 @@ public class AiModelServiceImpl implements AiModelService { } private String resolveModelListUrl(String providerKey, String baseUrl, String apiKey) { - if ("Custom".equalsIgnoreCase(providerKey)) { + if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(providerKey)) { return baseUrl+"/api/asrconfig"; } if ("gemini".equals(providerKey) || "google".equals(providerKey)) { @@ -740,6 +745,7 @@ public class AiModelServiceImpl implements AiModelService { if (Integer.valueOf(1).equals(dto.getIsDefault()) && !Integer.valueOf(1).equals(dto.getStatus())) { throw new RuntimeException("默认模型必须为启用状态"); } + validateTencentAsrConfig(dto); // if ("custom".equals(normalizeProvider(dto.getProvider()))) { // if (TYPE_ASR.equals(normalizeType(dto.getModelType()))) { // Map mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig(); @@ -832,13 +838,20 @@ public class AiModelServiceImpl implements AiModelService { } private void pushAsrConfig(AsrModel entity) { - if ("custom".equals(normalizeProvider(entity.getProvider()))) { + String provider = normalizeProvider(entity.getProvider()); + if (ModelProviderEnum.LOCAL.getCode().equals(provider)) { if (entity.getApiKey() == null || entity.getApiKey().isBlank()) { log.info("Skip syncing local ASR profile because apiKey is blank, modelName={}", entity.getModelName()); return; } updateLocalProfile(entity); return; + } + if ("custom".equals(provider)) { + return; + } + if (TENCENT_PROVIDER.equals(provider)) { + return; } if (entity.getBaseUrl() == null || entity.getBaseUrl().isBlank()) { throw new RuntimeException("ASR 模型必须配置 baseUrl"); @@ -934,6 +947,28 @@ public class AiModelServiceImpl implements AiModelService { return text.isEmpty() ? null : text; } + private void validateTencentAsrConfig(AiModelDTO dto) { + if (!TYPE_ASR.equals(normalizeType(dto.getModelType()))) { + return; + } + if (!TENCENT_PROVIDER.equals(normalizeProvider(dto.getProvider()))) { + return; + } + Map mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig(); + if (readConfigString(mediaConfig.get(MEDIA_TENCENT_APP_ID)) == null) { + throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentAppId"); + } + if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_ID)) == null) { + throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretId"); + } + if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_KEY)) == null) { + throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey"); + } + if (dto.getModelCode() == null || dto.getModelCode().isBlank()) { + throw new RuntimeException("腾讯实时 ASR 模型必须配置 modelCode"); + } + } + private BigDecimal readConfigDecimal(Object value) { if (value == null) { return null; diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/AiTaskServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/AiTaskServiceImpl.java index 3681ece..d61460f 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/AiTaskServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/AiTaskServiceImpl.java @@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.imeeting.common.SysParamKeys; import com.imeeting.common.MeetingProgressStage; import com.imeeting.dto.biz.AiModelVO; import com.imeeting.dto.biz.MeetingSummarySource; @@ -66,6 +67,8 @@ public class AiTaskServiceImpl extends ServiceImpl impleme private static final Duration ASR_SUBMIT_REQUEST_TIMEOUT = Duration.ofSeconds(30); private static final Duration ASR_QUERY_REQUEST_TIMEOUT = Duration.ofSeconds(30); + private static final String DISPATCH_MODE_PARALLEL = "PARALLEL"; + private static final String DISPATCH_MODE_SERIAL = "SERIAL"; private final MeetingMapper meetingMapper; private final MeetingTranscriptMapper transcriptMapper; @@ -278,6 +281,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme failPendingSummaryTask(sumTask, "没有可用于总结的转录内容"); updateMeetingStatus(meetingId, 4); updateProgress(meetingId, -1, "未识别到可用于总结的转录内容", 0); + androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.FAILED_TRANSCRIBING.getCode()); return; } if (!asrText.isBlank()) { @@ -287,8 +291,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme sumTask == null ? null : sumTask.getId()); meetingProgressService.markStage(meetingId, asrTask, 1, MeetingProgressStage.ASR_COMPLETED, 80, "转写完成，准备生成总结", 0); scheduleQueuedAsrTasks(); - self.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); - self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + dispatchPostAsrTasks(meeting, chapterTask, sumTask); return; } if (sumTask != null && canExecuteTask(sumTask)) { @@ -342,8 +345,20 @@ public class AiTaskServiceImpl extends ServiceImpl impleme return; } executeChapterFlow(meeting, chapterTask); + if (shouldRunSummaryAfterChapter(meeting, chapterTask)) { + AiTask summaryTask = findLatestTask(meetingId, "SUMMARY"); + if (summaryTask != null && canExecuteTask(summaryTask)) { + self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + return; + } + } reconcileMeetingStatus(meetingId); - androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.COMPLETED.getCode()); + androidMeetingPushService.pushMeetingStatusChanged( + meetingId, + MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED) + ? UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode() + : UnifiedMeetingStatusStage.COMPLETED.getCode() + ); log.info("[CHAPTER-FLOW] 章节任务流程结束: meetingId={}, chapterTaskId={}, costMs={}", meetingId, chapterTask.getId(), System.currentTimeMillis() - startMillis); } @@ -1106,7 +1121,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme meetingPointsService.recordSummarySuccessCharge(meeting, taskRecord); AiTask latestChapterTask = findLatestTask(meeting.getId(), "CHAPTER"); - if (latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus())) { + if (!resolveAiCatalogEnabled() ||(latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus()))) { updateProgress(meeting.getId(), 100, "全流程分析完成", 0); } else { updateProgress(meeting.getId(), 95, "总结生成完成，等待 AI 目录完成...", 0); @@ -1177,7 +1192,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme log.info("[SUMMARY-EXEC] 已获取总结锁，开始构建总结来源: meetingId={}, sumTaskId={}", meeting.getId(), sumTask == null ? null : sumTask.getId()); try { - MeetingSummarySource summarySource = buildRawTranscriptSummarySource(meeting); + MeetingSummarySource summarySource = buildSummarySourceForExecution(meeting, sumTask); if (summarySource.getText() == null || summarySource.getText().isBlank()) { log.warn("[SUMMARY-EXEC] 无转录内容，无法生成总结: meetingId={}, sumTaskId={}", meeting.getId(), sumTask == null ? null : sumTask.getId()); @@ -1207,6 +1222,85 @@ public class AiTaskServiceImpl extends ServiceImpl impleme .build(); } + private MeetingSummarySource buildSummarySourceForExecution(Meeting meeting, AiTask sumTask) { + if (shouldUseChapterBackedSummarySource()) { + return meetingTranscriptChapterService.resolveSummarySource(meeting, sumTask); + } + return buildRawTranscriptSummarySource(meeting); + } + + private void dispatchPostAsrTasks(Meeting meeting, AiTask chapterTask, AiTask summaryTask) { + if (meeting == null) { + return; + } + if (!resolveAiCatalogEnabled()) { + if (summaryTask != null && canExecuteTask(summaryTask)) { + self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId()); + } + return; + } + if (isSerialDispatchMode()) { + if (chapterTask != null && canExecuteTask(chapterTask)) { + self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId()); + } else { + log.warn("[ASR-FLOW] 串行模式下缺少可执行章节任务，跳过总结派发: meetingId={}, chapterTaskId={}, chapterStatus={}", + meeting.getId(), + chapterTask == null ? null : chapterTask.getId(), + chapterTask == null ? null : chapterTask.getStatus()); + } + return; + } + if (chapterTask != null && canExecuteTask(chapterTask)) { + self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId()); + } + if (summaryTask != null && canExecuteTask(summaryTask)) { + self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId()); + } + } + + private boolean shouldRunSummaryAfterChapter(Meeting meeting, AiTask chapterTask) { + return meeting != null + && resolveAiCatalogEnabled() + && isSerialDispatchMode() + && isTaskCompleted(chapterTask) + && !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED); + } + + private boolean shouldUseChapterBackedSummarySource() { + return resolveAiCatalogEnabled() && isSerialDispatchMode(); + } + + private boolean resolveAiCatalogEnabled() { + if (sysParamService == null) { + return false; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false"); + if (rawValue == null || rawValue.isBlank()) { + return false; + } + String normalized = rawValue.trim().toLowerCase(); + return "1".equals(normalized) + || "true".equals(normalized) + || "yes".equals(normalized) + || "on".equals(normalized); + } + + private boolean isSerialDispatchMode() { + return DISPATCH_MODE_SERIAL.equals(resolveSummaryDispatchMode()); + } + + private String resolveSummaryDispatchMode() { + if (sysParamService == null) { + return DISPATCH_MODE_PARALLEL; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, DISPATCH_MODE_PARALLEL); + if (rawValue == null || rawValue.isBlank()) { + return DISPATCH_MODE_PARALLEL; + } + String normalized = rawValue.trim().toUpperCase(Locale.ROOT); + return DISPATCH_MODE_SERIAL.equals(normalized) ? DISPATCH_MODE_SERIAL : DISPATCH_MODE_PARALLEL; + } + private AiTask findLatestTask(Long meetingId, String taskType) { return this.getOne(new LambdaQueryWrapper() .eq(AiTask::getMeetingId, meetingId) @@ -1275,7 +1369,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme updateMeetingStatus(meetingId, 4); return; } - if (isTaskCompleted(chapterTask) && isTaskCompleted(summaryTask)) { + if ( isTaskCompleted(summaryTask) && (!resolveAiCatalogEnabled() || isTaskCompleted(chapterTask))) { updateMeetingStatus(meetingId, 3); return; } @@ -1488,6 +1582,7 @@ public class AiTaskServiceImpl extends ServiceImpl impleme if ("SUMMARY".equals(task.getTaskType())) { meetingPointsService.markSummaryChargeFailed(task.getId(), error); } + androidMeetingPushService.pushMeetingStatusChanged(task.getMeetingId(), UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode()); } private String buildAsrFailureMessage(Exception ex) { diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java index 5b04fda..5179b68 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java @@ -62,6 +62,7 @@ public class MeetingAudioUploadSupport { Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); } validateStoredAudio(targetPath, extension); + } catch (Exception ex) { Files.deleteIfExists(targetPath); throw ex; @@ -69,6 +70,55 @@ public class MeetingAudioUploadSupport { return buildStagingAudioToken(storedFileName); } + public String storeUploadedAudioFromPath(Path sourceFile, String originalFilename) throws IOException { + if (sourceFile == null || !Files.exists(sourceFile)) { + throw new RuntimeException("音频文件不能为空"); + } + + long fileSize = Files.size(sourceFile); + long maxUploadSizeMb = resolveMaxUploadSizeMb(); + long maxUploadSizeBytes = maxUploadSizeMb * 1024 * 1024; + if (fileSize > maxUploadSizeBytes) { + throw new RuntimeException("音频文件大小不能超过 " + maxUploadSizeMb + "MB"); + } + + String extension = resolveExtension(originalFilename); + validateFileHeaderFromPath(sourceFile, extension); + + Path stagingDir = resolveStagingAudioDirectory(uploadPath); + Files.createDirectories(stagingDir); + + String storedFileName = UUID.randomUUID() + "." + extension; + Path targetPath = stagingDir.resolve(storedFileName); + try { + Files.move(sourceFile, targetPath, StandardCopyOption.REPLACE_EXISTING); + validateStoredAudio(targetPath, extension); + } catch (Exception ex) { + Files.deleteIfExists(targetPath); + throw ex; + } + return buildStagingAudioToken(storedFileName); + } + + private void validateFileHeaderFromPath(Path sourceFile, String extension) throws IOException { + if (Files.size(sourceFile) <= 0) { + return; + } + byte[] header; + try (InputStream inputStream = Files.newInputStream(sourceFile)) { + header = inputStream.readNBytes(HEADER_SIZE); + } + boolean valid = switch (extension) { + case "wav" -> isWav(header); + case "mp3" -> isMp3(header); + case "m4a" -> isM4a(header); + default -> false; + }; + if (!valid) { + throw new RuntimeException("上传文件内容与音频格式不匹配，仅支持 mp3、wav、m4a"); + } + } + public static boolean isStagingAudioToken(String audioUrl) { return StringUtils.hasText(audioUrl) && audioUrl.startsWith(STAGING_AUDIO_TOKEN_PREFIX); } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java index 7aa55c0..17a61aa 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java @@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.fasterxml.jackson.databind.ObjectMapper; import com.imeeting.common.MeetingConstants; import com.imeeting.common.RedisKeys; +import com.imeeting.common.SysParamKeys; import com.imeeting.dto.android.AndroidPendingMeetingDraft; import com.imeeting.dto.biz.CreateMeetingCommand; import com.imeeting.dto.biz.CreateRealtimeMeetingCommand; @@ -27,6 +28,7 @@ import com.imeeting.entity.biz.HotWord; import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.MeetingTranscript; import com.imeeting.entity.biz.MeetingTranscriptChapterVersion; +import com.imeeting.enums.BusinessErrorCodeEnum; import com.imeeting.enums.MeetingStatusEnum; import com.imeeting.service.android.AndroidPendingMeetingDraftService; import com.imeeting.service.android.AndroidPushMessageService; @@ -45,6 +47,10 @@ import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; import com.imeeting.support.redis.MeetingAsrPermitCache; import com.imeeting.support.redis.MeetingLockCache; +import com.unisbase.common.exception.BusinessException; +import com.unisbase.common.exception.ErrorCodeEnum; +import com.imeeting.websocket.RealtimeMeetingProxyWebSocketHandler; +import com.unisbase.service.SysParamService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -77,7 +83,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { private final MeetingDomainSupport meetingDomainSupport; private final MeetingRuntimeProfileResolver meetingRuntimeProfileResolver; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; - private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService; + private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService; + private final RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler; private final MeetingProgressService meetingProgressService; private final MeetingPointsService meetingPointsService; private final MeetingSummaryPromptAssembler meetingSummaryPromptAssembler; @@ -87,6 +94,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { private final AndroidPendingMeetingDraftService androidPendingMeetingDraftService; private final MeetingLockCache meetingLockCache; private final MeetingAsrPermitCache meetingAsrPermitCache; + private final SysParamService sysParamService; @Value("${imeeting.summary-orchestration.mode:INTERNAL_BUILTIN}") private String summaryOrchestrationMode; @@ -103,7 +111,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { MeetingDomainSupport meetingDomainSupport, MeetingRuntimeProfileResolver meetingRuntimeProfileResolver, RealtimeMeetingSessionStateService realtimeMeetingSessionStateService, - RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService, + RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService, RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler, MeetingProgressService meetingProgressService, MeetingPointsService meetingPointsService, MeetingSummaryPromptAssembler meetingSummaryPromptAssembler, @@ -112,7 +120,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { AndroidPushMessageService androidPushMessageService, AndroidPendingMeetingDraftService androidPendingMeetingDraftService, MeetingLockCache meetingLockCache, - MeetingAsrPermitCache meetingAsrPermitCache) { + MeetingAsrPermitCache meetingAsrPermitCache, + SysParamService sysParamService) { this.meetingService = meetingService; this.aiTaskService = aiTaskService; this.hotWordService = hotWordService; @@ -124,7 +133,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { this.meetingDomainSupport = meetingDomainSupport; this.meetingRuntimeProfileResolver = meetingRuntimeProfileResolver; this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService; - this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService; + this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService; + this.realtimeMeetingProxyWebSocketHandler = realtimeMeetingProxyWebSocketHandler; this.meetingProgressService = meetingProgressService; this.meetingPointsService = meetingPointsService; this.meetingSummaryPromptAssembler = meetingSummaryPromptAssembler; @@ -134,6 +144,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { this.androidPendingMeetingDraftService = androidPendingMeetingDraftService; this.meetingLockCache = meetingLockCache; this.meetingAsrPermitCache = meetingAsrPermitCache; + this.sysParamService = sysParamService; } @Override @@ -176,7 +187,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { aiTaskService.save(asrTask); Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId(); - meetingDomainSupport.createChapterTask( + createChapterTaskIfEnabled( meeting.getId(), runtimeProfile.getResolvedSummaryModelId(), chapterModelId, @@ -228,7 +239,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { hostUserId, hostName, runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedPromptId(), summaryDetailLevel, 0); meetingService.save(meeting); Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId(); - meetingDomainSupport.createChapterTask( + createChapterTaskIfEnabled( meeting.getId(), runtimeProfile.getResolvedSummaryModelId(), chapterModelId, @@ -339,58 +350,6 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { deleteMeetingArtifactsAfterCommit(id); } - @Override - @Transactional(rollbackFor = Exception.class) - public void appendRealtimeTranscripts(Long meetingId, List items) { - if (items == null || items.isEmpty()) { - return; - } - - Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper() - .eq(MeetingTranscript::getMeetingId, meetingId) - .orderByDesc(MeetingTranscript::getSortOrder) - .last("LIMIT 1")) - .stream() - .findFirst() - .map(MeetingTranscript::getSortOrder) - .orElse(0); - - int nextSortOrder = maxSortOrder == null ? 0 : maxSortOrder + 1; - boolean inserted = false; - for (RealtimeTranscriptItemDTO item : items) { - if (item.getContent() == null || item.getContent().isBlank()) { - continue; - } - - MeetingTranscript existing = transcriptMapper.selectOne(new LambdaQueryWrapper() - .eq(MeetingTranscript::getMeetingId, meetingId) - .eq(MeetingTranscript::getContent, item.getContent().trim()) - .eq(item.getSpeakerId() != null && !item.getSpeakerId().isBlank(), MeetingTranscript::getSpeakerId, item.getSpeakerId()) - .eq(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime()) - .eq(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime()) - .last("LIMIT 1")); - if (existing != null) { - continue; - } - - MeetingTranscript transcript = new MeetingTranscript(); - transcript.setMeetingId(meetingId); - transcript.setSpeakerId(meetingDomainSupport.resolveSpeakerId(item.getSpeakerId())); - transcript.setSpeakerName(meetingDomainSupport.resolveSpeakerName(item.getSpeakerId(), item.getSpeakerName())); - transcript.setContent(item.getContent().trim()); - transcript.setStartTime(item.getStartTime()); - transcript.setEndTime(item.getEndTime()); - transcript.setSortOrder(nextSortOrder++); - transcriptMapper.insert(transcript); - inserted = true; - } - - if (inserted) { - meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId); - realtimeMeetingSessionStateService.refreshAfterTranscript(meetingId); - } - } - @Override @Transactional(rollbackFor = Exception.class) public void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult) { @@ -449,11 +408,11 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { if (meeting == null) { throw new RuntimeException("会议不存在"); } - + realtimeMeetingProxyWebSocketHandler.closeMeetingSession(meetingId); RealtimeMeetingSessionStatusVO currentStatus = realtimeMeetingSessionStateService.getStatus(meetingId); if (overwriteAudio) { if (audioUrl == null || audioUrl.isBlank()) { - throw new RuntimeException("overwriteAudio 为 true 时必须提供音频地址"); + throw new RuntimeException("overwriteAudio=true requires audioUrl"); } meetingDomainSupport.applyMeetingAudioMetadata( meeting, @@ -494,10 +453,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { realtimeMeetingSessionStateService.clear(meetingId); meeting.setStatus(MeetingStatusEnum.SUMMARIZING.getCode()); meetingService.updateById(meeting); - updateMeetingProgress(meetingId, 85, "正在生成 AI 目录与总结...", 0); + updateMeetingProgress(meetingId, resolveAiCatalogEnabled() ? 85 : 90, resolveAiCatalogEnabled() ? "正在生成 AI 目录与总结..." : "正在生成会议总结...", 0); meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl()); - aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); - aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + if (!resolveAiCatalogEnabled()) { + aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + } else if (isParallelDispatchMode()) { + aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + } else { + aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); + } } @Override @@ -522,6 +487,22 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { meetingService.updateById(meeting); } + @Override + @Transactional(rollbackFor = Exception.class) + public void failOfflineTranscription(Long meetingId, String failureMessage) { + Meeting meeting = meetingService.getById(meetingId); + if (meeting == null) { + throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(),"会议不存在"); + } + if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) { + throw new RuntimeException("会议不是离线会议"); + } + meeting.setStatus(MeetingStatusEnum.FAILED.getCode()); + meeting.setOfflineRecordingStatus(MeetingConstants.OFFLINE_RECORDING_UPLOAD_FINISHED); + markAudioSaveFailure(meeting, failureMessage); + meetingService.updateById(meeting); + } + private void applyRealtimeAudioFinalizeResult(Meeting meeting, RealtimeMeetingAudioStorageService.FinalizeResult result) { if (result == null) { markAudioSaveFailure(meeting, RealtimeMeetingAudioStorageService.DEFAULT_FAILURE_MESSAGE); @@ -767,6 +748,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { @Transactional(rollbackFor = Exception.class) public MeetingTranscriptChapterImportResultVO importTranscriptChapters(MeetingTranscriptChapterImportDTO command) { ensureExternalSummaryModeEnabled(); + ensureAiCatalogEnabled(); if (command == null || command.getMeetingId() == null) { throw new RuntimeException("缺少会议ID，无法导入章节"); } @@ -787,6 +769,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { .last("LIMIT 1")); if (latestChapterTask == null) { + ensureAiCatalogEnabled(); Long summaryModelId = resolveSummaryModelId(command, latestSummaryTask); Long chapterModelId = resolveChapterModelId(command, latestSummaryTask, summaryModelId); Long promptId = resolvePromptId(command, latestSummaryTask); @@ -1120,7 +1103,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { .eq(AiTask::getTaskType, "CHAPTER") .orderByDesc(AiTask::getId) .last("LIMIT 1")); - if (chapterTask == null) { + if (resolveAiCatalogEnabled() && chapterTask == null) { chapterTask = meetingDomainSupport.createChapterTask( meetingId, effectiveSummaryModelId, @@ -1129,7 +1112,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { effectiveUserPrompt, effectiveSummaryDetailLevel ); - } else { + } else if (resolveAiCatalogEnabled()) { resetAiTask(chapterTask, meetingSummaryPromptAssembler.buildTaskConfig( effectiveSummaryModelId, effectiveChapterModelId, @@ -1185,6 +1168,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { if (!Integer.valueOf(3).equals(summaryTask.getStatus())) { throw new RuntimeException("当前总结环节未失败，无需重试"); } + if (resolveAiCatalogEnabled() && isSerialDispatchMode()) { + AiTask chapterTask = aiTaskService.getOne(new LambdaQueryWrapper() + .eq(AiTask::getMeetingId, meetingId) + .eq(AiTask::getTaskType, "CHAPTER") + .orderByDesc(AiTask::getId) + .last("LIMIT 1")); + if (chapterTask == null || !Integer.valueOf(2).equals(chapterTask.getStatus())) { + throw new RuntimeException("串行模式下缺少成功的 AI 目录产物，无法重试总结"); + } + } Long effectiveSummaryModelId = resolveMeetingSummaryModelId(meeting, summaryTask); resetAiTask(summaryTask, buildSummaryTaskConfigForRetry( summaryTask, @@ -1207,6 +1200,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { @Override @Transactional(rollbackFor = Exception.class) public void retryChapter(Long meetingId) { + ensureAiCatalogEnabled(); Meeting meeting = meetingService.getById(meetingId); if (meeting == null) { throw new RuntimeException("会议不存在"); @@ -1248,6 +1242,25 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { dispatchChapterTaskAfterCommit(meetingId, meeting.getTenantId(), meeting.getCreatorId()); } + private AiTask createChapterTaskIfEnabled(Long meetingId, + Long summaryModelId, + Long chapterModelId, + Long promptId, + String userPrompt, + String summaryDetailLevel) { + if (!resolveAiCatalogEnabled()) { + return null; + } + return meetingDomainSupport.createChapterTask( + meetingId, + summaryModelId, + chapterModelId, + promptId, + userPrompt, + summaryDetailLevel + ); + } + private void clearLegacyDispatchState(Long meetingId) { if (meetingId == null) { return; @@ -1586,4 +1599,45 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { } return MeetingConstants.SUMMARY_DETAIL_STANDARD; } + + private void ensureAiCatalogEnabled() { + if (!resolveAiCatalogEnabled()) { + throw new RuntimeException("AI目录功能未开启"); + } + } + + private boolean resolveAiCatalogEnabled() { + if (sysParamService == null) { + return false; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false"); + if (rawValue == null || rawValue.isBlank()) { + return false; + } + String normalized = rawValue.trim().toLowerCase(); + return "1".equals(normalized) + || "true".equals(normalized) + || "yes".equals(normalized) + || "on".equals(normalized); + } + + private boolean isSerialDispatchMode() { + return "SERIAL".equals(resolveSummaryDispatchMode()); + } + + private boolean isParallelDispatchMode() { + return "PARALLEL".equals(resolveSummaryDispatchMode()); + } + + private String resolveSummaryDispatchMode() { + if (sysParamService == null) { + return "PARALLEL"; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, "PARALLEL"); + if (rawValue == null || rawValue.isBlank()) { + return "PARALLEL"; + } + String normalized = rawValue.trim().toUpperCase(); + return "SERIAL".equals(normalized) ? "SERIAL" : "PARALLEL"; + } } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java index 9e5c6a0..0789854 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java @@ -1,7 +1,9 @@ package com.imeeting.service.biz.impl; +import cn.hutool.core.date.StopWatch; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.imeeting.common.MeetingConstants; +import com.imeeting.common.SysParamKeys; import com.imeeting.entity.biz.AiTask; import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.MeetingTranscript; @@ -13,6 +15,7 @@ import com.imeeting.service.biz.MeetingSummaryFileService; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; import com.unisbase.entity.SysUser; import com.unisbase.mapper.SysUserMapper; +import com.unisbase.service.SysParamService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -54,6 +57,7 @@ public class MeetingDomainSupport { private final ApplicationEventPublisher eventPublisher; private final MeetingSummaryFileService meetingSummaryFileService; private final MeetingPlaybackAudioResolver meetingPlaybackAudioResolver; + private final SysParamService sysParamService; @Value("${unisbase.app.upload-path}") private String uploadPath; @@ -399,6 +403,7 @@ public class MeetingDomainSupport { vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus()); vo.setSummaryModelId(meeting.getSummaryModelId()); vo.setPromptId(meeting.getPromptId()); + vo.setAiCatalogEnabled(resolveAiCatalogEnabled()); vo.setSummaryDetailLevel(normalizeSummaryDetailLevel(meeting.getSummaryDetailLevel())); vo.setAudioSaveStatus(meeting.getAudioSaveStatus()); vo.setAudioSaveMessage(meeting.getAudioSaveMessage()); @@ -648,6 +653,21 @@ public class MeetingDomainSupport { return path != null && path.contains("\\"); } + public boolean resolveAiCatalogEnabled() { + if (sysParamService == null) { + return false; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false"); + if (rawValue == null || rawValue.isBlank()) { + return false; + } + String normalized = rawValue.trim().toLowerCase(); + return "1".equals(normalized) + || "true".equals(normalized) + || "yes".equals(normalized) + || "on".equals(normalized); + } + private Path resolvePublicAudioPath(String audioUrl) { if (audioUrl == null || audioUrl.isBlank()) { return null; diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java index 729980d..865b842 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java @@ -485,7 +485,7 @@ public class MeetingPlaybackAudioResolver { "-vn", "-ar", String.valueOf(BROWSER_SAMPLE_RATE), "-c:a", "aac", - "-f", "mp4", + "-ac", "1", targetPath.toString() ); executeCommand(command, targetPath); diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingProgressServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingProgressServiceImpl.java index 829bcf2..05c02d5 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingProgressServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingProgressServiceImpl.java @@ -241,6 +241,14 @@ public class MeetingProgressServiceImpl implements MeetingProgressService { if (latestChapter != null && Integer.valueOf(1).equals(latestChapter.getStatus())) { return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0); } + if (MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.SUMMARIZING)) { + if (latestSummary != null && Integer.valueOf(2).equals(latestSummary.getStatus())) { + return buildSnapshot(meetingId, latestSummary, meeting.getStatus(), MeetingProgressStage.SUMMARY_RUNNING, 90, "正在生成会议总结...", 0); + } + if (latestChapter != null && Integer.valueOf(0).equals(latestChapter.getStatus())) { + return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0); + } + } AiTask latestAsr = findLatestTask(meetingId, "ASR"); if (latestAsr != null) { if (Integer.valueOf(1).equals(latestAsr.getStatus())) { diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java index f263c0e..67e825f 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java @@ -1,5 +1,6 @@ package com.imeeting.service.biz.impl; +import cn.hutool.core.date.StopWatch; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.imeeting.dto.biz.MeetingSummaryPromptContextRequestDTO; @@ -19,6 +20,7 @@ import com.imeeting.service.biz.MeetingService; import com.imeeting.service.biz.MeetingTranscriptChapterService; import com.unisbase.dto.PageResult; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -27,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +@Slf4j @Service @RequiredArgsConstructor public class MeetingQueryServiceImpl implements MeetingQueryService { @@ -83,10 +86,12 @@ public class MeetingQueryServiceImpl implements MeetingQueryService { return meeting != null ? toVO(meeting, true) : null; } + + @Override - public MeetingVO getDetailIgnoreTenant(Long id) { + public MeetingVO getDetailIgnoreTenant(Long id, Boolean includeAudio) { Meeting meeting = meetingMapper.selectByIdIgnoreTenant(id); - return meeting != null ? toVO(meeting, true) : null; + return meeting != null ? toVO(meeting, includeAudio) : null; } @Override diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingTranscriptChapterServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingTranscriptChapterServiceImpl.java index 192140c..82c1e9f 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingTranscriptChapterServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingTranscriptChapterServiceImpl.java @@ -369,28 +369,29 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha private String buildChapterSystemPrompt() { return """ - 你负责对会议转录分段做章节边界识别。 - 只允许返回 JSON。 - 只能返回 chapters 数组。 - JSON里面必须包含chapters 数组,就算只有一个章节 - 每个章节只允许包含： - chapterNo,title,summary,keywords,startTranscriptId,endTranscriptId,confidence - 不得改写原文。 - 不得输出章节正文。 - 不得归一化数字、日期、金额、时间点。 - 所有章节必须完整覆盖全部 transcript。 - 章节必须严格连续： - - 第一个章节 startTranscriptId 必须为 转录原文的起始transcriptId - - 下一个章节的 startTranscriptId 必须等于上一个章节的 endTranscriptId + 1 - - 最后一个章节必须覆盖最后一条 transcript - 禁止： - - transcript 遗漏 - - transcript 重复 - - 章节重叠 - - 跳跃式分段 - 章节标题、摘要、关键词必须基于对应章节原文生成，不得虚构。 - 若无法识别明确的话题边界，则将全部 transcript 作为一个章节返回. - + 你是会议转录分段任务中的“章节边界识别器”。 + 基于输入 transcript 列表，进行语义分段，输出章节结构。 + 输出格式（严格） + 只允许输出 JSON，且必须符合以下结构： + { + "chapters": [ + { + "chapterNo": number, + "title": string, + "summary": string, + "keywords": [string], + "startTranscriptId": number, + "endTranscriptId": number, + "confidence": number + } + ] + } + 规则： + 1. 必须按顺序分段，不允许交叉或跳跃 + 2. 必须覆盖全部 transcript + 3. 若无明显边界，则合并为一个章节 + 4. title 必须基于该段内容生成 + 5. 只输出 JSON，不要任何解释 """; } @@ -1017,9 +1018,9 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha Long startTranscriptId = longValue(item.path("startTranscriptId").asText(null)); Long endTranscriptId = longValue(item.path("endTranscriptId").asText(null)); Integer chapterNo = item.path("chapterNo").isInt() ? item.path("chapterNo").asInt() : null; - if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) { - throw new RuntimeException("章节模型返回了不完整的章节边界"); - } +// if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) { +// throw new RuntimeException("章节模型返回了不完整的章节边界"); +// } List keywords = new ArrayList<>(); if (item.path("keywords").isArray()) { for (JsonNode keyword : item.path("keywords")) { diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingUnifiedStatusServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingUnifiedStatusServiceImpl.java index 4a64add..14435c8 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingUnifiedStatusServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingUnifiedStatusServiceImpl.java @@ -2,6 +2,7 @@ package com.imeeting.service.biz.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.imeeting.common.MeetingConstants; +import com.imeeting.common.SysParamKeys; import com.imeeting.dto.biz.MeetingProgressSnapshot; import com.imeeting.dto.biz.MeetingVO; import com.imeeting.dto.biz.UnifiedMeetingStatusStage; @@ -17,6 +18,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptChapterVersionMapper; import com.imeeting.mapper.biz.MeetingTranscriptMapper; import com.imeeting.service.biz.MeetingUnifiedStatusService; import com.imeeting.support.redis.MeetingProgressCache; +import com.unisbase.service.SysParamService; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -31,6 +33,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ private final MeetingTranscriptMapper meetingTranscriptMapper; private final MeetingTranscriptChapterVersionMapper chapterVersionMapper; private final MeetingProgressCache meetingProgressCache; + private final SysParamService sysParamService; @Override public UnifiedMeetingStatusVO resolve(MeetingVO meeting, MeetingProgressSnapshot snapshot) { @@ -87,12 +90,12 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ if (isAndroidOfflineMeetingWaitingUpload(meeting)) { return UnifiedMeetingStatusStage.WAITING_UPLOAD; } - UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot); + MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot); + UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot, context); if (stageFromSnapshot != null) { return stageFromSnapshot; } - MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot); if (isTranscribing(context)) { return UnifiedMeetingStatusStage.TRANSCRIBING; } @@ -103,7 +106,8 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ return UnifiedMeetingStatusStage.INITIALIZING; } - private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot) { + private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot, + MeetingUnifiedStageContext context) { if (snapshot == null || snapshot.getStage() == null || snapshot.getStage().isBlank()) { return null; } @@ -112,15 +116,31 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ case "completed" -> UnifiedMeetingStatusStage.COMPLETED; case "summary_running", "chapter_running" -> UnifiedMeetingStatusStage.SUMMARIZING; case "asr_running", "asr_completed", "asr_submitted" -> UnifiedMeetingStatusStage.TRANSCRIBING; - case "queued" -> UnifiedMeetingStatusStage.INITIALIZING; + case "queued" -> resolveQueuedSnapshotStage(context); default -> null; }; } + private UnifiedMeetingStatusStage resolveQueuedSnapshotStage(MeetingUnifiedStageContext context) { + if (context == null) { + return UnifiedMeetingStatusStage.INITIALIZING; + } + if (isSummarizing(context)) { + return UnifiedMeetingStatusStage.SUMMARIZING; + } + if (isTranscribing(context)) { + return UnifiedMeetingStatusStage.TRANSCRIBING; + } + return UnifiedMeetingStatusStage.INITIALIZING; + } + private UnifiedMeetingStatusStage resolveFailedStage(MeetingVO meeting) { if (meeting == null || !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)) { return null; } + if (isAndroidOfflineEmptyUploadFailure(meeting)) { + return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING; + } AiTask asrTask = findLatestTask(meeting.getId(), "ASR"); if (isTaskFailed(asrTask)) { return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING; @@ -140,6 +160,13 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ return UnifiedMeetingStatusStage.FAILED_INITIALIZING; } + private boolean isAndroidOfflineEmptyUploadFailure(MeetingVO meeting) { + return meeting != null + && MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType()) + && MeetingConstants.SOURCE_ANDROID.equalsIgnoreCase(meeting.getMeetingSource()) + && "FAILED".equalsIgnoreCase(meeting.getAudioSaveStatus()); + } + private boolean isAndroidOfflineMeetingWaitingUpload(MeetingVO meeting) { return meeting != null && MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType()) @@ -162,6 +189,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ } private Integer resolvePercent(MeetingProgressSnapshot snapshot, UnifiedMeetingStatusStage stage) { + if (stage != null && stage.getCode().startsWith("FAILED_")) { + return -1; + } if (snapshot != null && snapshot.getPercent() != null) { return snapshot.getPercent(); } @@ -203,6 +233,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ if (meeting == null) { return "处理失败"; } + if (meeting.getAudioSaveMessage() != null && !meeting.getAudioSaveMessage().isBlank()) { + return meeting.getAudioSaveMessage(); + } if (meeting.getLatestSummaryAttemptErrorMsg() != null && !meeting.getLatestSummaryAttemptErrorMsg().isBlank()) { return meeting.getLatestSummaryAttemptErrorMsg(); } @@ -222,6 +255,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ } private boolean canViewAiChapters(Long meetingId) { + if (!resolveAiCatalogEnabled()) { + return false; + } return meetingId != null && chapterVersionMapper.selectCount(new LambdaQueryWrapper() .eq(MeetingTranscriptChapterVersion::getMeetingId, meetingId) .eq(MeetingTranscriptChapterVersion::getIsCurrent, 1) @@ -274,6 +310,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ vo.setSourceDeviceCode(meeting.getSourceDeviceCode()); vo.setSourceDeviceMode(meeting.getSourceDeviceMode()); vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus()); + vo.setAiCatalogEnabled(resolveAiCatalogEnabled()); vo.setSummaryDetailLevel(meeting.getSummaryDetailLevel()); vo.setAudioSaveStatus(meeting.getAudioSaveStatus()); vo.setAudioSaveMessage(meeting.getAudioSaveMessage()); @@ -284,6 +321,21 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ return vo; } + private boolean resolveAiCatalogEnabled() { + if (sysParamService == null) { + return false; + } + String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false"); + if (rawValue == null || rawValue.isBlank()) { + return false; + } + String normalized = rawValue.trim().toLowerCase(); + return "1".equals(normalized) + || "true".equals(normalized) + || "yes".equals(normalized) + || "on".equals(normalized); + } + private record MeetingUnifiedStageContext(AiTask asrTask, AiTask chapterTask, AiTask summaryTask, diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSessionStateServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSessionStateServiceImpl.java index 38578a1..744656f 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSessionStateServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSessionStateServiceImpl.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.imeeting.dto.biz.RealtimeMeetingResumeConfig; import com.imeeting.dto.biz.RealtimeMeetingSessionState; import com.imeeting.dto.biz.RealtimeMeetingSessionStatusVO; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState; import com.imeeting.entity.biz.Meeting; import com.imeeting.entity.biz.MeetingTranscript; import com.imeeting.enums.MeetingStatusEnum; @@ -12,6 +13,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptMapper; import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.support.redis.MeetingLockCache; import com.imeeting.support.redis.RealtimeMeetingSessionCache; +import com.imeeting.support.redis.RealtimeMeetingTranscriptCache; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -31,6 +33,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe private final MeetingLockCache meetingLockCache; private final MeetingTranscriptMapper transcriptMapper; private final MeetingMapper meetingMapper; + private final RealtimeMeetingTranscriptCache realtimeMeetingTranscriptCache; @Value("${imeeting.realtime.resume-window-minutes:30}") private String resumeWindowMinutesValue; @@ -49,8 +52,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe next.setTenantId(tenantId); next.setUserId(userId); next.setStatus("IDLE"); - next.setHasTranscript(countTranscripts(meetingId) > 0); - next.setTranscriptCountSnapshot(countTranscripts(meetingId)); + long transcriptCount = countCapturedTranscripts(meetingId); + next.setHasTranscript(transcriptCount > 0); + next.setTranscriptCountSnapshot(transcriptCount); next.setUpdatedAt(System.currentTimeMillis()); writeState(next); } @@ -59,6 +63,22 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe public void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig) { RealtimeMeetingSessionState state = getOrCreateState(meetingId); state.setResumeConfig(resumeConfig); + state.setUpdatedAt(System.currentTimeMillis()); + writeState(state); + } + + @Override + public void rememberSpeakerContext(Long meetingId, String speakerContextId) { + if (meetingId == null || speakerContextId == null || speakerContextId.isBlank()) { + return; + } + RealtimeMeetingSessionState state = getOrCreateState(meetingId); + RealtimeMeetingResumeConfig resumeConfig = state.getResumeConfig(); + if (resumeConfig == null) { + resumeConfig = new RealtimeMeetingResumeConfig(); + state.setResumeConfig(resumeConfig); + } + resumeConfig.setSpeakerContextId(speakerContextId.trim()); state.setUpdatedAt(System.currentTimeMillis()); writeState(state); } @@ -72,16 +92,16 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe String currentStatus = status.getStatus(); if ("COMPLETING".equals(currentStatus)) { - throw new RuntimeException("实时会议正在结束处理中"); + throw new RuntimeException("Realtime meeting is completing"); } if ("COMPLETED".equals(currentStatus)) { - throw new RuntimeException("实时会议已结束"); + throw new RuntimeException("Realtime meeting is already completed"); } if ("ACTIVE".equals(currentStatus) || Boolean.TRUE.equals(status.getActiveConnection())) { - throw new RuntimeException("实时会议已存在活动连接"); + throw new RuntimeException("Realtime meeting already has an active connection"); } if ("PAUSED_RESUMABLE".equals(currentStatus) && !Boolean.TRUE.equals(status.getCanResume())) { - throw new RuntimeException("实时会议恢复窗口已过期"); + throw new RuntimeException("Realtime meeting resume window has expired"); } } @@ -102,7 +122,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe } long now = System.currentTimeMillis(); - long transcriptCount = countTranscripts(meetingId); + long transcriptCount = countCapturedTranscripts(meetingId); state.setStatus("ACTIVE"); state.setHasTranscript(transcriptCount > 0); state.setTranscriptCountSnapshot(transcriptCount); @@ -182,9 +202,13 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe @Override public void refreshAfterTranscript(Long meetingId) { + refreshAfterTranscriptCapture(meetingId, countTranscripts(meetingId)); + } + + @Override + public void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount) { RealtimeMeetingSessionState state = getOrCreateState(meetingId); long now = System.currentTimeMillis(); - long transcriptCount = countTranscripts(meetingId); state.setHasTranscript(transcriptCount > 0); state.setTranscriptCountSnapshot(transcriptCount); @@ -214,7 +238,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe return false; } - long transcriptCount = countTranscripts(meetingId); + long transcriptCount = countCapturedTranscripts(meetingId); if (transcriptCount <= 0) { clear(meetingId); return false; @@ -256,7 +280,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe } private RealtimeMeetingSessionStatusVO pauseState(Long meetingId, RealtimeMeetingSessionState state) { - long transcriptCount = countTranscripts(meetingId); + long transcriptCount = countCapturedTranscripts(meetingId); long now = System.currentTimeMillis(); state.setHasTranscript(transcriptCount > 0); @@ -307,7 +331,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe } else { vo.setStatus("IDLE"); } - vo.setHasTranscript(countTranscripts(meetingId) > 0); + vo.setHasTranscript(countCapturedTranscripts(meetingId) > 0); vo.setCanResume(false); vo.setRemainingSeconds(0L); vo.setActiveConnection(false); @@ -362,8 +386,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe RealtimeMeetingSessionState next = new RealtimeMeetingSessionState(); next.setMeetingId(meetingId); next.setStatus("IDLE"); - next.setHasTranscript(countTranscripts(meetingId) > 0); - next.setTranscriptCountSnapshot(countTranscripts(meetingId)); + long transcriptCount = countCapturedTranscripts(meetingId); + next.setHasTranscript(transcriptCount > 0); + next.setTranscriptCountSnapshot(transcriptCount); next.setUpdatedAt(System.currentTimeMillis()); return next; } @@ -400,4 +425,21 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe return transcriptMapper.selectCount(new LambdaQueryWrapper() .eq(MeetingTranscript::getMeetingId, meetingId)); } + + private long countCapturedTranscripts(Long meetingId) { + return Math.max(countTranscripts(meetingId), countCachedTranscripts(meetingId)); + } + + private long countCachedTranscripts(Long meetingId) { + if (meetingId == null) { + return 0L; + } + RealtimeMeetingTranscriptCacheState state = realtimeMeetingTranscriptCache.getState(meetingId); + if (state == null || state.getItems() == null || state.getItems().isEmpty()) { + return 0L; + } + return state.getItems().stream() + .filter(item -> item.getContent() != null && !item.getContent().isBlank()) + .count(); + } } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSocketSessionServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSocketSessionServiceImpl.java index 6477014..b15fcf7 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSocketSessionServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/RealtimeMeetingSocketSessionServiceImpl.java @@ -10,12 +10,13 @@ import com.imeeting.service.biz.AiModelService; import com.imeeting.service.biz.MeetingAccessService; import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.biz.RealtimeMeetingSocketSessionService; +import com.imeeting.service.realtime.RealtimeAsrChannel; +import com.imeeting.service.realtime.RealtimeAsrChannelFactory; import com.imeeting.support.redis.RealtimeMeetingSocketSessionCache; import com.unisbase.security.LoginUser; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -30,6 +31,7 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS private final MeetingAccessService meetingAccessService; private final AiModelService aiModelService; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; + private final RealtimeAsrChannelFactory realtimeAsrChannelFactory; @Override public RealtimeSocketSessionVO createSession(Long meetingId, Long asrModelId, String mode, String language, @@ -37,10 +39,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS Boolean enableTextRefine, Boolean saveAudio, List> hotwords, LoginUser loginUser) { if (meetingId == null) { - throw new RuntimeException("会议 ID 不能为空"); + throw new RuntimeException("会议 ID 不能为空"); } if (asrModelId == null) { - throw new RuntimeException("ASR 模型 ID 不能为空"); + throw new RuntimeException("ASR 模型 ID 不能为空"); } Meeting meeting = meetingAccessService.requireMeeting(meetingId); @@ -51,12 +53,13 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS AiModelVO asrModel = aiModelService.getModelById(asrModelId, "ASR"); if (asrModel == null) { - throw new RuntimeException("ASR 模型不存在"); + throw new RuntimeException("ASR 模型不存在"); } - String targetWsUrl = resolveWsUrl(asrModel); + RealtimeAsrChannel realtimeAsrChannel = realtimeAsrChannelFactory.getRequired(asrModel.getProvider()); + String targetWsUrl = realtimeAsrChannel.resolveTargetWsUrl(asrModel); if (targetWsUrl == null || targetWsUrl.isBlank()) { - throw new RuntimeException("ASR 模型未配置 WebSocket 地址"); + throw new RuntimeException("ASR 模型未配置 WebSocket 地址"); } RealtimeMeetingResumeConfig resumeConfig = new RealtimeMeetingResumeConfig(); @@ -82,7 +85,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS sessionData.setUserId(loginUser.getUserId()); sessionData.setTenantId(loginUser.getTenantId()); sessionData.setAsrModelId(asrModelId); + sessionData.setProvider(realtimeAsrChannelFactory.normalizeProvider(asrModel.getProvider())); sessionData.setTargetWsUrl(targetWsUrl); + sessionData.setModelCode(asrModel.getModelCode()); + sessionData.setMediaConfig(asrModel.getMediaConfig()); String sessionToken = UUID.randomUUID().toString().replace("-", ""); socketSessionCache.save(sessionToken, sessionData); @@ -91,9 +97,8 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS vo.setSessionToken(sessionToken); vo.setPath(WS_PATH); vo.setExpiresInSeconds(socketSessionCache.getSessionTtlSeconds()); - vo.setStartMessage(buildStartMessage( + vo.setStartMessage(realtimeAsrChannel.buildStartMessage( asrModel, - meetingId, mode, language, useSpkId, @@ -110,74 +115,4 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS public RealtimeSocketSessionData getSessionData(String sessionToken) { return socketSessionCache.get(sessionToken); } - - private String resolveWsUrl(AiModelVO model) { - if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) { - return model.getWsUrl(); - } - if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) { - return ""; - } - return model.getBaseUrl() - .replaceFirst("^http://", "ws://") - .replaceFirst("^https://", "wss://"); - } - - private Map buildStartMessage(AiModelVO model, Long meetingId, String mode, String language, - Integer useSpkId, Boolean enablePunctuation, Boolean enableItn, - Boolean enableTextRefine, Boolean saveAudio, - List> hotwords) { - Map root = new HashMap<>(); - root.put("type", "start"); - root.put("request_id", "web_" + System.currentTimeMillis() + "_" + meetingId); - root.put("authorization", buildAuthorization(model.getApiKey())); - - Map config = new HashMap<>(); - Map audio = new HashMap<>(); - audio.put("format", "pcm"); - audio.put("sample_rate", 16000); - audio.put("channels", 1); - config.put("audio", audio); - - Map recognition = new HashMap<>(); - recognition.put("language", normalizeLanguage(language)); - recognition.put("enable_punctuation", boolOrDefault(enablePunctuation, true)); - recognition.put("enable_itn", boolOrDefault(enableItn, true)); - recognition.put("enable_speaker", Integer.valueOf(1).equals(useSpkId)); - recognition.put("enable_two_pass", !"online".equalsIgnoreCase(mode)); - recognition.put("enable_text_refine", boolOrDefault(enableTextRefine, false)); - recognition.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig())); - recognition.put("hotwords", hotwords == null ? List.of() : hotwords); - config.put("recognition", recognition); - - config.put("model", model.getModelCode()); - config.put("save_audio", boolOrDefault(saveAudio, false)); - root.put("config", config); - return root; - } - - private String buildAuthorization(String apiKey) { - if (apiKey == null || apiKey.isBlank()) { - return ""; - } - return apiKey.startsWith("Bearer ") ? apiKey : "Bearer " + apiKey; - } - - private Object readSpeakerThreshold(Map mediaConfig) { - if (mediaConfig == null) { - return null; - } - return mediaConfig.get("svThreshold"); - } - - private String normalizeLanguage(String language) { - if (language == null || language.isBlank()) { - return "auto"; - } - return language.trim(); - } - - private boolean boolOrDefault(Boolean value, boolean defaultValue) { - return value != null ? value : defaultValue; - } } diff --git a/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannel.java b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannel.java new file mode 100644 index 0000000..c7cc23e --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannel.java @@ -0,0 +1,36 @@ +package com.imeeting.service.realtime; + +import com.imeeting.dto.biz.AiModelVO; + +import java.util.List; +import java.util.Map; + +public interface RealtimeAsrChannel { + boolean supports(String provider); + + String resolveTargetWsUrl(AiModelVO model); + + Map buildStartMessage(AiModelVO model, + String mode, + String language, + Integer useSpkId, + Boolean enablePunctuation, + Boolean enableItn, + Boolean enableTextRefine, + Boolean saveAudio, + List> hotwords); + + void connect(RealtimeAsrChannelContext context) throws Exception; + + void handleFrontendText(RealtimeAsrChannelContext context, String payload); + + void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload); + + default void onFrontendDetached(RealtimeAsrChannelContext context) { + // default no-op + } + + void closeMeeting(RealtimeAsrChannelContext context); + + boolean isOpen(RealtimeAsrChannelContext context); +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelCallback.java b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelCallback.java new file mode 100644 index 0000000..e5697f6 --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelCallback.java @@ -0,0 +1,17 @@ +package com.imeeting.service.realtime; + +import org.springframework.web.socket.CloseStatus; + +public interface RealtimeAsrChannelCallback { + void onChannelOpen(Long meetingId) throws Exception; + + void sendFrontendText(Long meetingId, String payload) throws Exception; + + void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception; + + void sendFrontendError(Long meetingId, String code, String message); + + void removeMeetingSession(Long meetingId); + + void closeFrontend(Long meetingId, CloseStatus status); +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelContext.java b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelContext.java new file mode 100644 index 0000000..0098dfa --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelContext.java @@ -0,0 +1,26 @@ +package com.imeeting.service.realtime; + +import lombok.Data; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Data +public class RealtimeAsrChannelContext { + private Long meetingId; + private String provider; + private String targetWsUrl; + private WebSocketSession rawSession; + private ConcurrentWebSocketSessionDecorator frontendSession; + private RealtimeAsrChannelCallback callback; + private final ConcurrentMap channelState = new ConcurrentHashMap<>(); + private volatile ConcurrentMap frontendState = new ConcurrentHashMap<>(); + + public void bindFrontendSession(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) { + this.rawSession = rawSession; + this.frontendSession = frontendSession; + this.frontendState = new ConcurrentHashMap<>(); + } +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelFactory.java b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelFactory.java new file mode 100644 index 0000000..8b305ae --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/RealtimeAsrChannelFactory.java @@ -0,0 +1,29 @@ +package com.imeeting.service.realtime; + +import com.imeeting.enums.ModelProviderEnum; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@RequiredArgsConstructor +public class RealtimeAsrChannelFactory { + + private final List channels; + + public RealtimeAsrChannel getRequired(String provider) { + String normalizedProvider = normalizeProvider(provider); + return channels.stream() + .filter(channel -> channel.supports(normalizedProvider)) + .findFirst() + .orElseThrow(() -> new RuntimeException("暂不支持的实时 ASR 渠道: " + provider)); + } + + public String normalizeProvider(String provider) { + if (provider == null || provider.isBlank()) { + return ModelProviderEnum.LOCAL.getCode(); + } + return provider.trim().toLowerCase(); + } +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/RealtimeMeetingTranscriptCacheService.java b/backend/src/main/java/com/imeeting/service/realtime/RealtimeMeetingTranscriptCacheService.java new file mode 100644 index 0000000..a272936 --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/RealtimeMeetingTranscriptCacheService.java @@ -0,0 +1,13 @@ +package com.imeeting.service.realtime; + +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem; + +import java.util.List; + +public interface RealtimeMeetingTranscriptCacheService { + void mergeUpstreamMessage(Long meetingId, String payload); + + List listOrderedItems(Long meetingId); + + void clear(Long meetingId); +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/impl/LocalRealtimeAsrChannel.java b/backend/src/main/java/com/imeeting/service/realtime/impl/LocalRealtimeAsrChannel.java new file mode 100644 index 0000000..5b03acc --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/impl/LocalRealtimeAsrChannel.java @@ -0,0 +1,538 @@ +package com.imeeting.service.realtime.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.imeeting.dto.biz.AiModelVO; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem; +import com.imeeting.enums.ModelProviderEnum; +import com.imeeting.service.biz.RealtimeMeetingSessionStateService; +import com.imeeting.service.realtime.RealtimeAsrChannel; +import com.imeeting.service.realtime.RealtimeAsrChannelContext; +import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.CloseStatus; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +@Slf4j +@Component +@RequiredArgsConstructor +public class LocalRealtimeAsrChannel implements RealtimeAsrChannel { + + private static final String STATE_UPSTREAM_SOCKET = "upstreamSocket"; + private static final String STATE_CLOSE_AFTER_END = "closeAfterEnd"; + private static final String STATE_START_MESSAGE_FORWARDED = "startMessageForwarded"; + private static final String STATE_UPSTREAM_SEND_CHAIN = "upstreamSendChain"; + private static final String STATE_START_MESSAGE_SENT = "startMessageSent"; + private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames"; + private static final CompletableFuture COMPLETED = CompletableFuture.completedFuture(null); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String STOP_MESSAGE = "{\"type\":\"stop\"}"; + + private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; + private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService; + + @Override + public boolean supports(String provider) { + return ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(provider); + } + + @Override + public String resolveTargetWsUrl(AiModelVO model) { + if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) { + return model.getWsUrl(); + } + if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) { + return ""; + } + return model.getBaseUrl() + .replaceFirst("^http://", "ws://") + .replaceFirst("^https://", "wss://"); + } + + @Override + public Map buildStartMessage(AiModelVO model, + String mode, + String language, + Integer useSpkId, + Boolean enablePunctuation, + Boolean enableItn, + Boolean enableTextRefine, + Boolean saveAudio, + List> hotwords) { + Map root = new HashMap<>(); + root.put("type", "start"); + + Map payload = new HashMap<>(); + payload.put("format", "pcm"); + payload.put("sample_rate", 16000); + payload.put("language", normalizeProtocolLanguage(language)); + payload.put("context", ""); + payload.put("enable_inverse_text_normalization", boolOrDefault(enableItn, true)); + payload.put("unfixed_token_num", 3); + payload.put("silence_duration_ms", 800); + payload.put("min_partial_sec", 0.3D); + payload.put("pre_roll_ms", 240); + payload.put("max_sentence_count", 8); + payload.put("partial_holdback_chars", 2); + payload.put("enable_native_partial_stream", false); + payload.put("enable_speaker", Integer.valueOf(1).equals(useSpkId)); + payload.put("match_speaker_registry", false); + payload.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig())); + payload.put("enable_realtime_longform", false); + payload.put("enable_realtime_vad_split", false); + payload.put("force_stable_segment_sec", 6); + payload.put("force_stable_min_chars", 24); + payload.put("max_segment_sec", 12); + payload.put("hotwords", hotwords == null ? List.of() : hotwords); + root.put("payload", payload); + return root; + } + + @Override + public void connect(RealtimeAsrChannelContext context) throws Exception { + initializeFrontendState(context); + java.net.http.WebSocket upstreamSocket = java.net.http.HttpClient.newHttpClient() + .newWebSocketBuilder() + .buildAsync(URI.create(context.getTargetWsUrl()), new UpstreamListener(context)) + .get(); + context.getChannelState().put(STATE_UPSTREAM_SOCKET, upstreamSocket); + } + + @Override + public void handleFrontendText(RealtimeAsrChannelContext context, String payload) { + java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context); + if (upstreamSocket == null) { + return; + } + initializeFrontendState(context); + if (looksLikeStartMessage(payload)) { + context.getFrontendState().put(STATE_START_MESSAGE_SENT, Boolean.TRUE); + if (!Boolean.TRUE.equals(context.getChannelState().get(STATE_START_MESSAGE_FORWARDED))) { + context.getChannelState().put(STATE_START_MESSAGE_FORWARDED, Boolean.TRUE); + sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text-start"); + } + flushPendingAudioFrames(context, upstreamSocket); + return; + } + if (looksLikeStopMessage(payload)) { + context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE); + } + sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text"); + } + + @Override + public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) { + java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context); + if (upstreamSocket == null) { + return; + } + initializeFrontendState(context); + if (!Boolean.TRUE.equals(context.getFrontendState().get(STATE_START_MESSAGE_SENT))) { + queuePendingAudioFrame(context, payload); + return; + } + sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary"); + } + + @Override + public void closeMeeting(RealtimeAsrChannelContext context) { + if (context == null) { + return; + } + java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context); + if (upstreamSocket == null) { + return; + } + context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE); + upstreamSocket.sendText(STOP_MESSAGE, true); + } + + @Override + public boolean isOpen(RealtimeAsrChannelContext context) { + return getUpstreamSocket(context) != null; + } + + public static String buildFrontendTranscriptMessage(RealtimeMeetingTranscriptCacheItem item) throws JsonProcessingException { + ObjectNode root = OBJECT_MAPPER.createObjectNode(); + boolean isFinal = Boolean.TRUE.equals(item.getFinalResult()); + root.put("type", isFinal ? "segment" : "partial"); + ObjectNode data = root.putObject("data"); + data.put("text", item.getContent()); + data.put("is_final", isFinal); + if (item.getSentenceId() != null) { + data.put("sentence_id", item.getSentenceId()); + } + if (item.getStartTime() != null) { + data.put("start", item.getStartTime() / 1000D); + } + if (item.getEndTime() != null) { + data.put("end", item.getEndTime() / 1000D); + } + if (item.getSpeakerId() != null && !item.getSpeakerId().isBlank()) { + data.put("speaker_id", item.getSpeakerId()); + } + if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) { + data.put("speaker_name", item.getSpeakerName()); + } + if (item.getUserId() != null && !item.getUserId().isBlank()) { + data.put("user_id", item.getUserId()); + } + return OBJECT_MAPPER.writeValueAsString(root); + } + + private Object readSpeakerThreshold(Map mediaConfig) { + if (mediaConfig == null) { + return null; + } + return mediaConfig.get("svThreshold"); + } + + private String normalizeProtocolLanguage(String language) { + if (language == null || language.isBlank()) { + return null; + } + String normalized = language.trim(); + if ("auto".equalsIgnoreCase(normalized)) { + return null; + } + return normalized; + } + + private boolean boolOrDefault(Boolean value, boolean defaultValue) { + return value != null ? value : defaultValue; + } + + private void initializeFrontendState(RealtimeAsrChannelContext context) { + context.getFrontendState().putIfAbsent(STATE_UPSTREAM_SEND_CHAIN, COMPLETED); + context.getFrontendState().putIfAbsent(STATE_START_MESSAGE_SENT, Boolean.FALSE); + context.getFrontendState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new ArrayList()); + } + + private java.net.http.WebSocket getUpstreamSocket(RealtimeAsrChannelContext context) { + Object value = context.getChannelState().get(STATE_UPSTREAM_SOCKET); + return value instanceof java.net.http.WebSocket socket ? socket : null; + } + + @SuppressWarnings("unchecked") + private void sendUpstreamOrdered(RealtimeAsrChannelContext context, + Supplier> sendAction, + String messageType) { + ConcurrentMap frontendState = context.getFrontendState(); + synchronized (frontendState) { + CompletableFuture chain = (CompletableFuture) frontendState.getOrDefault(STATE_UPSTREAM_SEND_CHAIN, COMPLETED); + CompletableFuture nextChain = chain + .exceptionally(ex -> null) + .thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null)); + nextChain = nextChain.whenComplete((ignored, ex) -> { + if (ex != null) { + log.error("顺序发送上游消息失败：meetingId={}, sessionId={}, type={}", + context.getMeetingId(), currentConnectionId(context), messageType, ex); + } + }); + frontendState.put(STATE_UPSTREAM_SEND_CHAIN, nextChain); + } + } + + @SuppressWarnings("unchecked") + private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) { + ConcurrentMap frontendState = context.getFrontendState(); + synchronized (frontendState) { + List pendingFrames = (List) frontendState.get(STATE_PENDING_AUDIO_FRAMES); + if (pendingFrames == null) { + pendingFrames = new ArrayList<>(); + frontendState.put(STATE_PENDING_AUDIO_FRAMES, pendingFrames); + } + pendingFrames.add(payload); + } + } + + @SuppressWarnings("unchecked") + private void flushPendingAudioFrames(RealtimeAsrChannelContext context, java.net.http.WebSocket upstreamSocket) { + List pendingFrames; + ConcurrentMap frontendState = context.getFrontendState(); + synchronized (frontendState) { + pendingFrames = (List) frontendState.get(STATE_PENDING_AUDIO_FRAMES); + if (pendingFrames == null || pendingFrames.isEmpty()) { + return; + } + frontendState.put(STATE_PENDING_AUDIO_FRAMES, new ArrayList()); + } + log.info("start 后开始补发排队音频帧：meetingId={}, sessionId={}, frameCount={}", + context.getMeetingId(), currentConnectionId(context), pendingFrames.size()); + for (byte[] frame : pendingFrames) { + sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush"); + } + } + + private String currentConnectionId(RealtimeAsrChannelContext context) { + return context.getRawSession() == null ? null : context.getRawSession().getId(); + } + + private static ByteBuffer copyBuffer(ByteBuffer source) { + ByteBuffer duplicate = source.asReadOnlyBuffer(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return ByteBuffer.wrap(bytes); + } + + private static boolean shouldLogBinaryFrame(int count) { + return count <= 3 || count % 25 == 0; + } + + private static String summarizeText(String payload) { + if (payload == null) { + return ""; + } + String normalized = payload.replaceAll("\\s+", " ").trim(); + if (normalized.length() <= 240) { + return normalized; + } + return normalized.substring(0, 240) + "..."; + } + + private static boolean looksLikeStartMessage(String payload) { + if (payload == null || payload.isBlank()) { + return false; + } + String normalized = payload.replaceAll("\\s+", ""); + return normalized.contains("\"type\":\"start\""); + } + + private static boolean looksLikeStopMessage(String payload) { + if (payload == null || payload.isBlank()) { + return false; + } + String normalized = payload.replaceAll("\\s+", ""); + return normalized.contains("\"type\":\"stop\""); + } + + private static boolean looksLikeEndMessage(String payload) { + if (payload == null || payload.isBlank()) { + return false; + } + try { + JsonNode root = OBJECT_MAPPER.readTree(payload); + return "end".equals(root.path("type").asText("")); + } catch (Exception ex) { + return false; + } + } + + private static List normalizeFrontendMessages(String upstreamPayload) { + try { + JsonNode root = OBJECT_MAPPER.readTree(upstreamPayload); + String type = root.path("type").asText(""); + if (!"sentences".equals(type) && !"end".equals(type)) { + return List.of(upstreamPayload); + } + List normalizedMessages = new ArrayList<>(); + JsonNode sentences = root.path("sentences"); + if (sentences.isArray()) { + for (JsonNode sentence : sentences) { + String text = sentence.path("sentence").asText("").trim(); + if (text.isEmpty()) { + continue; + } + boolean isFinal = sentence.path("sentence_type").asInt(0) != 0 || "end".equals(type); + normalizedMessages.add(buildFrontendTranscriptMessage(sentence, text, isFinal)); + } + } + if (normalizedMessages.isEmpty()) { + String fallbackText = root.path("result").path("voice_text_str").asText("").trim(); + if (!fallbackText.isEmpty()) { + normalizedMessages.add(buildFrontendTranscriptMessage((JsonNode) null, fallbackText, true)); + } + } + return normalizedMessages.isEmpty() ? List.of(upstreamPayload) : normalizedMessages; + } catch (Exception ex) { + return List.of(upstreamPayload); + } + } + + private static String buildFrontendTranscriptMessage(JsonNode sentence, String text, boolean isFinal) throws JsonProcessingException { + ObjectNode root = OBJECT_MAPPER.createObjectNode(); + root.put("type", isFinal ? "segment" : "partial"); + ObjectNode data = root.putObject("data"); + data.put("text", text); + data.put("is_final", isFinal); + if (sentence != null && sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt()) { + data.put("sentence_id", sentence.get("sentence_id").asInt()); + } + if (sentence != null) { + copyOptionalTimeSeconds(sentence, data, "start_time", "start"); + copyOptionalTimeSeconds(sentence, data, "end_time", "end"); + copyOptionalText(sentence, data, "speaker_id"); + copyOptionalText(sentence, data, "speaker_name"); + copyOptionalText(sentence, data, "user_id"); + } + return OBJECT_MAPPER.writeValueAsString(root); + } + + private static void copyOptionalTimeSeconds(JsonNode source, ObjectNode target, String sourceFieldName, String targetFieldName) { + if (source == null) { + return; + } + if (source.has(sourceFieldName) && source.get(sourceFieldName).isNumber()) { + target.put(targetFieldName, source.get(sourceFieldName).asDouble() / 1000D); + return; + } + if (source.has(targetFieldName) && source.get(targetFieldName).isNumber()) { + target.put(targetFieldName, source.get(targetFieldName).asDouble()); + } + } + + private static void copyOptionalText(JsonNode source, ObjectNode target, String fieldName) { + if (source == null || !source.has(fieldName) || source.get(fieldName).isNull()) { + return; + } + String value = source.get(fieldName).asText("").trim(); + if (!value.isEmpty()) { + target.put(fieldName, value); + } + } + + private final class UpstreamListener implements java.net.http.WebSocket.Listener { + private final RealtimeAsrChannelContext context; + private final StringBuilder textBuffer = new StringBuilder(); + private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream(); + private final AtomicInteger upstreamTextCount = new AtomicInteger(); + private final AtomicInteger upstreamBinaryCount = new AtomicInteger(); + + private UpstreamListener(RealtimeAsrChannelContext context) { + this.context = context; + } + + @Override + public void onOpen(java.net.http.WebSocket webSocket) { + context.getChannelState().put(STATE_UPSTREAM_SOCKET, webSocket); + log.info("上游 ASR websocket 已打开：meetingId={}, sessionId={}, upstream={}", + context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl()); + String connectionId = currentConnectionId(context); + if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) { + context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接"); + webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "当前会议无法激活这条前端连接"); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接")); + return; + } + try { + context.getCallback().onChannelOpen(context.getMeetingId()); + } catch (Exception ex) { + log.error("通知前端上游就绪失败：meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR); + return; + } + webSocket.request(1); + } + + @Override + public java.util.concurrent.CompletionStage onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) { + textBuffer.append(data); + if (last) { + int count = upstreamTextCount.incrementAndGet(); + String upstreamPayload = textBuffer.toString(); + realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), upstreamPayload); + try { + for (String frontendPayload : normalizeFrontendMessages(upstreamPayload)) { + context.getCallback().sendFrontendText(context.getMeetingId(), frontendPayload); + } + log.info("上游 ASR 文本 -> 前端：meetingId={}, sessionId={}, count={}, payload={}", + context.getMeetingId(), currentConnectionId(context), count, summarizeText(upstreamPayload)); + if (Boolean.TRUE.equals(context.getChannelState().get(STATE_CLOSE_AFTER_END)) + && looksLikeEndMessage(upstreamPayload)) { + webSocket.sendClose(CloseStatus.NORMAL.getCode(), "meeting-complete"); + } + } catch (Exception ex) { + log.error("转发上游 ASR 文本失败：meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR); + } finally { + textBuffer.setLength(0); + } + } + webSocket.request(1); + return COMPLETED; + } + + @Override + public java.util.concurrent.CompletionStage onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) { + byte[] chunk = new byte[data.remaining()]; + data.get(chunk); + binaryBuffer.writeBytes(chunk); + if (last) { + int count = upstreamBinaryCount.incrementAndGet(); + try { + context.getCallback().sendFrontendBinary(context.getMeetingId(), binaryBuffer.toByteArray()); + if (shouldLogBinaryFrame(count)) { + log.info("上游 ASR 二进制消息 -> 前端：meetingId={}, sessionId={}, count={}, bytes={}", + context.getMeetingId(), currentConnectionId(context), count, binaryBuffer.size()); + } + } catch (Exception ex) { + log.error("转发上游 ASR 二进制消息失败：meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR); + } finally { + binaryBuffer.reset(); + } + } + webSocket.request(1); + return COMPLETED; + } + + @Override + public java.util.concurrent.CompletionStage onPing(java.net.http.WebSocket webSocket, ByteBuffer message) { + webSocket.sendPong(copyBuffer(message)); + log.info("上游 ASR ping 已本地响应：meetingId={}, sessionId={}, bytes={}", + context.getMeetingId(), currentConnectionId(context), message.remaining()); + webSocket.request(1); + return COMPLETED; + } + + @Override + public java.util.concurrent.CompletionStage onPong(java.net.http.WebSocket webSocket, ByteBuffer message) { + log.debug("上游 ASR pong 已本地忽略：meetingId={}, sessionId={}, bytes={}", + context.getMeetingId(), currentConnectionId(context), message.remaining()); + webSocket.request(1); + return COMPLETED; + } + + @Override + public java.util.concurrent.CompletionStage onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) { + log.info("上游 ASR websocket 已关闭：meetingId={}, sessionId={}, code={}, reason={}", + context.getMeetingId(), currentConnectionId(context), statusCode, reason); + context.getChannelState().remove(STATE_UPSTREAM_SOCKET); + context.getCallback().removeMeetingSession(context.getMeetingId()); + context.getCallback().sendFrontendError(context.getMeetingId(), + "REALTIME_UPSTREAM_CLOSED", + reason == null || reason.isBlank() ? "上游 ASR WebSocket 已断开" : "上游 ASR WebSocket 已断开: " + reason); + context.getCallback().closeFrontend(context.getMeetingId(), new CloseStatus(statusCode, reason)); + return COMPLETED; + } + + @Override + public void onError(java.net.http.WebSocket webSocket, Throwable error) { + log.error("上游 ASR websocket 异常：meetingId={}, sessionId={}, upstream={}", + context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl(), error); + context.getChannelState().remove(STATE_UPSTREAM_SOCKET); + context.getCallback().removeMeetingSession(context.getMeetingId()); + context.getCallback().sendFrontendError(context.getMeetingId(), + "REALTIME_UPSTREAM_ERROR", + error == null || error.getMessage() == null || error.getMessage().isBlank() + ? "上游 ASR WebSocket 连接异常" + : "上游 ASR WebSocket 连接异常: " + error.getMessage()); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR); + } + } +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/impl/RealtimeMeetingTranscriptCacheServiceImpl.java b/backend/src/main/java/com/imeeting/service/realtime/impl/RealtimeMeetingTranscriptCacheServiceImpl.java new file mode 100644 index 0000000..235b14a --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/impl/RealtimeMeetingTranscriptCacheServiceImpl.java @@ -0,0 +1,291 @@ +package com.imeeting.service.realtime.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState; +import com.imeeting.entity.biz.MeetingTranscript; +import com.imeeting.mapper.biz.MeetingTranscriptMapper; +import com.imeeting.service.biz.MeetingTranscriptFileService; +import com.imeeting.service.biz.RealtimeMeetingSessionStateService; +import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService; +import com.imeeting.support.redis.RealtimeMeetingTranscriptCache; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +@Service +public class RealtimeMeetingTranscriptCacheServiceImpl implements RealtimeMeetingTranscriptCacheService { + + private final RealtimeMeetingTranscriptCache transcriptCache; + private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; + private final MeetingTranscriptMapper transcriptMapper; + private final MeetingTranscriptFileService meetingTranscriptFileService; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final Map meetingLocks = new ConcurrentHashMap<>(); + + @Autowired + public RealtimeMeetingTranscriptCacheServiceImpl(RealtimeMeetingTranscriptCache transcriptCache, + RealtimeMeetingSessionStateService realtimeMeetingSessionStateService, + MeetingTranscriptMapper transcriptMapper, + MeetingTranscriptFileService meetingTranscriptFileService) { + this.transcriptCache = transcriptCache; + this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService; + this.transcriptMapper = transcriptMapper; + this.meetingTranscriptFileService = meetingTranscriptFileService; + } + + @Override + public void mergeUpstreamMessage(Long meetingId, String payload) { + if (meetingId == null || payload == null || payload.isBlank()) { + return; + } + synchronized (lockForMeeting(meetingId)) { + RealtimeMeetingTranscriptCacheState state = getOrCreateState(meetingId); + try { + JsonNode root = objectMapper.readTree(payload); + String type = root.path("type").asText(""); + if (!"sentences".equals(type) && !"end".equals(type)) { + return; + } + JsonNode sentences = root.path("sentences"); + if (!sentences.isArray()) { + return; + } + for (JsonNode sentence : sentences) { + RealtimeMeetingTranscriptCacheItem item = mergeSentenceNode(state, sentence, "end".equals(type)); + persistFinalSentence(meetingId, item); + } + state.setUpdatedAt(System.currentTimeMillis()); + transcriptCache.saveState(state); + realtimeMeetingSessionStateService.refreshAfterTranscriptCapture(meetingId, countNonEmptyItems(state)); + } catch (Exception ignored) { + // ignore malformed upstream payload + } + } + } + + @Override + public List listOrderedItems(Long meetingId) { + RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId); + if (state == null || state.getItems() == null || state.getItems().isEmpty()) { + return List.of(); + } + return state.getItems().stream() + .filter(item -> item.getContent() != null && !item.getContent().isBlank()) + .sorted(Comparator.comparing(item -> item.getSortOrder() == null ? Integer.MAX_VALUE : item.getSortOrder())) + .toList(); + } + + @Override + public void clear(Long meetingId) { + transcriptCache.clear(meetingId); + } + + private RealtimeMeetingTranscriptCacheItem mergeSentenceNode(RealtimeMeetingTranscriptCacheState state, + JsonNode sentence, + boolean fromEndMessage) { + String text = sentence.path("sentence").asText("").trim(); + if (text.isEmpty()) { + return null; + } + Integer sentenceId = sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt() + ? sentence.get("sentence_id").asInt() + : null; + String upstreamSentenceKey = readText(sentence, "sentence_key"); + String sentenceKey = upstreamSentenceKey != null && !upstreamSentenceKey.isBlank() + ? upstreamSentenceKey + : sentenceId == null ? "sentence-" + nextLegacySequence(state) : "sentence-" + sentenceId; + RealtimeMeetingTranscriptCacheItem item = findBySentenceKey(state, sentenceKey); + long now = System.currentTimeMillis(); + if (item == null) { + item = new RealtimeMeetingTranscriptCacheItem(); + item.setSentenceKey(sentenceKey); + item.setSentenceGroupKey(upstreamSentenceKey); + item.setSentenceId(sentenceId); + item.setSortOrder(nextSortOrder(state)); + item.setFirstReceivedAt(now); + state.getItems().add(item); + } + item.setSentenceGroupKey(upstreamSentenceKey); + item.setSentenceType(readInteger(sentence, "sentence_type")); + item.setSpeakerId(resolveSpeakerId(sentence)); + item.setSpeakerName(readText(sentence, "speaker_name")); + item.setUserId(readText(sentence, "user_id")); + item.setStartTime(readTimeMilliseconds(sentence, "start_time", "start")); + item.setEndTime(readTimeMilliseconds(sentence, "end_time", "end")); + item.setContent(text); + item.setFinalResult(fromEndMessage || Objects.equals(item.getSentenceType(), 1)); + item.setUpdatedAt(now); + return item; + } + + private void persistFinalSentence(Long meetingId, RealtimeMeetingTranscriptCacheItem item) { + if (meetingId == null || item == null || !Boolean.TRUE.equals(item.getFinalResult())) { + return; + } + String content = item.getContent() == null ? null : item.getContent().trim(); + if (content == null || content.isBlank()) { + return; + } + String speakerId = resolveSpeakerId(item); + String speakerName = resolveSpeakerName(item); + if (item.getTranscriptId() != null) { + transcriptMapper.update(null, new LambdaUpdateWrapper() + .eq(MeetingTranscript::getId, item.getTranscriptId()) + .set(MeetingTranscript::getSpeakerId, speakerId) + .set(MeetingTranscript::getSpeakerName, speakerName) + .set(MeetingTranscript::getContent, content) + .set(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime()) + .set(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime())); + meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId); + return; + } + + MeetingTranscript transcript = new MeetingTranscript(); + transcript.setMeetingId(meetingId); + transcript.setSpeakerId(speakerId); + transcript.setSpeakerName(speakerName); + transcript.setContent(content); + transcript.setStartTime(item.getStartTime()); + transcript.setEndTime(item.getEndTime()); + transcript.setSortOrder(nextPersistedSortOrder(meetingId)); + transcriptMapper.insert(transcript); + item.setTranscriptId(transcript.getId()); + meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId); + } + + private int nextPersistedSortOrder(Long meetingId) { + Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper() + .eq(MeetingTranscript::getMeetingId, meetingId) + .orderByDesc(MeetingTranscript::getSortOrder) + .last("LIMIT 1")) + .stream() + .findFirst() + .map(MeetingTranscript::getSortOrder) + .orElse(0); + return maxSortOrder == null ? 0 : maxSortOrder + 1; + } + + private RealtimeMeetingTranscriptCacheItem findBySentenceKey(RealtimeMeetingTranscriptCacheState state, String sentenceKey) { + if (state.getItems() == null || state.getItems().isEmpty()) { + return null; + } + return state.getItems().stream() + .filter(item -> sentenceKey.equals(item.getSentenceKey())) + .findFirst() + .orElse(null); + } + + private String resolveSpeakerId(RealtimeMeetingTranscriptCacheItem item) { + if (item == null) { + return null; + } + if (item.getUserId() != null && !item.getUserId().isBlank()) { + return item.getUserId().trim(); + } + if (item.getSpeakerId() == null || item.getSpeakerId().isBlank() || "-1".equals(item.getSpeakerId().trim())) { + return null; + } + return item.getSpeakerId().trim(); + } + + private String resolveSpeakerName(RealtimeMeetingTranscriptCacheItem item) { + if (item == null) { + return null; + } + if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) { + return item.getSpeakerName().trim(); + } + String speakerId = resolveSpeakerId(item); + return speakerId == null || speakerId.isBlank() ? null : speakerId; + } + + private String resolveSpeakerId(JsonNode sentence) { + String userId = readText(sentence, "user_id"); + if (userId != null && !userId.isBlank()) { + return userId; + } + return readText(sentence, "speaker_id"); + } + + private Integer readInteger(JsonNode node, String fieldName) { + if (node == null || !node.has(fieldName) || !node.get(fieldName).canConvertToInt()) { + return null; + } + return node.get(fieldName).asInt(); + } + + private Integer readTimeMilliseconds(JsonNode node, String primaryField, String fallbackField) { + if (node == null) { + return null; + } + if (node.has(primaryField) && node.get(primaryField).canConvertToInt()) { + return node.get(primaryField).asInt(); + } + if (node.has(fallbackField) && node.get(fallbackField).isNumber()) { + return Math.round((float) (node.get(fallbackField).asDouble() * 1000)); + } + return null; + } + + private String readText(JsonNode node, String fieldName) { + if (node == null || !node.has(fieldName) || node.get(fieldName).isNull()) { + return null; + } + String value = node.get(fieldName).asText(""); + return value == null ? null : value.trim(); + } + + private long countNonEmptyItems(RealtimeMeetingTranscriptCacheState state) { + if (state.getItems() == null || state.getItems().isEmpty()) { + return 0L; + } + return state.getItems().stream() + .filter(item -> item.getContent() != null && !item.getContent().isBlank()) + .count(); + } + + private int nextSortOrder(RealtimeMeetingTranscriptCacheState state) { + Integer current = state.getNextSortOrder(); + int next = current == null ? 0 : current; + state.setNextSortOrder(next + 1); + return next; + } + + private int nextLegacySequence(RealtimeMeetingTranscriptCacheState state) { + Integer current = state.getNextLegacySequence(); + int next = current == null ? 0 : current; + state.setNextLegacySequence(next + 1); + return next; + } + + private RealtimeMeetingTranscriptCacheState getOrCreateState(Long meetingId) { + RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId); + if (state != null) { + if (state.getItems() == null) { + state.setItems(new ArrayList<>()); + } + return state; + } + RealtimeMeetingTranscriptCacheState next = new RealtimeMeetingTranscriptCacheState(); + next.setMeetingId(meetingId); + next.setItems(new ArrayList<>()); + next.setNextSortOrder(0); + next.setNextLegacySequence(0); + next.setUpdatedAt(System.currentTimeMillis()); + return next; + } + + private Object lockForMeeting(Long meetingId) { + return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object()); + } +} diff --git a/backend/src/main/java/com/imeeting/service/realtime/impl/TencentRealtimeAsrChannel.java b/backend/src/main/java/com/imeeting/service/realtime/impl/TencentRealtimeAsrChannel.java new file mode 100644 index 0000000..fd68fd8 --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/realtime/impl/TencentRealtimeAsrChannel.java @@ -0,0 +1,506 @@ +package com.imeeting.service.realtime.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.imeeting.dto.biz.AiModelVO; +import com.imeeting.enums.ModelProviderEnum; +import com.imeeting.service.biz.RealtimeMeetingSessionStateService; +import com.imeeting.service.realtime.RealtimeAsrChannel; +import com.imeeting.service.realtime.RealtimeAsrChannelContext; +import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService; +import com.tencent.asrspeaker.SpeakerConstant; +import com.tencent.asrspeaker.SpeakerRecognitionListener; +import com.tencent.asrspeaker.SpeakerRecognitionResponse; +import com.tencent.asrspeaker.SpeakerRecognizer; +import com.tencent.asrspeaker.SpeakerRecognizerRequest; +import com.tencent.asrspeaker.SpeakerSentenceItem; +import com.tencent.core.ws.Credential; +import com.tencent.core.ws.SpeechClient; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.CloseStatus; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Component +@RequiredArgsConstructor +public class TencentRealtimeAsrChannel implements RealtimeAsrChannel { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String TARGET_WS_URL = "tencent-sdk://speaker-recognizer"; + private static final String MEDIA_TENCENT_APP_ID = "tencentAppId"; + private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId"; + private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey"; + private static final String STATE_CONNECTED = "tencentConnected"; + private static final String STATE_STARTED = "tencentStarted"; + private static final String STATE_RECOGNIZER = "tencentRecognizer"; + private static final String STATE_SPEECH_CLIENT = "tencentSpeechClient"; + private static final String STATE_STOP_REQUESTED = "tencentStopRequested"; + private static final String STATE_MEETING_COMPLETE_REQUESTED = "tencentMeetingCompleteRequested"; + private static final String STATE_FRONTEND_DETACHED = "tencentFrontendDetached"; + private static final String STATE_SPEAKER_CONTEXT_ID = "speakerContextId"; + private static final String STATE_VOICE_ID = "voiceId"; + private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames"; + private static final String STATE_MODEL_CODE = "modelCode"; + private static final String STATE_MEDIA_CONFIG = "mediaConfig"; + + private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; + private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService; + + @Override + public boolean supports(String provider) { + return ModelProviderEnum.TENCENT.getCode().equalsIgnoreCase(provider); + } + + @Override + public String resolveTargetWsUrl(AiModelVO model) { + return TARGET_WS_URL; + } + + @Override + public Map buildStartMessage(AiModelVO model, + String mode, + String language, + Integer useSpkId, + Boolean enablePunctuation, + Boolean enableItn, + Boolean enableTextRefine, + Boolean saveAudio, + List> hotwords) { + Map payload = new HashMap<>(); + payload.put("provider", ModelProviderEnum.TENCENT.getCode()); + payload.put("engine_model_type", model.getModelCode()); + payload.put("language", language); + + Map root = new HashMap<>(); + root.put("type", "start"); + root.put("payload", payload); + return root; + } + + @Override + public void connect(RealtimeAsrChannelContext context) throws Exception { + String connectionId = currentConnectionId(context); + if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) { + context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接"); + context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接")); + return; + } + context.getChannelState().put(STATE_CONNECTED, Boolean.TRUE); + context.getChannelState().put(STATE_STARTED, Boolean.FALSE); + context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.FALSE); + context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.FALSE); + context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.FALSE); + context.getChannelState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new java.util.ArrayList()); + context.getCallback().onChannelOpen(context.getMeetingId()); + } + + @Override + public void handleFrontendText(RealtimeAsrChannelContext context, String payload) { + if (looksLikeStartMessage(payload)) { + startRecognizerIfNecessary(context); + return; + } + if (looksLikeStopMessage(payload)) { + context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE); + stopRecognizer(context); + } + } + + @Override + public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) { + SpeakerRecognizer recognizer = getRecognizer(context); + if (payload == null || payload.length == 0) { + return; + } + if (recognizer == null) { + queuePendingAudioFrame(context, payload); + return; + } + try { + recognizer.write(payload); + } catch (Exception ex) { + handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 音频发送失败", ex); + } + } + + @Override + public void closeMeeting(RealtimeAsrChannelContext context) { + context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE); + context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.TRUE); + stopRecognizer(context); + } + + @Override + public boolean isOpen(RealtimeAsrChannelContext context) { + return !Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED)); + } + + @Override + public void onFrontendDetached(RealtimeAsrChannelContext context) { + context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.TRUE); + context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE); + stopRecognizer(context); + } + + static String buildFrontendTranscriptMessage(String sentenceKey, + String text, + boolean isFinal, + Integer sentenceId, + Long startTime, + Long endTime, + Integer speakerId) throws JsonProcessingException { + ObjectNode root = OBJECT_MAPPER.createObjectNode(); + root.put("type", isFinal ? "segment" : "partial"); + ObjectNode data = root.putObject("data"); + data.put("text", text); + data.put("is_final", isFinal); + if (sentenceId != null) { + data.put("sentence_id", sentenceId); + } + if (sentenceKey != null && !sentenceKey.isBlank()) { + data.put("sentence_key", sentenceKey); + } + if (startTime != null) { + data.put("start", startTime / 1000D); + } + if (endTime != null) { + data.put("end", endTime / 1000D); + } + if (speakerId != null) { + data.put("speaker_id", String.valueOf(speakerId)); + } + return OBJECT_MAPPER.writeValueAsString(root); + } + + private void startRecognizerIfNecessary(RealtimeAsrChannelContext context) { + synchronized (context.getChannelState()) { + if (Boolean.TRUE.equals(context.getChannelState().get(STATE_STARTED))) { + return; + } + try { + SpeechClient speechClient = createSpeechClient(); + SpeakerRecognizerRequest request = createRecognizerRequest(context); + SpeakerRecognizer recognizer = createRecognizer(context, speechClient, request); + context.getChannelState().put(STATE_SPEECH_CLIENT, speechClient); + context.getChannelState().put(STATE_RECOGNIZER, recognizer); + context.getChannelState().put(STATE_VOICE_ID, request.getVoiceId()); + recognizer.start(); + context.getChannelState().put(STATE_STARTED, Boolean.TRUE); + flushPendingAudioFrames(context, recognizer); + } catch (Exception ex) { + handleChannelFailure(context, "REALTIME_UPSTREAM_CONNECT_FAILED", "腾讯实时 ASR 启动失败", ex); + } + } + } + + protected SpeechClient createSpeechClient() { + return new SpeechClient(SpeakerConstant.DEFAULT_RT_REQ_URL); + } + + protected SpeakerRecognizer createRecognizer(RealtimeAsrChannelContext context, + SpeechClient speechClient, + SpeakerRecognizerRequest request) { + return new SpeakerRecognizer( + speechClient, + buildCredential(context), + request, + new TencentRecognitionListener(context) + ); + } + + private Credential buildCredential(RealtimeAsrChannelContext context) { + Map mediaConfig = getMediaConfig(context); + String appId = readConfigString(mediaConfig, MEDIA_TENCENT_APP_ID); + String secretId = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_ID); + String secretKey = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_KEY); + if (appId == null || secretId == null || secretKey == null) { + throw new RuntimeException("腾讯实时 ASR 会话缺少鉴权配置"); + } + return new Credential(appId, secretId, secretKey); + } + + SpeakerRecognizerRequest createRecognizerRequest(RealtimeAsrChannelContext context) { + SpeakerRecognizerRequest request = SpeakerRecognizerRequest.init(); + request.setEngineModelType(resolveEngineModelType(context)); + request.setVoiceFormat(SpeakerConstant.AUDIO_FORMAT_PCM); + request.setVoiceId(UUID.randomUUID().toString()); + //是否需要vad + request.setNeedVad(1); + //vad静默时间 + request.setVadSilenceTime(1000); +// 分句策略参数 0小1大 + request.setSentenceStrategy(0); + //是否进行阿拉伯数字智能转换 0否1智能 23:打开数学相关转化 + request.setConvertNumMode(1); + + request.setSpeakerDiarization(1); + //启动断点续传 + request.setEnableSpeakerContext(1); + String speakerContextId = resolveSpeakerContextId(context); + if (speakerContextId != null) { + request.setSpeakerContextId(speakerContextId); + } + return request; + } + + private String resolveEngineModelType(RealtimeAsrChannelContext context) { + Object modelCode = context.getChannelState().get(STATE_MODEL_CODE); + if (modelCode instanceof String value && !value.isBlank()) { + return value; + } + return "16k_zh"; + } + + private String resolveSpeakerContextId(RealtimeAsrChannelContext context) { + Object speakerContextId = context.getChannelState().get(STATE_SPEAKER_CONTEXT_ID); + if (speakerContextId instanceof String value && !value.isBlank()) { + return value; + } + var status = realtimeMeetingSessionStateService.getStatus(context.getMeetingId()); + if (status == null || status.getResumeConfig() == null) { + return null; + } + String value = status.getResumeConfig().getSpeakerContextId(); + if (value == null || value.isBlank()) { + return null; + } + context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, value); + return value; + } + + @SuppressWarnings("unchecked") + private Map getMediaConfig(RealtimeAsrChannelContext context) { + Object mediaConfig = context.getChannelState().get(STATE_MEDIA_CONFIG); + if (mediaConfig instanceof Map map) { + return (Map) map; + } + return Map.of(); + } + + private String readConfigString(Map mediaConfig, String key) { + Object value = mediaConfig.get(key); + if (value == null) { + return null; + } + String text = String.valueOf(value).trim(); + return text.isEmpty() ? null : text; + } + + private SpeakerRecognizer getRecognizer(RealtimeAsrChannelContext context) { + Object recognizer = context.getChannelState().get(STATE_RECOGNIZER); + return recognizer instanceof SpeakerRecognizer value ? value : null; + } + + private SpeechClient getSpeechClient(RealtimeAsrChannelContext context) { + Object speechClient = context.getChannelState().get(STATE_SPEECH_CLIENT); + return speechClient instanceof SpeechClient value ? value : null; + } + + private void stopRecognizer(RealtimeAsrChannelContext context) { + SpeakerRecognizer recognizer = getRecognizer(context); + if (recognizer == null) { + shutdownSdkResources(context); + return; + } + try { + recognizer.stop(); + } catch (Exception ex) { + log.warn("Tencent realtime ASR stop failed, meetingId={}, sessionId={}", + context.getMeetingId(), currentConnectionId(context), ex); + shutdownSdkResources(context); + } + } + + private void shutdownSdkResources(RealtimeAsrChannelContext context) { + SpeakerRecognizer recognizer = getRecognizer(context); + if (recognizer != null) { + try { + recognizer.close(); + } catch (Exception ignored) { + // ignore + } + } + SpeechClient speechClient = getSpeechClient(context); + if (speechClient != null) { + try { + speechClient.shutdown(); + } catch (Exception ignored) { + // ignore + } + } + context.getChannelState().remove(STATE_RECOGNIZER); + context.getChannelState().remove(STATE_SPEECH_CLIENT); + context.getChannelState().put(STATE_STARTED, Boolean.FALSE); + } + + private void forwardResponse(RealtimeAsrChannelContext context, + SpeakerRecognitionResponse response, + boolean forceFinal) { + if (response == null || response.getSentences() == null || response.getSentences().getSentenceList() == null) { + return; + } + try { + rememberSpeakerContext(context, response); + String cachePayload = buildCachePayload(response, forceFinal); + realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), cachePayload); + for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) { + if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) { + continue; + } + boolean isFinal = forceFinal || item.getSentenceType() == 1; + context.getCallback().sendFrontendText( + context.getMeetingId(), + buildFrontendTranscriptMessage( + buildSentenceKey(context, item.getSentenceId()), + item.getSentence().trim(), + isFinal, + item.getSentenceId(), + item.getStartTime(), + item.getEndTime(), + item.getSpeakerId() + ) + ); + } + } catch (Exception ex) { + handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 结果转发失败", ex); + } + } + + private void rememberSpeakerContext(RealtimeAsrChannelContext context, SpeakerRecognitionResponse response) { + if (response == null || response.getSpeakerContextId() == null || response.getSpeakerContextId().isBlank()) { + return; + } + String speakerContextId = response.getSpeakerContextId().trim(); + context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, speakerContextId); + realtimeMeetingSessionStateService.rememberSpeakerContext(context.getMeetingId(), speakerContextId); + } + + private String buildSentenceKey(RealtimeAsrChannelContext context, Integer sentenceId) { + Object voiceId = context.getChannelState().get(STATE_VOICE_ID); + if (!(voiceId instanceof String value) || value.isBlank() || sentenceId == null) { + return null; + } + return value + "-" + sentenceId; + } + + @SuppressWarnings("unchecked") + private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) { + Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES); + if (frames instanceof List list) { + ((List) list).add(payload.clone()); + return; + } + List next = new java.util.ArrayList<>(); + next.add(payload.clone()); + context.getChannelState().put(STATE_PENDING_AUDIO_FRAMES, next); + } + + @SuppressWarnings("unchecked") + private void flushPendingAudioFrames(RealtimeAsrChannelContext context, SpeakerRecognizer recognizer) { + Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES); + if (!(frames instanceof List list) || list.isEmpty()) { + return; + } + List pendingFrames = (List) list; + for (byte[] frame : pendingFrames) { + if (frame != null && frame.length > 0) { + recognizer.write(frame); + } + } + pendingFrames.clear(); + } + + private String buildCachePayload(SpeakerRecognitionResponse response, boolean forceFinal) throws JsonProcessingException { + ObjectNode root = OBJECT_MAPPER.createObjectNode(); + root.put("type", forceFinal ? "end" : "sentences"); + ArrayNode sentences = root.putArray("sentences"); + for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) { + if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) { + continue; + } + ObjectNode sentenceNode = sentences.addObject(); + sentenceNode.put("sentence", item.getSentence().trim()); + sentenceNode.put("sentence_type", forceFinal ? 1 : item.getSentenceType()); + sentenceNode.put("sentence_id", item.getSentenceId()); + sentenceNode.put("speaker_id", String.valueOf(item.getSpeakerId())); + sentenceNode.put("start_time", item.getStartTime()); + sentenceNode.put("end_time", item.getEndTime()); + } + return OBJECT_MAPPER.writeValueAsString(root); + } + + private void handleChannelFailure(RealtimeAsrChannelContext context, String code, String message, Exception ex) { + log.error("Tencent realtime ASR channel failed, meetingId={}, sessionId={}", + context.getMeetingId(), currentConnectionId(context), ex); + shutdownSdkResources(context); + context.getCallback().sendFrontendError(context.getMeetingId(), code, message); + CompletableFuture.delayedExecutor(200, TimeUnit.MILLISECONDS).execute( + () -> context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR) + ); + } + + private String currentConnectionId(RealtimeAsrChannelContext context) { + return context.getRawSession() == null ? null : context.getRawSession().getId(); + } + + private static boolean looksLikeStartMessage(String payload) { + if (payload == null || payload.isBlank()) { + return false; + } + String normalized = payload.replaceAll("\\s+", ""); + return normalized.contains("\"type\":\"start\""); + } + + private static boolean looksLikeStopMessage(String payload) { + if (payload == null || payload.isBlank()) { + return false; + } + String normalized = payload.replaceAll("\\s+", ""); + return normalized.contains("\"type\":\"stop\""); + } + + private final class TencentRecognitionListener extends SpeakerRecognitionListener { + private final RealtimeAsrChannelContext context; + + private TencentRecognitionListener(RealtimeAsrChannelContext context) { + this.context = context; + } + + @Override + public void onRecognitionStart(SpeakerRecognitionResponse response) { + rememberSpeakerContext(context, response); + log.info("Tencent realtime ASR started, meetingId={}, sessionId={}", + context.getMeetingId(), currentConnectionId(context)); + } + + @Override + public void onRecognitionSentences(SpeakerRecognitionResponse response) { + forwardResponse(context, response, false); + } + + @Override + public void onSentenceEnd(SpeakerRecognitionResponse response) { + forwardResponse(context, response, true); + shutdownSdkResources(context); + if (Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED))) { + context.getCallback().removeMeetingSession(context.getMeetingId()); + } + } + + @Override + public void onFail(SpeakerRecognitionResponse response, Exception error) { + handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 识别失败", error); + } + } +} diff --git a/backend/src/main/java/com/imeeting/support/redis/RealtimeMeetingTranscriptCache.java b/backend/src/main/java/com/imeeting/support/redis/RealtimeMeetingTranscriptCache.java new file mode 100644 index 0000000..f432915 --- /dev/null +++ b/backend/src/main/java/com/imeeting/support/redis/RealtimeMeetingTranscriptCache.java @@ -0,0 +1,39 @@ +package com.imeeting.support.redis; + +import com.imeeting.common.RedisKeys; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState; +import com.imeeting.support.RedisSupport; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.time.Duration; + +@Component +@RequiredArgsConstructor +public class RealtimeMeetingTranscriptCache { + + private static final Duration CACHE_TTL = Duration.ofHours(12); + + private final RedisSupport redisSupport; + + public RealtimeMeetingTranscriptCacheState getState(Long meetingId) { + if (meetingId == null) { + return null; + } + return redisSupport.getJsonQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId), RealtimeMeetingTranscriptCacheState.class); + } + + public void saveState(RealtimeMeetingTranscriptCacheState state) { + if (state == null || state.getMeetingId() == null) { + return; + } + redisSupport.setJson(RedisKeys.realtimeMeetingTranscriptCacheKey(state.getMeetingId()), state, CACHE_TTL); + } + + public void clear(Long meetingId) { + if (meetingId == null) { + return; + } + redisSupport.deleteQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId)); + } +} diff --git a/backend/src/main/java/com/imeeting/websocket/RealtimeMeetingProxyWebSocketHandler.java b/backend/src/main/java/com/imeeting/websocket/RealtimeMeetingProxyWebSocketHandler.java index 0ffee12..d19ac61 100644 --- a/backend/src/main/java/com/imeeting/websocket/RealtimeMeetingProxyWebSocketHandler.java +++ b/backend/src/main/java/com/imeeting/websocket/RealtimeMeetingProxyWebSocketHandler.java @@ -1,173 +1,133 @@ package com.imeeting.websocket; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem; import com.imeeting.dto.biz.RealtimeSocketSessionData; import com.imeeting.service.biz.RealtimeMeetingSessionStateService; import com.imeeting.service.biz.RealtimeMeetingSocketSessionService; +import com.imeeting.service.realtime.RealtimeAsrChannel; +import com.imeeting.service.realtime.RealtimeAsrChannelCallback; +import com.imeeting.service.realtime.RealtimeAsrChannelContext; +import com.imeeting.service.realtime.RealtimeAsrChannelFactory; import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService; +import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService; +import com.imeeting.service.realtime.impl.LocalRealtimeAsrChannel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.web.socket.BinaryMessage; import org.springframework.web.socket.CloseStatus; -import org.springframework.web.socket.PingMessage; import org.springframework.web.socket.PongMessage; import org.springframework.web.socket.TextMessage; import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.AbstractWebSocketHandler; import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator; -import java.io.ByteArrayOutputStream; import java.net.URI; import java.net.URLDecoder; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; @Slf4j @Component @RequiredArgsConstructor public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandler { - private static final String ATTR_FRONTEND_SESSION = "frontendSession"; - private static final String ATTR_UPSTREAM_SOCKET = "upstreamSocket"; private static final String ATTR_MEETING_ID = "meetingId"; private static final String ATTR_TARGET_WS_URL = "targetWsUrl"; + private static final String ATTR_PROVIDER = "provider"; private static final String ATTR_FRONTEND_TEXT_COUNT = "frontendTextCount"; private static final String ATTR_FRONTEND_BINARY_COUNT = "frontendBinaryCount"; - private static final String ATTR_UPSTREAM_SEND_CHAIN = "upstreamSendChain"; - private static final String ATTR_START_MESSAGE_SENT = "startMessageSent"; - private static final String ATTR_PENDING_AUDIO_FRAMES = "pendingAudioFrames"; - private static final CompletableFuture COMPLETED = CompletableFuture.completedFuture(null); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final RealtimeMeetingSocketSessionService realtimeMeetingSocketSessionService; private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService; + private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService; + private final RealtimeAsrChannelFactory realtimeAsrChannelFactory; + private final ConcurrentMap meetingSessions = new ConcurrentHashMap<>(); + private final ConcurrentMap meetingLocks = new ConcurrentHashMap<>(); @Override public void afterConnectionEstablished(WebSocketSession session) throws Exception { String sessionToken = extractQueryParam(session.getUri(), "sessionToken"); RealtimeSocketSessionData sessionData = realtimeMeetingSocketSessionService.getSessionData(sessionToken); if (sessionData == null) { - log.warn("Realtime websocket rejected: invalid session token, sessionId={}", session.getId()); + log.warn("实时会议 websocket 拒绝连接：会话令牌无效，sessionId={}", session.getId()); session.close(CloseStatus.POLICY_VIOLATION.withReason("实时 Socket 会话无效")); return; } ConcurrentWebSocketSessionDecorator frontendSession = new ConcurrentWebSocketSessionDecorator(session, (int) Duration.ofSeconds(15).toMillis(), 1024 * 1024); - session.getAttributes().put(ATTR_FRONTEND_SESSION, frontendSession); session.getAttributes().put(ATTR_MEETING_ID, sessionData.getMeetingId()); session.getAttributes().put(ATTR_TARGET_WS_URL, sessionData.getTargetWsUrl()); + session.getAttributes().put(ATTR_PROVIDER, sessionData.getProvider()); session.getAttributes().put(ATTR_FRONTEND_TEXT_COUNT, new AtomicInteger()); session.getAttributes().put(ATTR_FRONTEND_BINARY_COUNT, new AtomicInteger()); - session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED); - session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.FALSE); - session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList()); realtimeMeetingAudioStorageService.openSession(sessionData.getMeetingId(), session.getId()); - log.info("Realtime websocket accepted: meetingId={}, sessionId={}, upstream={}", - sessionData.getMeetingId(), session.getId(), sessionData.getTargetWsUrl()); + log.info("实时会议 websocket 已接入：meetingId={}, sessionId={}, provider={}, upstream={}", + sessionData.getMeetingId(), session.getId(), sessionData.getProvider(), sessionData.getTargetWsUrl()); - java.net.http.WebSocket upstreamSocket; - try { - upstreamSocket = java.net.http.HttpClient.newHttpClient() - .newWebSocketBuilder() - .buildAsync(URI.create(sessionData.getTargetWsUrl()), - new UpstreamListener( - frontendSession, - session, - sessionData.getMeetingId(), - sessionData.getTargetWsUrl(), - realtimeMeetingSessionStateService - )) - .get(); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - log.error("Realtime websocket upstream connect interrupted: meetingId={}, sessionId={}", - sessionData.getMeetingId(), session.getId(), ex); - sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接第三方识别服务时被中断"); - realtimeMeetingAudioStorageService.closeSession(session.getId()); - frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断")); - return; - } catch (ExecutionException | CompletionException ex) { - log.warn("Failed to connect upstream websocket, meetingId={}, target={}", sessionData.getMeetingId(), sessionData.getTargetWsUrl(), ex); - sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接第三方识别服务失败，请检查模型 WebSocket 配置或服务状态"); - realtimeMeetingAudioStorageService.closeSession(session.getId()); - frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败")); - return; - } - - session.getAttributes().put(ATTR_UPSTREAM_SOCKET, upstreamSocket); + attachFrontendSession(sessionData, session, frontendSession); } @Override protected void handleTextMessage(WebSocketSession session, TextMessage message) { - java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); - if (upstreamSocket == null) { - log.warn("Frontend text ignored because upstream socket is unavailable, meetingId={}, sessionId={}", + MeetingChannelSession meetingSession = getMeetingSession(session); + if (meetingSession == null || !meetingSession.isChannelOpen()) { + log.warn("前端文本消息已忽略：上游 ASR 连接不可用，meetingId={}, sessionId={}", session.getAttributes().get(ATTR_MEETING_ID), session.getId()); return; } int count = nextCount(session, ATTR_FRONTEND_TEXT_COUNT); - log.info("Frontend text -> upstream: meetingId={}, sessionId={}, count={}, payload={}", - session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, summarizeText(message.getPayload())); - sendUpstreamOrdered(session, () -> upstreamSocket.sendText(message.getPayload(), true), "text"); - if (looksLikeStartMessage(message.getPayload())) { - session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.TRUE); - flushPendingAudioFrames(session, upstreamSocket); - } + String payload = message.getPayload(); + log.info("前端文本 -> ASR 渠道：meetingId={}, sessionId={}, provider={}, count={}, payload={}", + session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, summarizeText(payload)); + meetingSession.channel.handleFrontendText(meetingSession.context, payload); } @Override protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); - if (upstreamSocket == null) { - log.warn("Frontend binary ignored because upstream socket is unavailable, meetingId={}, sessionId={}", + MeetingChannelSession meetingSession = getMeetingSession(session); + if (meetingSession == null || !meetingSession.isChannelOpen()) { + log.warn("前端音频帧已忽略：上游 ASR 连接不可用，meetingId={}, sessionId={}", session.getAttributes().get(ATTR_MEETING_ID), session.getId()); return; } int count = nextCount(session, ATTR_FRONTEND_BINARY_COUNT); int bytes = message.getPayloadLength(); if (shouldLogBinaryFrame(count)) { - log.info("Frontend binary -> upstream: meetingId={}, sessionId={}, count={}, bytes={}", - session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes); + log.info("前端音频帧 -> ASR 渠道：meetingId={}, sessionId={}, provider={}, count={}, bytes={}", + session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, bytes); } byte[] payload = toByteArray(message.getPayload()); realtimeMeetingAudioStorageService.append(session.getId(), payload); - if (!Boolean.TRUE.equals(session.getAttributes().get(ATTR_START_MESSAGE_SENT))) { - queuePendingAudioFrame(session, payload); - if (shouldLogBinaryFrame(count)) { - log.warn("Frontend binary queued before start message: meetingId={}, sessionId={}, count={}, bytes={}", - session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes); - } - return; - } - sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary"); + meetingSession.channel.handleFrontendBinary(meetingSession.context, payload); } @Override protected void handlePongMessage(WebSocketSession session, PongMessage message) { - java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); - if (upstreamSocket == null) { + if (getMeetingSession(session) == null) { return; } - sendUpstreamOrdered(session, () -> upstreamSocket.sendPong(copyBuffer(message.getPayload())), "pong"); + log.debug("前端 pong 已在本地忽略：meetingId={}, sessionId={}, bytes={}", + session.getAttributes().get(ATTR_MEETING_ID), session.getId(), message.getPayloadLength()); } @Override public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception { - log.error("Realtime websocket transport error: meetingId={}, sessionId={}, upstream={}", + log.error("实时会议 websocket 传输异常：meetingId={}, sessionId={}, upstream={}", session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_TARGET_WS_URL), exception); - closeUpstreamSocket(session, CloseStatus.SERVER_ERROR); + detachFrontend(session); + realtimeMeetingAudioStorageService.closeSession(session.getId()); if (session.isOpen()) { session.close(CloseStatus.SERVER_ERROR); } @@ -175,32 +135,160 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl @Override public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { - log.info("Realtime websocket closed: meetingId={}, sessionId={}, code={}, reason={}", + log.info("实时会议 websocket 已关闭：meetingId={}, sessionId={}, code={}, reason={}", session.getAttributes().get(ATTR_MEETING_ID), session.getId(), status.getCode(), status.getReason()); Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID); if (meetingIdValue instanceof Long meetingId) { + detachFrontend(meetingId, session.getId()); realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, session.getId()); } realtimeMeetingAudioStorageService.closeSession(session.getId()); - closeUpstreamSocket(session, status); } - private java.net.http.WebSocket getUpstreamSocket(WebSocketSession session) { - Object socket = session.getAttributes().get(ATTR_UPSTREAM_SOCKET); - if (socket instanceof java.net.http.WebSocket webSocket) { - return webSocket; + public void closeMeetingSession(Long meetingId) { + if (meetingId == null) { + return; + } + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null || meetingSession.channel == null) { + return; + } + meetingSession.channel.closeMeeting(meetingSession.context); + } + + private void attachFrontendSession(RealtimeSocketSessionData sessionData, + WebSocketSession rawSession, + ConcurrentWebSocketSessionDecorator frontendSession) throws Exception { + Long meetingId = sessionData.getMeetingId(); + MeetingChannelSession meetingSession; + boolean reused = false; + synchronized (lockForMeeting(meetingId)) { + meetingSession = meetingSessions.get(meetingId); + if (meetingSession != null && meetingSession.isChannelOpen()) { + String previousSessionId = meetingSession.context.getRawSession() == null + ? null + : meetingSession.context.getRawSession().getId(); + meetingSession.clearFrontendIfClosed(); + if (previousSessionId != null && !meetingSession.hasOpenFrontend()) { + realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, previousSessionId); } - return null; + if (meetingSession.hasOpenFrontend()) { + sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃前端连接"); + frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("已存在活跃的前端连接")); + realtimeMeetingAudioStorageService.closeSession(rawSession.getId()); + return; + } + if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) { + sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_REJECTED", "当前状态下无法继续会议"); + frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("当前状态下无法继续会议")); + realtimeMeetingAudioStorageService.closeSession(rawSession.getId()); + return; + } + meetingSession.bindFrontend(rawSession, frontendSession); + reused = true; + } else { + RealtimeAsrChannel channel = realtimeAsrChannelFactory.getRequired(sessionData.getProvider()); + RealtimeAsrChannelContext context = new RealtimeAsrChannelContext(); + context.setMeetingId(meetingId); + context.setProvider(realtimeAsrChannelFactory.normalizeProvider(sessionData.getProvider())); + context.setTargetWsUrl(sessionData.getTargetWsUrl()); + context.setCallback(new HandlerChannelCallback()); + context.bindFrontendSession(rawSession, frontendSession); + context.getChannelState().put("modelCode", sessionData.getModelCode()); + context.getChannelState().put("mediaConfig", sessionData.getMediaConfig()); + meetingSession = new MeetingChannelSession(meetingId, channel, context); + meetingSessions.put(meetingId, meetingSession); + } } - private void closeUpstreamSocket(WebSocketSession session, CloseStatus status) { - java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session); - if (upstreamSocket != null) { - upstreamSocket.sendClose(status.getCode(), status.getReason() == null ? "" : status.getReason()); - session.getAttributes().remove(ATTR_UPSTREAM_SOCKET); + if (reused) { + sendProxyReady(frontendSession); + replayCachedMessages(meetingId, frontendSession); + return; + } + + try { + meetingSession.channel.connect(meetingSession.context); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + removeMeetingSession(meetingId, meetingSession); + log.error("连接上游 ASR websocket 时被中断：meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); + sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接上游 ASR 服务时被中断"); + realtimeMeetingAudioStorageService.closeSession(rawSession.getId()); + frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断")); + } catch (Exception ex) { + removeMeetingSession(meetingId, meetingSession); + log.warn("连接上游 ASR websocket 失败：meetingId={}, provider={}, target={}", + meetingId, sessionData.getProvider(), sessionData.getTargetWsUrl(), ex); + sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接上游 ASR 服务失败"); + realtimeMeetingAudioStorageService.closeSession(rawSession.getId()); + frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败")); + } + } + + private void replayCachedMessages(Long meetingId, ConcurrentWebSocketSessionDecorator frontendSession) { + try { + if (!frontendSession.isOpen()) { + return; + } + for (RealtimeMeetingTranscriptCacheItem item : realtimeMeetingTranscriptCacheService.listOrderedItems(meetingId)) { + frontendSession.sendMessage(new TextMessage(LocalRealtimeAsrChannel.buildFrontendTranscriptMessage(item))); + } + } catch (Exception ex) { + log.warn("回放缓存转写消息失败：meetingId={}", meetingId, ex); + } + } + + private void sendProxyReady(ConcurrentWebSocketSessionDecorator frontendSession) throws Exception { + if (frontendSession.isOpen()) { + frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}")); + } + } + + private void detachFrontend(WebSocketSession session) { + Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID); + if (meetingIdValue instanceof Long meetingId) { + detachFrontend(meetingId, session.getId()); + } + } + + private void detachFrontend(Long meetingId, String sessionId) { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { + return; + } + synchronized (lockForMeeting(meetingId)) { + if (meetingSession.context.getRawSession() != null && meetingSession.context.getRawSession().getId().equals(sessionId)) { + meetingSession.channel.onFrontendDetached(meetingSession.context); + } + meetingSession.detachFrontend(sessionId); + } + } + + private MeetingChannelSession getMeetingSession(WebSocketSession session) { + Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID); + if (!(meetingIdValue instanceof Long meetingId)) { + return null; + } + return meetingSessions.get(meetingId); + } + + void removeMeetingSession(Long meetingId) { + synchronized (lockForMeeting(meetingId)) { + meetingSessions.remove(meetingId); + } + } + + void removeMeetingSession(Long meetingId, MeetingChannelSession meetingSession) { + synchronized (lockForMeeting(meetingId)) { + meetingSessions.remove(meetingId, meetingSession); } } + private Object lockForMeeting(Long meetingId) { + return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object()); + } + private String extractQueryParam(URI uri, String key) { if (uri == null || uri.getQuery() == null || uri.getQuery().isBlank()) { return null; @@ -213,13 +301,6 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl .orElse(null); } - private ByteBuffer copyBuffer(ByteBuffer source) { - ByteBuffer duplicate = source.asReadOnlyBuffer(); - byte[] bytes = new byte[duplicate.remaining()]; - duplicate.get(bytes); - return ByteBuffer.wrap(bytes); - } - private byte[] toByteArray(ByteBuffer source) { ByteBuffer duplicate = source.asReadOnlyBuffer(); byte[] bytes = new byte[duplicate.remaining()]; @@ -235,21 +316,18 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl return 0; } - @SuppressWarnings("unchecked") - private void sendUpstreamOrdered(WebSocketSession session, Supplier> sendAction, String messageType) { - synchronized (session) { - CompletableFuture chain = (CompletableFuture) session.getAttributes() - .getOrDefault(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED); - CompletableFuture nextChain = chain - .exceptionally(ex -> null) - .thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null)); - nextChain = nextChain.whenComplete((ignored, ex) -> { - if (ex != null) { - log.error("Ordered upstream send failed: meetingId={}, sessionId={}, type={}", - session.getAttributes().get(ATTR_MEETING_ID), session.getId(), messageType, ex); - } - }); - session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, nextChain); + private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) { + try { + if (!frontendSession.isOpen()) { + return; + } + Map payload = new HashMap<>(); + payload.put("type", "error"); + payload.put("code", code); + payload.put("message", message); + frontendSession.sendMessage(new TextMessage(OBJECT_MAPPER.writeValueAsString(payload))); + } catch (Exception ex) { + log.warn("向前端发送实时代理错误消息失败：code={}", code, ex); } } @@ -268,237 +346,113 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl return normalized.substring(0, 240) + "..."; } - private boolean looksLikeStartMessage(String payload) { - if (payload == null || payload.isBlank()) { - return false; - } - String normalized = payload.replaceAll("\\s+", ""); - return normalized.contains("\"type\":\"start\""); - } - - private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) { - try { - if (!frontendSession.isOpen()) { - return; - } - Map payload = new HashMap<>(); - payload.put("type", "error"); - payload.put("code", code); - payload.put("message", message); - frontendSession.sendMessage(new TextMessage(new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(payload))); - } catch (Exception ex) { - log.warn("Failed to send realtime proxy error to frontend: code={}", code, ex); - } - } - - @SuppressWarnings("unchecked") - private void queuePendingAudioFrame(WebSocketSession session, byte[] payload) { - synchronized (session) { - List pendingFrames = (List) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES); - if (pendingFrames == null) { - pendingFrames = new ArrayList<>(); - session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, pendingFrames); - } - pendingFrames.add(payload); - } - } - - @SuppressWarnings("unchecked") - private void flushPendingAudioFrames(WebSocketSession session, java.net.http.WebSocket upstreamSocket) { - List pendingFrames; - synchronized (session) { - pendingFrames = (List) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES); - if (pendingFrames == null || pendingFrames.isEmpty()) { - return; - } - session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList()); - } - log.info("Flushing queued audio frames after start message: meetingId={}, sessionId={}, frameCount={}", - session.getAttributes().get(ATTR_MEETING_ID), session.getId(), pendingFrames.size()); - for (byte[] frame : pendingFrames) { - sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush"); - } - } - - private static final class UpstreamListener implements java.net.http.WebSocket.Listener { - private final ConcurrentWebSocketSessionDecorator frontendSession; - private final WebSocketSession rawSession; + static final class MeetingChannelSession { private final Long meetingId; - private final String targetWsUrl; - private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService; - private final StringBuilder textBuffer = new StringBuilder(); - private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream(); - private final AtomicInteger upstreamTextCount = new AtomicInteger(); - private final AtomicInteger upstreamBinaryCount = new AtomicInteger(); + private final RealtimeAsrChannel channel; + private final RealtimeAsrChannelContext context; - private UpstreamListener(ConcurrentWebSocketSessionDecorator frontendSession, WebSocketSession rawSession, - Long meetingId, String targetWsUrl, - RealtimeMeetingSessionStateService realtimeMeetingSessionStateService) { - this.frontendSession = frontendSession; - this.rawSession = rawSession; + private MeetingChannelSession(Long meetingId, RealtimeAsrChannel channel, RealtimeAsrChannelContext context) { this.meetingId = meetingId; - this.targetWsUrl = targetWsUrl; - this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService; + this.channel = channel; + this.context = context; + } + + private void bindFrontend(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) { + context.bindFrontendSession(rawSession, frontendSession); } + private void detachFrontend(String sessionId) { + if (context.getRawSession() != null && context.getRawSession().getId().equals(sessionId)) { + context.bindFrontendSession(null, null); + } + } + + private void clearFrontendIfClosed() { + if (context.getRawSession() != null && !context.getRawSession().isOpen()) { + context.bindFrontendSession(null, null); + } + } + + private boolean hasOpenFrontend() { + return context.getFrontendSession() != null + && context.getFrontendSession().isOpen() + && context.getRawSession() != null + && context.getRawSession().isOpen(); + } + + private boolean isChannelOpen() { + return channel != null && channel.isOpen(context); + } + } + + private final class HandlerChannelCallback implements RealtimeAsrChannelCallback { @Override - public void onOpen(java.net.http.WebSocket webSocket) { - log.info("Upstream websocket opened: meetingId={}, sessionId={}, upstream={}", - meetingId, rawSession.getId(), targetWsUrl); - if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) { - sendFrontendError("REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃实时连接，请先关闭旧连接后再继续"); - webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "Active realtime connection already exists"); - closeFrontend(CloseStatus.POLICY_VIOLATION.withReason("已存在活动的实时连接")); + public void onChannelOpen(Long meetingId) throws Exception { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { return; } - try { - if (frontendSession.isOpen()) { - frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}")); - } - } catch (Exception ex) { - log.error("Failed to notify frontend that upstream websocket is ready: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); - closeFrontend(CloseStatus.SERVER_ERROR); + ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession(); + if (frontendSession != null && frontendSession.isOpen()) { + sendProxyReady(frontendSession); + } + } + + @Override + public void sendFrontendText(Long meetingId, String payload) throws Exception { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { return; } - webSocket.request(1); + ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession(); + if (frontendSession != null && frontendSession.isOpen()) { + frontendSession.sendMessage(new TextMessage(payload)); + } } @Override - public java.util.concurrent.CompletionStage onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) { - textBuffer.append(data); - if (last) { - int count = upstreamTextCount.incrementAndGet(); - try { - if (frontendSession.isOpen()) { - frontendSession.sendMessage(new TextMessage(textBuffer.toString())); - } - log.info("Upstream text -> frontend: meetingId={}, sessionId={}, count={}, payload={}", - meetingId, rawSession.getId(), count, summarizeText(textBuffer.toString())); - } catch (Exception ex) { - log.error("Failed to forward upstream text: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); - closeFrontend(CloseStatus.SERVER_ERROR); - } finally { - textBuffer.setLength(0); - } - } - webSocket.request(1); - return COMPLETED; + public void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { + return; + } + ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession(); + if (frontendSession != null && frontendSession.isOpen()) { + frontendSession.sendMessage(new BinaryMessage(payload)); + } } @Override - public java.util.concurrent.CompletionStage onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) { - byte[] chunk = new byte[data.remaining()]; - data.get(chunk); - binaryBuffer.writeBytes(chunk); - if (last) { - int count = upstreamBinaryCount.incrementAndGet(); - try { - if (frontendSession.isOpen()) { - frontendSession.sendMessage(new BinaryMessage(binaryBuffer.toByteArray())); - } - if (shouldLogBinaryFrame(count)) { - log.info("Upstream binary -> frontend: meetingId={}, sessionId={}, count={}, bytes={}", - meetingId, rawSession.getId(), count, binaryBuffer.size()); - } - } catch (Exception ex) { - log.error("Failed to forward upstream binary: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); - closeFrontend(CloseStatus.SERVER_ERROR); - } finally { - binaryBuffer.reset(); - } - } - webSocket.request(1); - return COMPLETED; + public void sendFrontendError(Long meetingId, String code, String message) { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { + return; + } + ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession(); + if (frontendSession != null) { + RealtimeMeetingProxyWebSocketHandler.this.sendFrontendError(frontendSession, code, message); + } } @Override - public java.util.concurrent.CompletionStage onPing(java.net.http.WebSocket webSocket, ByteBuffer message) { + public void removeMeetingSession(Long meetingId) { + RealtimeMeetingProxyWebSocketHandler.this.removeMeetingSession(meetingId); + } + + @Override + public void closeFrontend(Long meetingId, CloseStatus status) { + MeetingChannelSession meetingSession = meetingSessions.get(meetingId); + if (meetingSession == null) { + return; + } try { - if (frontendSession.isOpen()) { - frontendSession.sendMessage(new PingMessage(copyBuffer(message))); - } - log.info("Upstream ping -> frontend: meetingId={}, sessionId={}, bytes={}", - meetingId, rawSession.getId(), message.remaining()); - } catch (Exception ex) { - log.error("Failed to forward upstream ping: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); - closeFrontend(CloseStatus.SERVER_ERROR); - } - webSocket.request(1); - return COMPLETED; - } - - @Override - public java.util.concurrent.CompletionStage onPong(java.net.http.WebSocket webSocket, ByteBuffer message) { - try { - if (frontendSession.isOpen()) { - frontendSession.sendMessage(new PongMessage(copyBuffer(message))); - } - log.info("Upstream pong -> frontend: meetingId={}, sessionId={}, bytes={}", - meetingId, rawSession.getId(), message.remaining()); - } catch (Exception ex) { - log.error("Failed to forward upstream pong: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex); - closeFrontend(CloseStatus.SERVER_ERROR); - } - webSocket.request(1); - return COMPLETED; - } - - @Override - public java.util.concurrent.CompletionStage onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) { - log.info("Upstream websocket closed: meetingId={}, sessionId={}, code={}, reason={}", - meetingId, rawSession.getId(), statusCode, reason); - sendFrontendError("REALTIME_UPSTREAM_CLOSED", reason == null || reason.isBlank() ? "第三方识别服务已断开连接" : "第三方识别服务已断开: " + reason); - closeFrontend(new CloseStatus(statusCode, reason)); - return COMPLETED; - } - - @Override - public void onError(java.net.http.WebSocket webSocket, Throwable error) { - log.error("Upstream websocket error: meetingId={}, sessionId={}, upstream={}", - meetingId, rawSession.getId(), targetWsUrl, error); - sendFrontendError("REALTIME_UPSTREAM_ERROR", error == null || error.getMessage() == null || error.getMessage().isBlank() - ? "第三方识别服务连接异常" - : "第三方识别服务连接异常: " + error.getMessage()); - closeFrontend(CloseStatus.SERVER_ERROR); - } - - private void sendFrontendError(String code, String message) { - try { - if (!frontendSession.isOpen()) { - return; - } - frontendSession.sendMessage(new TextMessage("{\"type\":\"error\",\"code\":\"" + code + "\",\"message\":\"" + escapeJson(message) + "\"}")); - } catch (Exception ex) { - log.warn("Failed to send upstream error to frontend: meetingId={}, sessionId={}, code={}", meetingId, rawSession.getId(), code, ex); - } - } - - private String escapeJson(String value) { - if (value == null) { - return ""; - } - return value - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\r", "\\r") - .replace("\n", "\\n"); - } - - private void closeFrontend(CloseStatus status) { - try { - if (rawSession.isOpen()) { + WebSocketSession rawSession = meetingSession.context.getRawSession(); + if (rawSession != null && rawSession.isOpen()) { rawSession.close(status); } } catch (Exception ignored) { // ignore close failure } } - - private ByteBuffer copyBuffer(ByteBuffer source) { - ByteBuffer duplicate = source.asReadOnlyBuffer(); - byte[] bytes = new byte[duplicate.remaining()]; - duplicate.get(bytes); - return ByteBuffer.wrap(bytes); - } } } diff --git a/backend/src/main/proto/android/push.proto b/backend/src/main/proto/android/push.proto index fcc64f3..0d0b81b 100644 --- a/backend/src/main/proto/android/push.proto +++ b/backend/src/main/proto/android/push.proto @@ -11,8 +11,23 @@ option java_outer_classname = "PushProto"; // ========================= enum Platform { PLATFORM_UNKNOWN = 0; + + // Mobile ANDROID = 1; IOS = 2; + HARMONY_MOBILE = 3; + + // Desktop + WINDOWS = 10; + MACOS = 11; + LINUX = 12; + + // Linux发行版（可选） + KYLIN = 20; + UOS = 21; + + // Harmony PC + HARMONY_PC = 30; } // ========================= diff --git a/backend/src/test/java/com/imeeting/service/biz/impl/AiModelServiceImplTest.java b/backend/src/test/java/com/imeeting/service/biz/impl/AiModelServiceImplTest.java index 65a57ce..6fad4b0 100644 --- a/backend/src/test/java/com/imeeting/service/biz/impl/AiModelServiceImplTest.java +++ b/backend/src/test/java/com/imeeting/service/biz/impl/AiModelServiceImplTest.java @@ -349,6 +349,64 @@ class AiModelServiceImplTest { assertNull(captor.getValue().getApiKey()); } + @Test + void saveModelShouldRejectTencentAsrWithoutSecretKey() { + AiModelServiceImpl service = new AiModelServiceImpl( + objectMapper, + mock(AsrModelMapper.class), + mock(LlmModelMapper.class) + ); + + AiModelDTO dto = new AiModelDTO(); + dto.setModelType("ASR"); + dto.setModelName("tencent-asr"); + dto.setProvider("tencent"); + dto.setModelCode("16k_zh"); + dto.setIsDefault(0); + dto.setStatus(1); + dto.setMediaConfig(Map.of( + "tencentAppId", "app-id", + "tencentSecretId", "secret-id" + )); + + RuntimeException ex = assertThrows(RuntimeException.class, () -> service.saveModel(dto)); + assertEquals("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey", ex.getMessage()); + } + + @Test + void saveModelShouldPersistTencentAsrWithoutBaseUrl() { + AsrModelMapper asrModelMapper = mock(AsrModelMapper.class); + when(asrModelMapper.insert(any(AsrModel.class))).thenReturn(1); + + AiModelServiceImpl service = new AiModelServiceImpl( + objectMapper, + asrModelMapper, + mock(LlmModelMapper.class) + ); + + AiModelDTO dto = new AiModelDTO(); + dto.setModelType("ASR"); + dto.setModelName("tencent-asr"); + dto.setProvider("tencent"); + dto.setModelCode("16k_zh"); + dto.setIsDefault(0); + dto.setStatus(1); + dto.setMediaConfig(Map.of( + "tencentAppId", "app-id", + "tencentSecretId", "secret-id", + "tencentSecretKey", "secret-key" + )); + + service.saveModel(dto); + + ArgumentCaptor captor = ArgumentCaptor.forClass(AsrModel.class); + verify(asrModelMapper, times(1)).insert(captor.capture()); + assertEquals("tencent", captor.getValue().getProvider()); + assertEquals("16k_zh", captor.getValue().getModelCode()); + assertEquals("secret-key", captor.getValue().getMediaConfig().get("tencentSecretKey")); + assertNull(captor.getValue().getBaseUrl()); + } + private void captureRequest(HttpExchange exchange, AtomicReference requestPath, AtomicReference authorization, diff --git a/frontend/src/api/business/meeting.ts b/frontend/src/api/business/meeting.ts index 537dbd7..df798ec 100644 --- a/frontend/src/api/business/meeting.ts +++ b/frontend/src/api/business/meeting.ts @@ -9,6 +9,7 @@ export type SummaryDetailLevel = "DETAILED" | "STANDARD" | "BRIEF"; export interface MeetingCreateConfig { offlineEnabled: boolean; realtimeEnabled: boolean; + aiCatalogEnabled?: boolean; offlineAudioMaxSizeMb: number; chunkUploadEnabled?: boolean; chunkDurationSeconds?: number; @@ -35,6 +36,7 @@ export interface MeetingVO { summaryDetailLevel?: SummaryDetailLevel; summaryModelId: number; promptId?: number; + aiCatalogEnabled?: boolean; audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED"; audioSaveMessage?: string; accessPassword?: string; @@ -247,12 +249,6 @@ export const createRealtimeMeeting = (data: CreateRealtimeMeetingCommand) => { ); }; -export const appendRealtimeTranscripts = (meetingId: number, data: RealtimeTranscriptItemDTO[]) => { - return http.post<{ code: string; data: boolean; msg: string }>( - `/api/biz/meeting/${meetingId}/realtime/transcripts`, - data - ); -}; export const getRealtimeMeetingSessionStatus = (meetingId: number) => { return http.get<{ code: string; data: RealtimeMeetingSessionStatus; msg: string }>( diff --git a/frontend/src/pages/access/users/index.tsx b/frontend/src/pages/access/users/index.tsx index c929bef..887504f 100644 --- a/frontend/src/pages/access/users/index.tsx +++ b/frontend/src/pages/access/users/index.tsx @@ -138,6 +138,7 @@ export default function Users() { const [filterTenantId, setFilterTenantId] = useState(undefined); const [drawerOpen, setDrawerOpen] = useState(false); const [editing, setEditing] = useState(null); + const [roleSelectOpen, setRoleSelectOpen] = useState(false); const [form] = Form.useForm(); const [resetPasswordOpen, setResetPasswordOpen] = useState(false); const [resetPasswordLoading, setResetPasswordLoading] = useState(false); @@ -264,6 +265,7 @@ export default function Users() { const openCreate = () => { setEditing(null); + setRoleSelectOpen(false); form.resetFields(); form.setFieldsValue({ status: 1, @@ -277,6 +279,7 @@ export default function Users() { const openEdit = async (record: SysUser) => { setEditing(record); + setRoleSelectOpen(false); try { const detail = await getUserDetail(record.userId); const roleIds = await listUserRoles(record.userId); @@ -594,10 +597,13 @@ export default function Users() { ) : null} setRoleSelectOpen(false)} + optionFilterProp={isPlatformMode ? "searchText" : "label"}/> {!isPlatformMode && } diff --git a/frontend/src/pages/business/AiModels.tsx b/frontend/src/pages/business/AiModels.tsx index 1c2c180..6886584 100644 --- a/frontend/src/pages/business/AiModels.tsx +++ b/frontend/src/pages/business/AiModels.tsx @@ -70,6 +70,7 @@ const AiModels: React.FC = () => { const provider = Form.useWatch("provider", form); const isDefaultChecked = Form.useWatch("isDefaultChecked", form); const isLocalProvider = String(provider || "").toLowerCase() === "custom"; + const isTencentProvider = String(provider || "").toLowerCase() === "tencent"; const isPlatformAdmin = useMemo(() => { const profileStr = sessionStorage.getItem("userProfile"); @@ -133,11 +134,17 @@ const AiModels: React.FC = () => { setEditingId(record.id); const speakerModel = record.mediaConfig?.speakerModel; const svThreshold = record.mediaConfig?.svThreshold; + const tencentAppId = record.mediaConfig?.tencentAppId; + const tencentSecretId = record.mediaConfig?.tencentSecretId; + const tencentSecretKey = record.mediaConfig?.tencentSecretKey; form.setFieldsValue({ ...record, modelType: record.modelType, speakerModel, svThreshold, + tencentAppId, + tencentSecretId, + tencentSecretKey, isDefaultChecked: record.isDefault === 1, statusChecked: record.status === 1, }); @@ -262,6 +269,12 @@ const AiModels: React.FC = () => { speakerModel: values.speakerModel, svThreshold: values.svThreshold, } + : activeType === "ASR" && isTencentProvider + ? { + tencentAppId: values.tencentAppId, + tencentSecretId: values.tencentSecretId, + tencentSecretKey: values.tencentSecretKey, + } : undefined, temperature: values.temperature, topP: values.topP, @@ -524,16 +537,20 @@ const AiModels: React.FC = () => { - - - + {!isTencentProvider && ( + <> + + + - - - + + + + + )} {(activeType === "LLM" || isLocalProvider) && ( @@ -556,7 +573,10 @@ const AiModels: React.FC = () => { { - + {!isTencentProvider && ( + + )} @@ -609,6 +631,38 @@ const AiModels: React.FC = () => { )} + {activeType === "ASR" && isTencentProvider && ( + + + + + + + + + + + + + + + + + + )} + {activeType === "LLM" && ( <> diff --git a/frontend/src/pages/business/MeetingDetail.tsx b/frontend/src/pages/business/MeetingDetail.tsx index edcd925..63df3a1 100644 --- a/frontend/src/pages/business/MeetingDetail.tsx +++ b/frontend/src/pages/business/MeetingDetail.tsx @@ -1226,7 +1226,7 @@ const MeetingDetail: React.FC = () => { const [expandKeywords, setExpandKeywords] = useState(false); const [expandSummary, setExpandSummary] = useState(false); const [selectedKeywords, setSelectedKeywords] = useState([]); - const [workspaceTab, setWorkspaceTab] = useState('catalog'); + const [workspaceTab, setWorkspaceTab] = useState('transcript'); const [addingHotwords, setAddingHotwords] = useState(false); const [editingTranscriptId, setEditingTranscriptId] = useState(null); const [savingTranscriptId, setSavingTranscriptId] = useState(null); @@ -1402,15 +1402,16 @@ const MeetingDetail: React.FC = () => { return false; }, [meeting]); + const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false; const canRetrySummary = isOwner && transcripts.length > 0 && meeting?.status !== 1 && meeting?.status !== 2 && meeting?.latestSummaryAttemptStatus !== 3 - && meeting?.latestChapterAttemptStatus !== 3; + && (!aiCatalogEnabled || meeting?.latestChapterAttemptStatus !== 3); const canRetryTranscription = isOwner && meeting?.status === 4 && transcripts.length === 0 && !!meeting?.audioUrl; const canRetryFailedSummaryTask = isOwner && meeting?.latestSummaryAttemptStatus === 3 && meeting?.status !== 2; - const canRetryFailedChapterTask = isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2; + const canRetryFailedChapterTask = aiCatalogEnabled && isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2; const canRetrySchedule = isOwner && meeting?.status === 0 && (!generationProgress || generationProgress.percent <= 0) && !!generationProgress?.queuedAt && dayjs().diff(dayjs(generationProgress.queuedAt)) >= QUEUED_RETRY_THRESHOLD_MS; @@ -1433,14 +1434,14 @@ const MeetingDetail: React.FC = () => { }; const hasSummaryContent = Boolean(meeting?.summaryContent?.trim()); - const hasCatalogContent = catalogChapterLinks.length > 0; + const hasCatalogContent = aiCatalogEnabled && catalogChapterLinks.length > 0; const generationFailureNotice = useMemo(() => { if (!meeting || meeting.status !== 4) { return null; } const hasFallbackContent = hasSummaryContent || hasCatalogContent; - if (meeting.latestChapterAttemptStatus === 3) { + if (aiCatalogEnabled && meeting.latestChapterAttemptStatus === 3) { const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败'; return { title: hasFallbackContent ? '历史内容仍可查看' : '本次 AI 目录生成失败', @@ -1467,45 +1468,13 @@ const MeetingDetail: React.FC = () => { } return { - title: hasFallbackContent ? '历史内容仍可查看' : '会议处理异常', - description: hasFallbackContent - ? '最近一次处理未成功，当前展示的是最近一次成功生成的内容。你可以继续查看，或重新发起识别/总结。' - : '会议在处理中遇到问题，暂时没有可展示的总结内容。你可以重新发起识别或总结。', + title: '会议处理异常', + description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。', type: hasFallbackContent ? 'info' : 'warning', hasFallbackContent, scope: 'global', }; - if (meeting.latestChapterAttemptStatus === 3) { - const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败'; - return { - key: `chapter-${meeting.latestChapterAttemptTaskId ?? 'latest'}`, - title: '本次重新总结失败', - description: hasFallbackContent - ? `章节生成失败，当前展示的是上一次成功的摘要和 AI 目录。失败原因：${detail}` - : `章节生成失败，且当前没有可展示的历史摘要或 AI 目录。失败原因：${detail}`, - hasFallbackContent, - }; - } - - if (meeting.latestSummaryAttemptStatus === 3) { - const detail = meeting.latestSummaryAttemptErrorMsg || '总结生成失败'; - return { - key: `summary-${meeting.latestSummaryAttemptTaskId ?? 'latest'}`, - title: '本次重新总结失败', - description: hasFallbackContent - ? `总结生成失败，当前展示的是上一次成功的摘要和 AI 目录。失败原因：${detail}` - : `总结生成失败，且当前没有可展示的历史摘要或 AI 目录。失败原因：${detail}`, - hasFallbackContent, - }; - } - - return { - key: 'general-failure', - title: '会议处理异常', - description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。', - hasFallbackContent, - }; - }, [hasCatalogContent, hasSummaryContent, meeting]); + }, [aiCatalogEnabled, hasCatalogContent, hasSummaryContent, meeting]); const summaryPanelNotice = useMemo(() => { if (!meeting || !hasSummaryContent) { return null; @@ -1522,7 +1491,7 @@ const MeetingDetail: React.FC = () => { return null; }, [generationFailureNotice, hasSummaryContent, meeting]); const catalogPanelNotice = useMemo(() => { - if (!generationFailureNotice || generationFailureNotice.scope !== 'catalog') { + if (!aiCatalogEnabled || !generationFailureNotice || generationFailureNotice.scope !== 'catalog') { return null; } if (generationFailureNotice.hasFallbackContent && hasCatalogContent) { @@ -1535,7 +1504,7 @@ const MeetingDetail: React.FC = () => { }; } return generationFailureNotice; - }, [generationFailureNotice, hasCatalogContent]); + }, [aiCatalogEnabled, generationFailureNotice, hasCatalogContent]); const emptyTranscriptFailureNotice = useMemo(() => { if (!meeting || meeting.status !== 4 || transcripts.length > 0) { return null; @@ -1557,12 +1526,18 @@ const MeetingDetail: React.FC = () => { } }, [meeting?.id, meeting?.status]); + useEffect(() => { + if (!aiCatalogEnabled && workspaceTab === 'catalog') { + setWorkspaceTab('transcript'); + } + }, [aiCatalogEnabled, workspaceTab]); + useEffect(() => { const attemptKey = String(meeting?.latestChapterAttemptTaskId ?? meeting?.latestSummaryAttemptTaskId ?? ''); if (meeting?.status !== 2 || !attemptKey) { return; } - if ((generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) { + if (!aiCatalogEnabled || (generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) { return; } if (autoOpenedCatalogAttemptRef.current === attemptKey) { @@ -1571,6 +1546,7 @@ const MeetingDetail: React.FC = () => { autoOpenedCatalogAttemptRef.current = attemptKey; setWorkspaceTab('catalog'); }, [ + aiCatalogEnabled, catalogChapterLinks.length, generationProgress?.percent, meeting?.latestChapterAttemptTaskId, @@ -2766,13 +2742,15 @@ const MeetingDetail: React.FC = () => { )}
- + {aiCatalogEnabled && ( + + )}
- {workspaceTab === 'catalog' ? ( + {aiCatalogEnabled && workspaceTab === 'catalog' ? (
{catalogPanelNotice && ( splitDisplayItems(meeting?.tags), [meeting?.tags]); const keywords = useMemo(() => analysis.keywords || [], [analysis.keywords]); const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]); + const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false; const statusMeta = STATUS_META[meeting?.status || 0] || { label: TEXT.statusPending, className: "is-warning", @@ -416,6 +417,12 @@ export default function MeetingPreview() { }); }, [analysis.chapters, meetingChapters, transcripts]); + useEffect(() => { + if (!aiCatalogEnabled && pageTab === "catalog") { + setPageTab("summary"); + } + }, [aiCatalogEnabled, pageTab]); + useEffect(() => { if (!activeTranscriptId) { return; @@ -967,7 +974,7 @@ export default function MeetingPreview() { onChange={(key) => setPageTab(key as PreviewPageTab)} items={[ { key: "summary", label: TEXT.pageSummary }, - { key: "catalog", label: TEXT.pageCatalog }, + ...(aiCatalogEnabled ? [{ key: "catalog", label: TEXT.pageCatalog }] : []), { key: "transcript", label: TEXT.pageTranscript }, ]} /> @@ -975,7 +982,7 @@ export default function MeetingPreview() {
{pageTab === "summary" ? summaryTabContent : null} - {pageTab === "catalog" ? catalogTabContent : null} + {aiCatalogEnabled && pageTab === "catalog" ? catalogTabContent : null} {pageTab === "transcript" ? transcriptTabContent : null}
diff --git a/frontend/src/pages/business/Meetings.tsx b/frontend/src/pages/business/Meetings.tsx index 2e4ad4e..4b0e3d8 100644 --- a/frontend/src/pages/business/Meetings.tsx +++ b/frontend/src/pages/business/Meetings.tsx @@ -10,7 +10,6 @@ InfoCircleOutlined, PauseCircleOutlined, PlusOutlined, - RightOutlined, SearchOutlined, SettingOutlined, SyncOutlined, @@ -136,6 +135,9 @@ const shouldPollMeetingCard = (item: MeetingVO) => const getUnifiedStatusCode = (progress: MeetingProgress | null | undefined) => progress?.unifiedStatus?.statusCode; +const isWaitingUploadProgress = (progress: MeetingProgress | null | undefined) => + getUnifiedStatusCode(progress) === "WAITING_UPLOAD"; + const getEffectiveStatus = (item: MeetingVO, progress: MeetingProgress | null) => { const unifiedStatusCode = getUnifiedStatusCode(progress); if (unifiedStatusCode === "WAITING_UPLOAD") { @@ -285,12 +287,17 @@ const MeetingCardItem: React.FC<{ }> = ({ item, config, progress, onOpenMeeting, onRetrySchedule, onDelete, retrying }) => { const effectiveStatus = getEffectiveStatus(item, progress); const isProcessing = shouldTrackGenerationProgress(item); + const isWaitingUpload = isWaitingUploadProgress(progress) || effectiveStatus === 8; const isPaused = effectiveStatus === PAUSED_DISPLAY_STATUS; const isRealtimeActive = effectiveStatus === REALTIME_ACTIVE_DISPLAY_STATUS; const isRealtimeIdle = effectiveStatus === REALTIME_IDLE_DISPLAY_STATUS; const isCrossPlatformRealtime = (isPaused || isRealtimeActive || isRealtimeIdle) && !canControlRealtimeFromCurrentPlatform(item); const crossPlatformHint = `在 ${getRealtimeSourceLabel(item)} 继续`; const canRetry = canRetryQueuedMeeting(item, progress); + const ownerName = item.creatorName || "未知"; + const processingMessage = isWaitingUpload + ? (progress?.message || progress?.unifiedStatus?.message || config.text) + : (progress?.unifiedStatus?.message || progress?.message || "深度分析中..."); const sourceColor = item.meetingSource === "ANDROID" ? "#10b981" : "#3b82f6"; @@ -375,26 +382,28 @@ const MeetingCardItem: React.FC<{ fontSize: "12px" }} > -
- {isProcessing ? : } - - {isProcessing ? (progress?.message || "深度分析中...") : (isCrossPlatformRealtime ? crossPlatformHint : config.text)} - +
+
+ {isProcessing ? : } + + {isProcessing ? processingMessage : (isCrossPlatformRealtime ? crossPlatformHint : config.text)} + +
+ {canRetry && ( + + )}
- {canRetry && ( - - )}
) : (
@@ -402,10 +411,6 @@ const MeetingCardItem: React.FC<{ {dayjs(item.meetingTime).format("MM-DD HH:mm")}
-
- - {item.creatorName || "未知"} -
)} @@ -433,34 +438,30 @@ const MeetingCardItem: React.FC<{ )) || 无标签} -
- - 查看详情 + + + {ownerName} +
@@ -1001,8 +1002,6 @@ const Meetings: React.FC = () => {