Merge branch 'dev_na' of http://111.10.202.195:3000/chenh/imeeting into dev_ymcg

同步 dev_na 分支的最新更新,确保 dev_ymcg 包含远程修复的功能模块和代码变更,避免后续冲突。
dev_na
puz 2026-06-26 14:14:38 +08:00
commit 255e268138
59 changed files with 3148 additions and 1077 deletions

View File

@ -177,6 +177,12 @@
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
<version>2.3.0</version>
</dependency>
<!-- Source: https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-speech-sdk-java -->
<dependency>
<groupId>com.tencentcloudapi</groupId>
<artifactId>tencentcloud-speech-sdk-java</artifactId>
<version>1.0.67</version>
</dependency>
</dependencies>
<build>

View File

@ -71,6 +71,10 @@ public final class RedisKeys {
return "biz:meeting:realtime:state:" + meetingId;
}
public static String realtimeMeetingTranscriptCacheKey(Long meetingId) {
return "biz:meeting:realtime:transcript-cache:" + meetingId;
}
public static String realtimeMeetingResumeTimeoutKey(Long meetingId) {
return realtimeMeetingResumeTimeoutPrefix() + meetingId;
}

View File

@ -10,6 +10,10 @@ public final class SysParamKeys {
public static final String CAPTCHA_ENABLED = "security.captcha.enabled";
/** AI 会议总结使用的系统提示词。 */
public static final String MEETING_SUMMARY_SYSTEM_PROMPT = "meeting.summary.system_prompt";
/** 是否启用 AI 目录。 */
public static final String MEETING_AI_CATALOG_ENABLED = "meeting.ai_catalog.enabled";
/** 会议总结派发模式PARALLEL / SERIAL。 */
public static final String MEETING_SUMMARY_DISPATCH_MODE = "meeting.summary.dispatch_mode";
/** 离线会议音频上传大小上限,单位 MB。 */
public static final String MEETING_OFFLINE_AUDIO_MAX_SIZE_MB = "meeting.offline_audio.max_size_mb";
/** 是否允许创建离线会议。 */

View File

@ -36,4 +36,16 @@ public class MeetingAsyncExecutorConfig {
executor.initialize();
return executor;
}
@Bean("chunkMergeExecutor")
public Executor chunkMergeExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(4);
executor.setQueueCapacity(10);
executor.setThreadNamePrefix("imeeting-chunk-merge-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.initialize();
return executor;
}
}

View File

@ -49,7 +49,7 @@ public class AndroidMeetingChunkUploadController {
"meetingId", meetingId,
"chunkIndex", chunkIndex,
"chunkFile", chunkFile);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request,true);
androidChunkUploadService.saveChunk(meetingId, chunkIndex, chunkFile, authContext);
return ApiResponse.ok(true);
}
@ -71,6 +71,7 @@ public class AndroidMeetingChunkUploadController {
"meetingId", meetingId,
"totalChunks", totalChunks);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
return ApiResponse.ok(androidChunkUploadService.completeUpload(meetingId, totalChunks, authContext));
androidChunkUploadService.completeUploadAsync(meetingId, totalChunks, authContext);
return ApiResponse.ok(new LegacyUploadAudioResponse(meetingId, null, "后台合并上传中"));
}
}

View File

@ -56,6 +56,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.StopWatch;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@ -79,6 +80,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@Tag(name = "Android会议接口")
@ -97,7 +99,7 @@ public class AndroidMeetingController {
private String h5BaseUrl;
private final AndroidAuthService androidAuthService;
private final AndroidMeetingPushService androidMeetingPushService;
private final AndroidMeetingPushService androidMeetingPushService;
private final AndroidChunkUploadService androidChunkUploadService;
private final LegacyMeetingAdapterService legacyMeetingAdapterService;
private final MeetingQueryService meetingQueryService;
@ -148,7 +150,7 @@ public class AndroidMeetingController {
this.paramService = paramService;
this.dictItemService = dictItemService;
this.meetingUnifiedStatusService = meetingUnifiedStatusService;
this.androidMeetingPushService = androidMeetingPushService;
this.androidMeetingPushService = androidMeetingPushService;
}
@Operation(summary = "创建Android离线会议")
@ -238,22 +240,24 @@ public class AndroidMeetingController {
AndroidRequestLogHelper.logRequest(log, "Android会议", "结束离线会议录音阶段",
"meetingId", meetingId,
"request", command);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
AndroidAuthContext authContext = androidAuthService.authenticateHttpIgnoreToken(request, true);
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser);
LegacyUploadAudioResponse uploadResult = null;
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId);
LegacyUploadAudioResponse uploadResult = new LegacyUploadAudioResponse();
if (isUploadFinishedStage(command)) {
uploadResult = androidChunkUploadService.completeUpload(
androidChunkUploadService.completeUploadAsync(
meeting.getId(),
command == null ? null : command.getTotalChunks(),
command.getTotalChunks(),
authContext
);
if (uploadResult == null || uploadResult.getAudioUrl() == null || uploadResult.getAudioUrl().isBlank()) {
throw new RuntimeException("分片上传完成后未生成 audio_url");
}
// if (uploadResult == null) {
// throw new RuntimeException("分片上传完成后未生成结果");
// }
uploadResult.setMeetingId(meetingId);
}
meetingCommandService.finishOfflineMeeting(meeting.getId(), command == null ? null : command.getFinishStage());
return ApiResponse.ok(uploadResult != null ? uploadResult : true);
return ApiResponse.ok(uploadResult);
}
@Operation(summary = "分页查询Android会议")
@ -292,7 +296,7 @@ public class AndroidMeetingController {
}
@Operation(summary = "查询Android会议统一状态")
@Operation(summary = "查询Android会议统一状态")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
@ -310,13 +314,14 @@ public class AndroidMeetingController {
"request", command);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
MeetingVO meeting = requireOperableOfflineMeeting(meetingId, authContext, loginUser);
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId,false);
UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId);
boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript());
boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary());
List<MeetingTranscriptVO> transcripts = includeTranscript ? meetingQueryService.getTranscripts(meetingId) : null;
String summaryContent = includeSummary ? meetingQueryService.getDetailIgnoreTenant(meetingId).getSummaryContent() : null;
return ApiResponse.ok(AndroidUnifiedMeetingStatusResponse.builder()
String summaryContent = includeSummary ? meeting.getSummaryContent() : null;
AndroidUnifiedMeetingStatusResponse build = AndroidUnifiedMeetingStatusResponse.builder()
.meetingId(meetingId)
.status(status)
.meeting(meeting)
@ -324,7 +329,9 @@ public class AndroidMeetingController {
.transcripts(transcripts)
.includesSummary(includeSummary)
.summaryContent(summaryContent)
.build());
.build();
log.info("[{}]{}.返回数据:[{}]", "Android会议", "查询会议统一状态", build);
return ApiResponse.ok(build);
}
@Operation(summary = "重试 Android 会议 ASR 识别")
@ -343,7 +350,7 @@ public class AndroidMeetingController {
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
meetingCommandService.retryTranscription(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode());
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.TRANSCRIBING.getCode());
return ApiResponse.ok(true);
}
@ -364,17 +371,18 @@ public class AndroidMeetingController {
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
meetingCommandService.retrySummary(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode());
return ApiResponse.ok(true);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.SUMMARIZING.getCode());
return ApiResponse.ok(true);
}
@Operation(summary = "更新Android会议访问密码")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "返回更新后的会议访问密码,传空时表示清空访问密码",
content = @Content(schema = @Schema(implementation = String.class))
)
})
@Operation(summary = "更新Android会议访问密码")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "返回更新后的会议访问密码,传空时表示清空访问密码",
content = @Content(schema = @Schema(implementation = String.class))
)
})
@PutMapping("/{meetingId}/access-password")
@Log(value = "修改Android会议访问密码", type = "Android会议管理")
public ApiResponse<String> updateAccessPassword(HttpServletRequest request,
@ -390,9 +398,9 @@ public class AndroidMeetingController {
return ApiResponse.error("仅会议创建人可设置访问密码");
}
String password = normalizePassword(command == null ? null : command.getPassword());
meetingService.update(new LambdaUpdateWrapper<Meeting>()
.eq(Meeting::getId,meeting.getId())
.set(Meeting::getAccessPassword, password));
meetingService.update(new LambdaUpdateWrapper<Meeting>()
.eq(Meeting::getId, meeting.getId())
.set(Meeting::getAccessPassword, password));
return ApiResponse.ok(password);
}
@ -415,54 +423,55 @@ public class AndroidMeetingController {
meetingCommandService.deleteMeeting(meetingId);
return ApiResponse.ok(true);
}
@GetMapping("/config")
@Log(value = "获取会议配置", type = "Android会议管理")
@Operation(summary = "获取会议配置")
@Anonymous
public ApiResponse<AndroidMeetingConfigVo> config(HttpServletRequest request) {
AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口");
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext);
Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId();
Long userId = loginUser != null ? loginUser.getUserId() : null;
boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin());
boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin());
AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo();
PageResult<List<PromptTemplateVO>> promptTemplateList = promptTemplateService.pageTemplates(
1,
1000,
null,
null,
tenantId,
userId,
isPlatformAdmin,
isTenantAdmin
);
List<PromptTemplateVO> enabledTemplates = promptTemplateList.getRecords() == null
? List.of()
: promptTemplateList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setTemplateList(enabledTemplates);
PageResult<List<AiModelVO>> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId);
List<AiModelVO> enabledModels = modelList.getRecords() == null
? List.of()
: modelList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setModelsList(enabledModels);
resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail"));
resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION,"30")));
resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10")));
resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION,String.valueOf(60*4))));
BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99"));
bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP);
resultVo.setPacketLossRate(bigDecimal );
resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false")));
resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60")));
return ApiResponse.ok(resultVo);
}
@GetMapping("/config")
@Log(value = "获取会议配置", type = "Android会议管理")
@Operation(summary = "获取会议配置")
@Anonymous
public ApiResponse<AndroidMeetingConfigVo> config(HttpServletRequest request) {
AndroidRequestLogHelper.logRequest(log, "Android会议", "获取会议配置接口");
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = AndroidLoginUserSupport.toLoginUser(authContext);
Long tenantId = loginUser != null ? loginUser.getTenantId() : authContext.getTenantId();
Long userId = loginUser != null ? loginUser.getUserId() : null;
boolean isPlatformAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsPlatformAdmin());
boolean isTenantAdmin = loginUser != null && Boolean.TRUE.equals(loginUser.getIsTenantAdmin());
AndroidMeetingConfigVo resultVo = new AndroidMeetingConfigVo();
PageResult<List<PromptTemplateVO>> promptTemplateList = promptTemplateService.pageTemplates(
1,
1000,
null,
null,
tenantId,
userId,
isPlatformAdmin,
isTenantAdmin
);
List<PromptTemplateVO> enabledTemplates = promptTemplateList.getRecords() == null
? List.of()
: promptTemplateList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setTemplateList(enabledTemplates);
PageResult<List<AiModelVO>> modelList = aiModelService.pageModels(1, 1000, null, "LLM", tenantId);
List<AiModelVO> enabledModels = modelList.getRecords() == null
? List.of()
: modelList.getRecords().stream()
.filter(item -> Integer.valueOf(1).equals(item.getStatus()))
.toList();
resultVo.setModelsList(enabledModels);
resultVo.setSummaryDegreeOfDetail(dictItemService.getItemsByTypeCode("summary_degree_detail"));
resultVo.setMaxMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_MEETING_DURATION, "30")));
resultVo.setMinMeetingDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MIN_MEETING_DURATION, "10")));
resultVo.setMaxPauseDuration(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, String.valueOf(60 * 4))));
BigDecimal bigDecimal = new BigDecimal(paramService.getParamValue(SysParamKeys.MEETING_MAX_PAUSE_DURATION, "99"));
bigDecimal = bigDecimal.setScale(2, RoundingMode.HALF_UP);
resultVo.setPacketLossRate(bigDecimal);
resultVo.setChunkUploadEnabled(Boolean.parseBoolean(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_UPLOAD_ENABLED, "false")));
resultVo.setChunkDurationSeconds(Integer.valueOf(paramService.getParamValue(SysParamKeys.MEETING_ANDROID_AUDIO_CHUNK_DURATION_SECONDS, "60")));
return ApiResponse.ok(resultVo);
}
private void resolvePublicDeviceTenantId(HttpServletRequest request,
AndroidOfflineMeetingCreateCommand command,
@ -545,29 +554,14 @@ public class AndroidMeetingController {
return ChronoUnit.DAYS.between(LocalDate.now(), meetingTime.toLocalDate());
}
private MeetingVO requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) {
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId);
if (meeting == null) {
throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(), "会议不存在");
}
private Meeting requireOperableOfflineMeeting(Long meetingId, AndroidAuthContext authContext, LoginUser loginUser) {
Meeting meeting = meetingAccessService.requireMeetingIgnoreTenant(meetingId);
if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) {
throw new RuntimeException("当前会议不是离线会议");
}
if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) {
throw new RuntimeException("设备ID不能为空");
}
// if (meeting.getSourceDeviceCode() == null || !meeting.getSourceDeviceCode().equals(authContext.getDeviceId())) {
// throw new RuntimeException("当前会议不属于该设备");
// }
// if (authContext.isAnonymous()) {
// if (!MeetingConstants.DEVICE_MODE_PUBLIC.equals(meeting.getSourceDeviceMode())) {
// throw new RuntimeException("当前会议不是公有设备会议");
// }
// return meeting;
// }
// if (loginUser == null || !Objects.equals(meeting.getCreatorId(), loginUser.getUserId())) {
// throw new RuntimeException("仅会议创建人可操作当前会议");
// }
return meeting;
}

View File

@ -3,6 +3,7 @@ package com.imeeting.controller.biz;
import com.imeeting.dto.biz.AiLocalProfileVO;
import com.imeeting.dto.biz.AiModelDTO;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.AiModelService;
import com.unisbase.common.ApiResponse;
import com.unisbase.common.annotation.Log;
@ -121,7 +122,7 @@ public class AiModelController {
@PostMapping("/llm-connectivity-test")
@PreAuthorize("isAuthenticated()")
public ApiResponse<Boolean> testLlmConnectivity(@RequestBody AiModelDTO dto) {
if ("custom".equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) {
if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(dto.getProvider()) && (dto.getBaseUrl() == null || dto.getBaseUrl().isBlank())) {
return ApiResponse.error("基础地址不能为空");
}
if (dto.getModelCode() == null || dto.getModelCode().isBlank()) {

View File

@ -212,6 +212,7 @@ public class MeetingController {
MeetingCreateConfigVO vo = new MeetingCreateConfigVO();
vo.setOfflineEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_OFFLINE_ENABLED, true));
vo.setRealtimeEnabled(resolveBooleanParam(SysParamKeys.MEETING_CREATE_REALTIME_ENABLED, true));
vo.setAiCatalogEnabled(resolveBooleanParam(SysParamKeys.MEETING_AI_CATALOG_ENABLED, false));
vo.setOfflineAudioMaxSizeMb(resolveLongParam(SysParamKeys.MEETING_OFFLINE_AUDIO_MAX_SIZE_MB, 1024L));
return ApiResponse.ok(vo);
}
@ -398,17 +399,6 @@ public class MeetingController {
return ApiResponse.ok(result);
}
@Operation(summary = "追加实时转写片段")
@PostMapping("/{id}/realtime/transcripts")
@PreAuthorize("isAuthenticated()")
public ApiResponse<Boolean> appendRealtimeTranscripts(@PathVariable Long id, @RequestBody List<RealtimeTranscriptItemDTO> items) {
LoginUser loginUser = currentLoginUser();
Meeting meeting = meetingAccessService.requireMeeting(id);
meetingAccessService.assertCanControlRealtimeMeeting(meeting, loginUser, MeetingConstants.SOURCE_WEB);
meetingCommandService.appendRealtimeTranscripts(id, items);
return ApiResponse.ok(true);
}
@Operation(summary = "暂停实时会议")
@PostMapping("/{id}/realtime/pause")
@PreAuthorize("isAuthenticated()")

View File

@ -6,12 +6,14 @@ import com.imeeting.dto.biz.UnifiedMeetingStatusVO;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.util.List;
@Data
@Builder
@Schema(description = "Android 统一会议状态响应")
@ToString
public class AndroidUnifiedMeetingStatusResponse {
@Schema(description = "会议ID")
private Long meetingId;

View File

@ -18,4 +18,12 @@ public class LegacyUploadAudioResponse {
@JsonProperty("audio_url")
@Schema(description = "上传后的音频访问地址")
private String audioUrl;
@JsonProperty("message")
@Schema(description = "结果提示")
private String message;
public LegacyUploadAudioResponse(Long meetingId, String audioUrl) {
this(meetingId, audioUrl, null);
}
}

View File

@ -13,6 +13,9 @@ public class MeetingCreateConfigVO {
@Schema(description = "是否启用实时会议")
private Boolean realtimeEnabled;
@Schema(description = "是否启用 AI 目录")
private Boolean aiCatalogEnabled;
@Schema(description = "离线音频上传大小上限,单位 MB")
private Long offlineAudioMaxSizeMb;
}

View File

@ -75,6 +75,9 @@ public class MeetingVO {
@Schema(description = "总结模板ID")
private Long promptId;
@Schema(description = "是否启用 AI 目录")
private Boolean aiCatalogEnabled;
@Schema(description = "音频保存状态")
private String audioSaveStatus;

View File

@ -27,4 +27,6 @@ public class RealtimeMeetingResumeConfig {
private Boolean saveAudio;
@Schema(description = "热词列表")
private List<Map<String, Object>> hotwords;
@Schema(description = "腾讯说话人上下文 ID")
private String speakerContextId;
}

View File

@ -0,0 +1,22 @@
package com.imeeting.dto.biz;
import lombok.Data;
@Data
public class RealtimeMeetingTranscriptCacheItem {
private String sentenceKey;
private String sentenceGroupKey;
private Integer sentenceId;
private Integer sentenceType;
private String speakerId;
private String speakerName;
private String userId;
private Integer startTime;
private Integer endTime;
private String content;
private Integer sortOrder;
private Boolean finalResult;
private Long transcriptId;
private Long firstReceivedAt;
private Long updatedAt;
}

View File

@ -0,0 +1,15 @@
package com.imeeting.dto.biz;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class RealtimeMeetingTranscriptCacheState {
private Long meetingId;
private Integer nextSortOrder;
private Integer nextLegacySequence;
private List<RealtimeMeetingTranscriptCacheItem> items = new ArrayList<>();
private Long updatedAt;
}

View File

@ -2,11 +2,16 @@ package com.imeeting.dto.biz;
import lombok.Data;
import java.util.Map;
@Data
public class RealtimeSocketSessionData {
private Long meetingId;
private Long userId;
private Long tenantId;
private Long asrModelId;
private String provider;
private String targetWsUrl;
private String modelCode;
private Map<String, Object> mediaConfig;
}

View File

@ -0,0 +1,17 @@
package com.imeeting.enums;
import lombok.Getter;
@Getter
public enum ModelProviderEnum {
LOCAL("local", "本地"),
TENCENT("tencent", "腾讯云");
private final String code;
private final String description;
ModelProviderEnum(String code, String description) {
this.code = code;
this.description = description;
}
}

View File

@ -254,7 +254,20 @@ public class AndroidPushGrpcService extends PushServiceGrpc.PushServiceImplBase
return switch (platform) {
case IOS -> "ios";
case ANDROID -> "android";
case PLATFORM_UNKNOWN, UNRECOGNIZED -> "android";
case PLATFORM_UNKNOWN,UNRECOGNIZED -> "android";
case HARMONY_MOBILE ->"harmony_mobile";
// Desktop
case WINDOWS ->"windows";
case MACOS->"macos";
case LINUX -> "linux";
// Linux发行版可选
case KYLIN ->"kylin";
case UOS ->"uos";
// Harmony PC
case HARMONY_PC ->"harmony_pc";
};
}
}

View File

@ -15,4 +15,12 @@ public interface AndroidChunkUploadService {
LegacyUploadAudioResponse completeUpload(Long meetingId,
Integer totalChunks,
AndroidAuthContext authContext) throws IOException;
/**
* + + 线
* 线Tomcat failOfflineTranscription
*/
void completeUploadAsync(Long meetingId,
Integer totalChunks,
AndroidAuthContext authContext);
}

View File

@ -5,14 +5,16 @@ import com.imeeting.dto.android.AndroidChunkUploadSessionState;
import com.imeeting.dto.android.legacy.LegacyUploadAudioResponse;
import com.imeeting.service.android.AndroidChunkUploadService;
import com.imeeting.service.android.legacy.LegacyMeetingAdapterService;
import com.imeeting.service.biz.MeetingCommandService;
import com.imeeting.service.biz.MeetingQueryService;
import com.imeeting.support.TaskSecurityContextRunner;
import com.imeeting.support.redis.AndroidChunkUploadSessionCache;
import com.unisbase.security.LoginUser;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
@ -30,12 +32,16 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Service
@RequiredArgsConstructor
@Slf4j
public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService {
private static final Pattern CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$");
private static final Pattern LEGACY_CHUNK_FILE_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)(\\..+)?$");
private static final Pattern CHUNK_DIR_NAME_PATTERN = Pattern.compile("^chunk-(\\d+)$");
private static final String CHUNK_ROOT_DIR = "chunks";
private final TaskSecurityContextRunner taskSecurityContextRunner;
private final AndroidChunkUploadSessionCache sessionCache;
private final LegacyMeetingAdapterService legacyMeetingAdapterService;
private final MeetingCommandService meetingCommandService;
private final java.util.concurrent.Executor chunkMergeExecutor;
@Value("${unisbase.app.upload-path}")
private String uploadPath;
@ -43,6 +49,18 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
@Value("${imeeting.audio.ffmpeg-path:ffmpeg}")
private String ffmpegPath;
public AndroidChunkUploadServiceImpl(AndroidChunkUploadSessionCache sessionCache,
LegacyMeetingAdapterService legacyMeetingAdapterService,
MeetingCommandService meetingCommandService,
@Qualifier("chunkMergeExecutor") java.util.concurrent.Executor chunkMergeExecutor,
TaskSecurityContextRunner taskSecurityContextRunner) {
this.sessionCache = sessionCache;
this.legacyMeetingAdapterService = legacyMeetingAdapterService;
this.meetingCommandService = meetingCommandService;
this.chunkMergeExecutor = chunkMergeExecutor;
this.taskSecurityContextRunner = taskSecurityContextRunner;
}
@Override
public void saveChunk(Long meetingId,
Integer chunkIndex,
@ -54,7 +72,7 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (chunkIndex == null || chunkIndex < 0) {
throw new RuntimeException("分片参数无效");
}
if (chunkFile == null) {
if (chunkFile == null) {
throw new RuntimeException("chunk_file不能为空");
}
@ -63,19 +81,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
throw new RuntimeException("分片上传会话与当前设备或会议不匹配");
}
String chunkFileName = buildStoredChunkFileName(chunkIndex, chunkFile.getOriginalFilename());
String previousFileName = state.getChunkFileNames().get(chunkIndex);
Path meetingDir = sessionDir(meetingId);
Files.createDirectories(meetingDir);
String originalFileName = resolveOriginalFileName(chunkFile.getOriginalFilename());
Path chunkFilePath = resolveChunkFilePath(meetingId, chunkIndex, originalFileName);
Path chunkDir = chunkFilePath.getParent();
clearChunkDirectory(chunkDir);
Files.createDirectories(chunkDir);
if (previousFileName != null && !previousFileName.equals(chunkFileName)) {
deleteQuietly(meetingDir.resolve(previousFileName));
Files.write(chunkFilePath, chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
String previousFileName = state.getChunkFileNames().put(chunkIndex, originalFileName);
if (previousFileName != null && !previousFileName.equals(originalFileName)) {
state.getUploadedChunkFileNames().remove(previousFileName);
}
Files.write(meetingDir.resolve(chunkFileName), chunkFile.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
state.getUploadedChunkFileNames().add(chunkFileName);
state.getChunkFileNames().put(chunkIndex, chunkFileName);
state.getUploadedChunkFileNames().add(originalFileName);
state.getReceivedChunks().add(chunkIndex);
saveState(meetingId, state);
}
@ -91,53 +109,70 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
throw new RuntimeException("total_chunks不能为空且必须大于0");
}
AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext);
AndroidChunkUploadSessionState state = loadStateForCompletion(meetingId, authContext);
if (!Objects.equals(state.getMeetingId(), meetingId) || !Objects.equals(state.getDeviceId(), authContext.getDeviceId())) {
throw new RuntimeException("分片上传会话与当前设备或会议不匹配");
}
Path meetingDir = sessionDir(meetingId);
Files.createDirectories(meetingDir);
Path meetingDir = sessionDir(meetingId);
Files.createDirectories(meetingDir);
state.setTotalChunks(totalChunks);
List<Path> orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks);
List<Path> orderedChunkPaths = rebuildChunkStateFromDisk(state, meetingDir, totalChunks);
saveState(meetingId, state);
Path mergedFile = mergeChunks(state, orderedChunkPaths);
Path mergedFile = mergeChunks(state, orderedChunkPaths);
if (mergedFile == null) {
meetingCommandService.failOfflineTranscription(meetingId, "安卓上传文件为空");
cleanup(meetingId);
return new LegacyUploadAudioResponse(meetingId, null, "无可合并音频");
}
MultipartFile mergedMultipart = new LocalMultipartFile(
buildMergedOriginalFilename(state, mergedFile),
resolveMergedOriginalFilename(state, orderedChunkPaths, mergedFile),
state.getContentType(),
Files.readAllBytes(mergedFile)
mergedFile
);
LegacyUploadAudioResponse response;
if (authContext.isAnonymous()) {
response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice(
meetingId,
null,
null,
false,
mergedMultipart,
authContext
);
} else {
LoginUser loginUser = toLoginUser(authContext);
response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcess(
meetingId,
null,
null,
false,
mergedMultipart,
authContext,
loginUser
);
}
LegacyUploadAudioResponse response = legacyMeetingAdapterService.uploadAndTriggerOfflineProcessForPublicDevice(
meetingId,
null,
null,
false,
mergedMultipart,
authContext
);
if (response != null) {
cleanup(meetingId);
}
return response;
}
@Override
public void completeUploadAsync(Long meetingId,
Integer totalChunks,
AndroidAuthContext authContext) {
if (meetingId == null) {
throw new RuntimeException("meeting_id不能为空");
}
if (totalChunks == null || totalChunks <= 0) {
throw new RuntimeException("total_chunks不能为空且必须大于0");
}
chunkMergeExecutor.execute( ()->taskSecurityContextRunner.runAsTenantUser( authContext.getTenantId(), authContext.getUserId(), () -> {
try {
completeUpload(meetingId, totalChunks, authContext);
} catch (Exception ex) {
log.error("[分片合并] 会议{}异步合并上传失败: {}", meetingId, ex.getMessage(), ex);
try {
meetingCommandService.failOfflineTranscription(meetingId, "音频合并上传失败: " + ex.getMessage());
} catch (Exception inner) {
log.error("[分片合并] 会议{}标记失败状态异常: {}", meetingId, inner.getMessage(), inner);
}
}
}));
}
private AndroidChunkUploadSessionState getOrCreateState(Long meetingId,
MultipartFile chunkFile,
AndroidAuthContext authContext) {
@ -148,155 +183,170 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
AndroidChunkUploadSessionState state = new AndroidChunkUploadSessionState();
state.setMeetingId(meetingId);
state.setDeviceId(authContext.getDeviceId());
state.setFileName(normalizeChunkSourceFileName(chunkFile.getOriginalFilename()));
state.setFileName(resolveOriginalFileName(chunkFile.getOriginalFilename()));
state.setContentType(chunkFile.getContentType());
saveState(meetingId, state);
return state;
}
private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) {
AndroidChunkUploadSessionState state = getState(meetingId);
if (state != null) {
return state;
}
AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState();
rebuiltState.setMeetingId(meetingId);
rebuiltState.setDeviceId(authContext.getDeviceId());
return rebuiltState;
}
private List<Path> rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state,
Path meetingDir,
int totalChunks) throws IOException {
Map<Integer, Path> validatedChunkFiles = scanChunkFiles(meetingDir, true);
Map<Integer, Path> mergeChunkFiles = scanChunkFiles(meetingDir, false);
state.getReceivedChunks().clear();
state.getUploadedChunkFileNames().clear();
state.getChunkFileNames().clear();
for (Map.Entry<Integer, Path> entry : validatedChunkFiles.entrySet()) {
Integer chunkIndex = entry.getKey();
Path stateChunkPath = mergeChunkFiles.getOrDefault(chunkIndex, entry.getValue());
String fileName = stateChunkPath.getFileName().toString();
state.getReceivedChunks().add(chunkIndex);
state.getUploadedChunkFileNames().add(fileName);
state.getChunkFileNames().put(chunkIndex, fileName);
private AndroidChunkUploadSessionState loadStateForCompletion(Long meetingId, AndroidAuthContext authContext) {
AndroidChunkUploadSessionState state = getState(meetingId);
if (state != null) {
return state;
}
AndroidChunkUploadSessionState rebuiltState = new AndroidChunkUploadSessionState();
rebuiltState.setMeetingId(meetingId);
rebuiltState.setDeviceId(authContext.getDeviceId());
return rebuiltState;
}
List<Path> orderedChunkPaths = new ArrayList<>(totalChunks);
for (int i = 0; i < totalChunks; i++) {
Path validatedChunkPath = validatedChunkFiles.get(i);
if (validatedChunkPath == null) {
throw new RuntimeException("分片未上传完整");
}
Path chunkPath = mergeChunkFiles.get(i);
if (chunkPath != null) {
orderedChunkPaths.add(chunkPath);
}
}
return orderedChunkPaths;
}
private List<Path> rebuildChunkStateFromDisk(AndroidChunkUploadSessionState state,
Path meetingDir,
int totalChunks) throws IOException {
Map<Integer, Path> chunkFiles = scanChunkFiles(meetingDir, true);
state.getReceivedChunks().clear();
state.getUploadedChunkFileNames().clear();
state.getChunkFileNames().clear();
private Map<Integer, Path> scanChunkFiles(Path meetingDir, boolean includePending) throws IOException {
Map<Integer, Path> chunkFiles = new TreeMap<>();
if (!Files.exists(meetingDir)) {
return chunkFiles;
}
try (var paths = Files.list(meetingDir)) {
paths.filter(Files::isRegularFile)
.filter(path -> isUsableChunkFile(path, includePending))
.forEach(path -> {
Integer chunkIndex = parseChunkIndex(path.getFileName().toString(), includePending);
if (chunkIndex == null) {
return;
}
Path existing = chunkFiles.get(chunkIndex);
if (existing == null || isPreferredChunkFile(path, existing)) {
chunkFiles.put(chunkIndex, path);
}
});
}
return chunkFiles;
}
for (Map.Entry<Integer, Path> entry : chunkFiles.entrySet()) {
Integer chunkIndex = entry.getKey();
Path chunkPath = entry.getValue();
String fileName = chunkPath.getFileName().toString();
state.getReceivedChunks().add(chunkIndex);
state.getUploadedChunkFileNames().add(fileName);
state.getChunkFileNames().put(chunkIndex, fileName);
}
private boolean isUsableChunkFile(Path path, boolean includePending) {
if (path == null || path.getFileName() == null) {
return false;
List<Path> orderedChunkPaths = new ArrayList<>(totalChunks);
for (int i = 0; i < totalChunks; i++) {
Path chunkPath = chunkFiles.get(i);
if (chunkPath == null) {
throw new RuntimeException("分片未上传完整");
}
orderedChunkPaths.add(chunkPath);
}
return orderedChunkPaths;
}
String fileName = normalizeChunkStorageFileName(path.getFileName().toString(), includePending);
if (fileName == null) {
return false;
}
return CHUNK_FILE_NAME_PATTERN.matcher(fileName).matches();
}
private Integer parseChunkIndex(String fileName, boolean includePending) {
String normalizedFileName = normalizeChunkStorageFileName(fileName, includePending);
if (normalizedFileName == null) {
return null;
}
Matcher matcher = CHUNK_FILE_NAME_PATTERN.matcher(normalizedFileName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
}
private Map<Integer, Path> scanChunkFiles(Path meetingDir, boolean includePending) throws IOException {
Map<Integer, Path> chunkFiles = new TreeMap<>();
if (!Files.exists(meetingDir)) {
return chunkFiles;
}
private boolean isPreferredChunkFile(Path candidate, Path existing) {
boolean candidatePending = isPendingChunkFile(candidate);
boolean existingPending = isPendingChunkFile(existing);
if (candidatePending != existingPending) {
return !candidatePending;
}
try {
return Files.getLastModifiedTime(candidate).compareTo(Files.getLastModifiedTime(existing)) >= 0;
} catch (IOException ex) {
return candidate.getFileName().toString().compareTo(existing.getFileName().toString()) >= 0;
}
}
Path chunkRoot = meetingDir.resolve(CHUNK_ROOT_DIR);
if (Files.exists(chunkRoot)) {
try (var dirs = Files.list(chunkRoot)) {
dirs.filter(Files::isDirectory).forEach(dir -> {
Integer chunkIndex = parseChunkDirIndex(dir.getFileName().toString());
if (chunkIndex == null || chunkFiles.containsKey(chunkIndex)) {
return;
}
Path chunkPath = pickChunkFile(dir, includePending);
if (chunkPath != null) {
chunkFiles.put(chunkIndex, chunkPath);
}
});
}
if (!chunkFiles.isEmpty()) {
return chunkFiles;
}
}
private String normalizeChunkStorageFileName(String fileName, boolean includePending) {
if (fileName == null) {
return null;
try (var paths = Files.list(meetingDir)) {
paths.filter(Files::isRegularFile)
.forEach(path -> {
Integer chunkIndex = parseLegacyChunkIndex(path.getFileName().toString());
if (chunkIndex != null) {
chunkFiles.put(chunkIndex, path);
}
});
}
return chunkFiles;
}
if (fileName.endsWith(".pending")) {
return includePending ? fileName.substring(0, fileName.length() - ".pending".length()) : null;
private Integer parseChunkDirIndex(String directoryName) {
if (directoryName == null) {
return null;
}
Matcher matcher = CHUNK_DIR_NAME_PATTERN.matcher(directoryName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
}
return fileName;
}
private boolean isPendingChunkFile(Path path) {
return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending");
}
private Integer parseLegacyChunkIndex(String fileName) {
if (fileName == null) {
return null;
}
Matcher matcher = LEGACY_CHUNK_FILE_NAME_PATTERN.matcher(fileName);
if (!matcher.matches()) {
return null;
}
return Integer.parseInt(matcher.group(1));
}
private Path mergeChunks(AndroidChunkUploadSessionState state, List<Path> chunkPaths) throws IOException {
private Path pickChunkFile(Path chunkDir, boolean includePending) {
if (chunkDir == null || !Files.isDirectory(chunkDir)) {
return null;
}
try (var files = Files.list(chunkDir)) {
Path preferred = files
.filter(Files::isRegularFile)
.filter(path -> includePending || !isPendingChunkFile(path))
.findFirst()
.orElse(null);
if (preferred != null) {
return preferred;
}
} catch (IOException ex) {
return null;
}
try (var files = Files.list(chunkDir)) {
return files.filter(Files::isRegularFile).findFirst().orElse(null);
} catch (IOException ex) {
return null;
}
}
private Path mergeChunks(AndroidChunkUploadSessionState state, List<Path> chunkPaths) throws IOException {
List<Path> mergeableChunkPaths = filterMergeableChunkPaths(chunkPaths);
Path meetingDir = sessionDir(state.getMeetingId());
String mergedExtension = resolveMergedExtension(state, chunkPaths);
String mergedExtension = resolveMergedExtension(state, mergeableChunkPaths);
Path mergedOutput = meetingDir.resolve("merged" + mergedExtension);
Path concatList = meetingDir.resolve("concat-inputs.txt");
Files.deleteIfExists(mergedOutput);
if (chunkPaths.isEmpty() || allChunkFilesEmpty(chunkPaths)) {
Files.write(mergedOutput, new byte[0], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
return mergedOutput;
}
if (mergeableChunkPaths.isEmpty() || allChunkFilesEmpty(mergeableChunkPaths)) {
return null;
}
if (chunkPaths.size() == 1) {
return chunkPaths.get(0);
}
if (mergeableChunkPaths.size() == 1) {
return mergeableChunkPaths.get(0);
}
writeConcatListFile(concatList, chunkPaths);
writeConcatListFile(concatList, mergeableChunkPaths);
executeFfmpegConcat(concatList, mergedOutput);
return mergedOutput;
}
private boolean allChunkFilesEmpty(List<Path> chunkPaths) throws IOException {
for (Path chunkPath : chunkPaths) {
if (Files.size(chunkPath) > 0) {
return false;
}
private boolean allChunkFilesEmpty(List<Path> chunkPaths) throws IOException {
for (Path chunkPath : chunkPaths) {
if (Files.size(chunkPath) > 0) {
return false;
}
}
return true;
}
return true;
private List<Path> filterMergeableChunkPaths(List<Path> chunkPaths) {
if (chunkPaths == null || chunkPaths.isEmpty()) {
return List.of();
}
return chunkPaths.stream()
.filter(path -> path != null && !isPendingChunkFile(path))
.toList();
}
private AndroidChunkUploadSessionState getState(Long meetingId) {
@ -329,22 +379,19 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
return Paths.get(normalizedBasePath, "android-chunks", String.valueOf(meetingId));
}
private String buildStoredChunkFileName(Integer chunkIndex, String originalFileName) {
String normalizedSourceName = normalizeChunkSourceFileName(originalFileName);
int extensionIndex = normalizedSourceName.lastIndexOf('.');
String extension = extensionIndex >= 0 ? normalizedSourceName.substring(extensionIndex) : "";
String safeExtension = extension.isBlank() ? ".bin" : extension;
return "chunk-" + (chunkIndex == null ? "unknown" : chunkIndex) + safeExtension;
private Path resolveChunkFilePath(Long meetingId, Integer chunkIndex, String originalFileName) {
return sessionDir(meetingId)
.resolve(CHUNK_ROOT_DIR)
.resolve("chunk-" + chunkIndex)
.resolve(originalFileName);
}
private String buildMergedOriginalFilename(AndroidChunkUploadSessionState state, Path mergedFile) {
if (state.getFileName() != null && !state.getFileName().isBlank()) {
String normalizedSourceName = normalizeChunkSourceFileName(state.getFileName());
int extensionIndex = normalizedSourceName.lastIndexOf('.');
if (extensionIndex >= 0) {
return "merged" + normalizedSourceName.substring(extensionIndex);
}
return normalizedSourceName;
private String resolveMergedOriginalFilename(AndroidChunkUploadSessionState state, List<Path> chunkPaths, Path mergedFile) {
if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) {
return state.getFileName();
}
if (chunkPaths != null && !chunkPaths.isEmpty() && chunkPaths.get(0).getFileName() != null) {
return chunkPaths.get(0).getFileName().toString();
}
return mergedFile == null || mergedFile.getFileName() == null ? "meeting-audio.bin" : mergedFile.getFileName().toString();
}
@ -353,41 +400,56 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (chunkPath == null || chunkPath.getFileName() == null) {
return ".bin";
}
String fileName = normalizeChunkSourceFileName(chunkPath.getFileName().toString());
String fileName = chunkPath.getFileName().toString();
int extensionIndex = fileName.lastIndexOf('.');
return extensionIndex >= 0 ? fileName.substring(extensionIndex) : ".bin";
}
private String resolveMergedExtension(AndroidChunkUploadSessionState state, List<Path> chunkPaths) {
if (chunkPaths != null && !chunkPaths.isEmpty()) {
return resolveMergedExtension(chunkPaths.get(0));
}
if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) {
int extensionIndex = state.getFileName().lastIndexOf('.');
if (extensionIndex >= 0) {
return state.getFileName().substring(extensionIndex);
}
}
return ".bin";
private String resolveMergedExtension(AndroidChunkUploadSessionState state, List<Path> chunkPaths) {
if (chunkPaths != null && !chunkPaths.isEmpty()) {
return resolveMergedExtension(chunkPaths.get(0));
}
if (state != null && state.getFileName() != null && !state.getFileName().isBlank()) {
int extensionIndex = state.getFileName().lastIndexOf('.');
if (extensionIndex >= 0) {
return state.getFileName().substring(extensionIndex);
}
}
return ".bin";
}
private String normalizeChunkSourceFileName(String fileName) {
if (fileName == null) {
return "";
private boolean isPendingChunkFile(Path path) {
return path != null && path.getFileName() != null && path.getFileName().toString().endsWith(".pending");
}
private String resolveOriginalFileName(String originalFileName) {
if (originalFileName == null || originalFileName.trim().isEmpty()) {
throw new RuntimeException("chunk_file原始文件名不能为空");
}
String normalized = Paths.get(fileName.trim()).getFileName().toString();
if (normalized.endsWith(".pending")) {
return normalized.substring(0, normalized.length() - ".pending".length());
if (originalFileName.contains("/") || originalFileName.contains("\\")) {
throw new RuntimeException("chunk_file文件名不合法");
}
return originalFileName;
}
private void clearChunkDirectory(Path chunkDir) throws IOException {
if (chunkDir == null || !Files.exists(chunkDir)) {
return;
}
try (var paths = Files.walk(chunkDir)) {
paths.sorted((left, right) -> right.compareTo(left)).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
});
}
return normalized;
}
private void writeConcatListFile(Path concatList, List<Path> chunkPaths) throws IOException {
List<String> lines = new ArrayList<>(chunkPaths.size());
for (Path chunkPath : chunkPaths) {
if (chunkPath.getFileName() != null && chunkPath.getFileName().toString().endsWith(".pending")) {
continue;
}
String normalizedPath = chunkPath.toAbsolutePath().toString().replace("'", "'\\''");
lines.add("file '" + normalizedPath + "'");
}
@ -424,46 +486,20 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
if (process.exitValue() != 0) {
throw new IOException("音频分片合并失败: " + new String(output, StandardCharsets.UTF_8));
}
if (!Files.exists(mergedOutput)) {
if (!Files.exists(mergedOutput)) {
throw new IOException("音频分片合并结果为空");
}
}
private void deleteQuietly(Path path) {
if (path == null) {
return;
}
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private LoginUser toLoginUser(AndroidAuthContext authContext) {
if (authContext == null || authContext.isAnonymous() || authContext.getUserId() == null || authContext.getTenantId() == null) {
throw new RuntimeException("安卓用户未登录或认证无效");
}
LoginUser loginUser = new LoginUser(
authContext.getUserId(),
authContext.getTenantId(),
authContext.getUsername(),
authContext.getPlatformAdmin(),
authContext.getTenantAdmin(),
authContext.getPermissions()
);
loginUser.setDisplayName(authContext.getDisplayName());
return loginUser;
}
private static final class LocalMultipartFile implements MultipartFile {
private final String originalFilename;
private final String contentType;
private final byte[] bytes;
private final Path filePath;
private LocalMultipartFile(String originalFilename, String contentType, byte[] bytes) {
private LocalMultipartFile(String originalFilename, String contentType, Path filePath) {
this.originalFilename = originalFilename;
this.contentType = contentType;
this.bytes = bytes == null ? new byte[0] : bytes;
this.filePath = filePath;
}
@Override
@ -483,27 +519,35 @@ public class AndroidChunkUploadServiceImpl implements AndroidChunkUploadService
@Override
public boolean isEmpty() {
return bytes.length == 0;
try {
return Files.size(filePath) == 0;
} catch (IOException ex) {
return true;
}
}
@Override
public long getSize() {
return bytes.length;
try {
return Files.size(filePath);
} catch (IOException ex) {
return 0;
}
}
@Override
public byte[] getBytes() {
return bytes;
public byte[] getBytes() throws IOException {
return Files.readAllBytes(filePath);
}
@Override
public InputStream getInputStream() {
return new ByteArrayInputStream(bytes);
public InputStream getInputStream() throws IOException {
return Files.newInputStream(filePath);
}
@Override
public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
Files.write(dest.toPath(), bytes);
Files.copy(filePath, dest.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
}
}
}

View File

@ -44,6 +44,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@Service
@ -289,6 +290,7 @@ public class LegacyMeetingAdapterServiceImpl implements LegacyMeetingAdapterServ
String relocatedUrl = meetingDomainSupport.relocateAudioUrl(meetingId, stagingUrl);
taskSecurityContextRunner.runAsTenantUser(meeting.getTenantId(), meeting.getCreatorId(), () -> {
meetingDomainSupport.applyMeetingAudioMetadata(meeting, relocatedUrl);
meetingDomainSupport.prewarmPlaybackAudioAfterCommit(relocatedUrl);
meeting.setSummaryModelId(profile.getResolvedSummaryModelId());
meeting.setPromptId(profile.getResolvedPromptId());
meeting.setAudioSaveStatus(RealtimeMeetingAudioStorageService.STATUS_SUCCESS);

View File

@ -28,14 +28,14 @@ public interface MeetingCommandService {
void deleteMeeting(Long id);
void appendRealtimeTranscripts(Long meetingId, List<RealtimeTranscriptItemDTO> items);
void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult);
void completeRealtimeMeeting(Long meetingId, String audioUrl, boolean overwriteAudio);
void finishOfflineMeeting(Long meetingId, String finishStage);
void failOfflineTranscription(Long meetingId, String failureMessage);
void updateSpeakerInfo(Long meetingId, String speakerId, String newName, String label);
void updateMeetingTranscript(UpdateMeetingTranscriptCommand command);

View File

@ -16,7 +16,10 @@ public interface MeetingQueryService {
MeetingVO getDetail(Long id);
MeetingVO getDetailIgnoreTenant(Long id);
default MeetingVO getDetailIgnoreTenant(Long id){
return getDetailIgnoreTenant(id,true);
};
MeetingVO getDetailIgnoreTenant(Long id,Boolean includeAudio);
List<MeetingTranscriptVO> getTranscripts(Long meetingId);

View File

@ -11,6 +11,8 @@ public interface RealtimeMeetingSessionStateService {
void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig);
void rememberSpeakerContext(Long meetingId, String speakerContextId);
void assertCanOpenSession(Long meetingId);
boolean activate(Long meetingId, String connectionId);
@ -25,6 +27,8 @@ public interface RealtimeMeetingSessionStateService {
void refreshAfterTranscript(Long meetingId);
void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount);
boolean markCompletingIfResumeExpired(Long meetingId);
void expireEmptySession(Long meetingId);

View File

@ -11,6 +11,7 @@ import com.imeeting.dto.biz.AiLocalProfileVO;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.entity.biz.AsrModel;
import com.imeeting.entity.biz.LlmModel;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.mapper.biz.AsrModelMapper;
import com.imeeting.mapper.biz.LlmModelMapper;
import com.imeeting.service.biz.AiModelService;
@ -44,6 +45,10 @@ public class AiModelServiceImpl implements AiModelService {
private static final String TYPE_ASR = "ASR";
private static final String TYPE_LLM = "LLM";
private static final String TENCENT_PROVIDER = "tencent";
private static final String MEDIA_TENCENT_APP_ID = "tencentAppId";
private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId";
private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey";
private static final int DEFAULT_SORT_ORDER = 0;
private static final String DEFAULT_LLM_API_PATH = "/v1/chat/completions";
private static final String DEFAULT_ANTHROPIC_API_PATH = "/messages";
@ -169,7 +174,7 @@ public class AiModelServiceImpl implements AiModelService {
if (resolvedBaseUrl == null || resolvedBaseUrl.isBlank()) {
return Collections.emptyList();
}
if ("custom".equals(providerKey)) {
if (ModelProviderEnum.LOCAL.getCode().equals(providerKey)) {
return fetchLocalProfile(resolvedBaseUrl, apiKey).getAsrModels();
}
String targetUrl = resolveModelListUrl(providerKey, resolvedBaseUrl, apiKey);
@ -464,7 +469,7 @@ public class AiModelServiceImpl implements AiModelService {
}
private String resolveModelListUrl(String providerKey, String baseUrl, String apiKey) {
if ("Custom".equalsIgnoreCase(providerKey)) {
if (ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(providerKey)) {
return baseUrl+"/api/asrconfig";
}
if ("gemini".equals(providerKey) || "google".equals(providerKey)) {
@ -740,6 +745,7 @@ public class AiModelServiceImpl implements AiModelService {
if (Integer.valueOf(1).equals(dto.getIsDefault()) && !Integer.valueOf(1).equals(dto.getStatus())) {
throw new RuntimeException("默认模型必须为启用状态");
}
validateTencentAsrConfig(dto);
// if ("custom".equals(normalizeProvider(dto.getProvider()))) {
// if (TYPE_ASR.equals(normalizeType(dto.getModelType()))) {
// Map<String, Object> mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig();
@ -832,13 +838,20 @@ public class AiModelServiceImpl implements AiModelService {
}
private void pushAsrConfig(AsrModel entity) {
if ("custom".equals(normalizeProvider(entity.getProvider()))) {
String provider = normalizeProvider(entity.getProvider());
if (ModelProviderEnum.LOCAL.getCode().equals(provider)) {
if (entity.getApiKey() == null || entity.getApiKey().isBlank()) {
log.info("Skip syncing local ASR profile because apiKey is blank, modelName={}", entity.getModelName());
return;
}
updateLocalProfile(entity);
return;
}
if ("custom".equals(provider)) {
return;
}
if (TENCENT_PROVIDER.equals(provider)) {
return;
}
if (entity.getBaseUrl() == null || entity.getBaseUrl().isBlank()) {
throw new RuntimeException("ASR 模型必须配置 baseUrl");
@ -934,6 +947,28 @@ public class AiModelServiceImpl implements AiModelService {
return text.isEmpty() ? null : text;
}
private void validateTencentAsrConfig(AiModelDTO dto) {
if (!TYPE_ASR.equals(normalizeType(dto.getModelType()))) {
return;
}
if (!TENCENT_PROVIDER.equals(normalizeProvider(dto.getProvider()))) {
return;
}
Map<String, Object> mediaConfig = dto.getMediaConfig() == null ? Collections.emptyMap() : dto.getMediaConfig();
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_APP_ID)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentAppId");
}
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_ID)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretId");
}
if (readConfigString(mediaConfig.get(MEDIA_TENCENT_SECRET_KEY)) == null) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey");
}
if (dto.getModelCode() == null || dto.getModelCode().isBlank()) {
throw new RuntimeException("腾讯实时 ASR 模型必须配置 modelCode");
}
}
private BigDecimal readConfigDecimal(Object value) {
if (value == null) {
return null;

View File

@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.common.SysParamKeys;
import com.imeeting.common.MeetingProgressStage;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.dto.biz.MeetingSummarySource;
@ -66,6 +67,8 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
private static final Duration ASR_SUBMIT_REQUEST_TIMEOUT = Duration.ofSeconds(30);
private static final Duration ASR_QUERY_REQUEST_TIMEOUT = Duration.ofSeconds(30);
private static final String DISPATCH_MODE_PARALLEL = "PARALLEL";
private static final String DISPATCH_MODE_SERIAL = "SERIAL";
private final MeetingMapper meetingMapper;
private final MeetingTranscriptMapper transcriptMapper;
@ -278,6 +281,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
failPendingSummaryTask(sumTask, "没有可用于总结的转录内容");
updateMeetingStatus(meetingId, 4);
updateProgress(meetingId, -1, "未识别到可用于总结的转录内容", 0);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.FAILED_TRANSCRIBING.getCode());
return;
}
if (!asrText.isBlank()) {
@ -287,8 +291,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
sumTask == null ? null : sumTask.getId());
meetingProgressService.markStage(meetingId, asrTask, 1, MeetingProgressStage.ASR_COMPLETED, 80, "转写完成,准备生成总结", 0);
scheduleQueuedAsrTasks();
self.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
dispatchPostAsrTasks(meeting, chapterTask, sumTask);
return;
}
if (sumTask != null && canExecuteTask(sumTask)) {
@ -342,8 +345,20 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
return;
}
executeChapterFlow(meeting, chapterTask);
if (shouldRunSummaryAfterChapter(meeting, chapterTask)) {
AiTask summaryTask = findLatestTask(meetingId, "SUMMARY");
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
return;
}
}
reconcileMeetingStatus(meetingId);
androidMeetingPushService.pushMeetingStatusChanged(meetingId, UnifiedMeetingStatusStage.COMPLETED.getCode());
androidMeetingPushService.pushMeetingStatusChanged(
meetingId,
MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)
? UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode()
: UnifiedMeetingStatusStage.COMPLETED.getCode()
);
log.info("[CHAPTER-FLOW] 章节任务流程结束: meetingId={}, chapterTaskId={}, costMs={}",
meetingId, chapterTask.getId(), System.currentTimeMillis() - startMillis);
}
@ -1106,7 +1121,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
meetingPointsService.recordSummarySuccessCharge(meeting, taskRecord);
AiTask latestChapterTask = findLatestTask(meeting.getId(), "CHAPTER");
if (latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus())) {
if (!resolveAiCatalogEnabled() ||(latestChapterTask != null && Integer.valueOf(2).equals(latestChapterTask.getStatus()))) {
updateProgress(meeting.getId(), 100, "全流程分析完成", 0);
} else {
updateProgress(meeting.getId(), 95, "总结生成完成,等待 AI 目录完成...", 0);
@ -1177,7 +1192,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
log.info("[SUMMARY-EXEC] 已获取总结锁,开始构建总结来源: meetingId={}, sumTaskId={}",
meeting.getId(), sumTask == null ? null : sumTask.getId());
try {
MeetingSummarySource summarySource = buildRawTranscriptSummarySource(meeting);
MeetingSummarySource summarySource = buildSummarySourceForExecution(meeting, sumTask);
if (summarySource.getText() == null || summarySource.getText().isBlank()) {
log.warn("[SUMMARY-EXEC] 无转录内容,无法生成总结: meetingId={}, sumTaskId={}",
meeting.getId(), sumTask == null ? null : sumTask.getId());
@ -1207,6 +1222,85 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
.build();
}
private MeetingSummarySource buildSummarySourceForExecution(Meeting meeting, AiTask sumTask) {
if (shouldUseChapterBackedSummarySource()) {
return meetingTranscriptChapterService.resolveSummarySource(meeting, sumTask);
}
return buildRawTranscriptSummarySource(meeting);
}
private void dispatchPostAsrTasks(Meeting meeting, AiTask chapterTask, AiTask summaryTask) {
if (meeting == null) {
return;
}
if (!resolveAiCatalogEnabled()) {
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
return;
}
if (isSerialDispatchMode()) {
if (chapterTask != null && canExecuteTask(chapterTask)) {
self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
} else {
log.warn("[ASR-FLOW] 串行模式下缺少可执行章节任务,跳过总结派发: meetingId={}, chapterTaskId={}, chapterStatus={}",
meeting.getId(),
chapterTask == null ? null : chapterTask.getId(),
chapterTask == null ? null : chapterTask.getStatus());
}
return;
}
if (chapterTask != null && canExecuteTask(chapterTask)) {
self.dispatchChapterTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
if (summaryTask != null && canExecuteTask(summaryTask)) {
self.dispatchSummaryTask(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId());
}
}
private boolean shouldRunSummaryAfterChapter(Meeting meeting, AiTask chapterTask) {
return meeting != null
&& resolveAiCatalogEnabled()
&& isSerialDispatchMode()
&& isTaskCompleted(chapterTask)
&& !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED);
}
private boolean shouldUseChapterBackedSummarySource() {
return resolveAiCatalogEnabled() && isSerialDispatchMode();
}
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private boolean isSerialDispatchMode() {
return DISPATCH_MODE_SERIAL.equals(resolveSummaryDispatchMode());
}
private String resolveSummaryDispatchMode() {
if (sysParamService == null) {
return DISPATCH_MODE_PARALLEL;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, DISPATCH_MODE_PARALLEL);
if (rawValue == null || rawValue.isBlank()) {
return DISPATCH_MODE_PARALLEL;
}
String normalized = rawValue.trim().toUpperCase(Locale.ROOT);
return DISPATCH_MODE_SERIAL.equals(normalized) ? DISPATCH_MODE_SERIAL : DISPATCH_MODE_PARALLEL;
}
private AiTask findLatestTask(Long meetingId, String taskType) {
return this.getOne(new LambdaQueryWrapper<AiTask>()
.eq(AiTask::getMeetingId, meetingId)
@ -1275,7 +1369,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
updateMeetingStatus(meetingId, 4);
return;
}
if (isTaskCompleted(chapterTask) && isTaskCompleted(summaryTask)) {
if ( isTaskCompleted(summaryTask) && (!resolveAiCatalogEnabled() || isTaskCompleted(chapterTask))) {
updateMeetingStatus(meetingId, 3);
return;
}
@ -1488,6 +1582,7 @@ public class AiTaskServiceImpl extends ServiceImpl<AiTaskMapper, AiTask> impleme
if ("SUMMARY".equals(task.getTaskType())) {
meetingPointsService.markSummaryChargeFailed(task.getId(), error);
}
androidMeetingPushService.pushMeetingStatusChanged(task.getMeetingId(), UnifiedMeetingStatusStage.FAILED_SUMMARIZING.getCode());
}
private String buildAsrFailureMessage(Exception ex) {

View File

@ -62,6 +62,7 @@ public class MeetingAudioUploadSupport {
Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
}
validateStoredAudio(targetPath, extension);
} catch (Exception ex) {
Files.deleteIfExists(targetPath);
throw ex;
@ -69,6 +70,55 @@ public class MeetingAudioUploadSupport {
return buildStagingAudioToken(storedFileName);
}
public String storeUploadedAudioFromPath(Path sourceFile, String originalFilename) throws IOException {
if (sourceFile == null || !Files.exists(sourceFile)) {
throw new RuntimeException("音频文件不能为空");
}
long fileSize = Files.size(sourceFile);
long maxUploadSizeMb = resolveMaxUploadSizeMb();
long maxUploadSizeBytes = maxUploadSizeMb * 1024 * 1024;
if (fileSize > maxUploadSizeBytes) {
throw new RuntimeException("音频文件大小不能超过 " + maxUploadSizeMb + "MB");
}
String extension = resolveExtension(originalFilename);
validateFileHeaderFromPath(sourceFile, extension);
Path stagingDir = resolveStagingAudioDirectory(uploadPath);
Files.createDirectories(stagingDir);
String storedFileName = UUID.randomUUID() + "." + extension;
Path targetPath = stagingDir.resolve(storedFileName);
try {
Files.move(sourceFile, targetPath, StandardCopyOption.REPLACE_EXISTING);
validateStoredAudio(targetPath, extension);
} catch (Exception ex) {
Files.deleteIfExists(targetPath);
throw ex;
}
return buildStagingAudioToken(storedFileName);
}
private void validateFileHeaderFromPath(Path sourceFile, String extension) throws IOException {
if (Files.size(sourceFile) <= 0) {
return;
}
byte[] header;
try (InputStream inputStream = Files.newInputStream(sourceFile)) {
header = inputStream.readNBytes(HEADER_SIZE);
}
boolean valid = switch (extension) {
case "wav" -> isWav(header);
case "mp3" -> isMp3(header);
case "m4a" -> isM4a(header);
default -> false;
};
if (!valid) {
throw new RuntimeException("上传文件内容与音频格式不匹配,仅支持 mp3、wav、m4a");
}
}
public static boolean isStagingAudioToken(String audioUrl) {
return StringUtils.hasText(audioUrl) && audioUrl.startsWith(STAGING_AUDIO_TOKEN_PREFIX);
}

View File

@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.common.MeetingConstants;
import com.imeeting.common.RedisKeys;
import com.imeeting.common.SysParamKeys;
import com.imeeting.dto.android.AndroidPendingMeetingDraft;
import com.imeeting.dto.biz.CreateMeetingCommand;
import com.imeeting.dto.biz.CreateRealtimeMeetingCommand;
@ -27,6 +28,7 @@ import com.imeeting.entity.biz.HotWord;
import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.entity.biz.MeetingTranscriptChapterVersion;
import com.imeeting.enums.BusinessErrorCodeEnum;
import com.imeeting.enums.MeetingStatusEnum;
import com.imeeting.service.android.AndroidPendingMeetingDraftService;
import com.imeeting.service.android.AndroidPushMessageService;
@ -45,6 +47,10 @@ import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.imeeting.support.redis.MeetingAsrPermitCache;
import com.imeeting.support.redis.MeetingLockCache;
import com.unisbase.common.exception.BusinessException;
import com.unisbase.common.exception.ErrorCodeEnum;
import com.imeeting.websocket.RealtimeMeetingProxyWebSocketHandler;
import com.unisbase.service.SysParamService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -77,7 +83,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
private final MeetingDomainSupport meetingDomainSupport;
private final MeetingRuntimeProfileResolver meetingRuntimeProfileResolver;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService;
private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService;
private final RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler;
private final MeetingProgressService meetingProgressService;
private final MeetingPointsService meetingPointsService;
private final MeetingSummaryPromptAssembler meetingSummaryPromptAssembler;
@ -87,6 +94,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
private final AndroidPendingMeetingDraftService androidPendingMeetingDraftService;
private final MeetingLockCache meetingLockCache;
private final MeetingAsrPermitCache meetingAsrPermitCache;
private final SysParamService sysParamService;
@Value("${imeeting.summary-orchestration.mode:INTERNAL_BUILTIN}")
private String summaryOrchestrationMode;
@ -103,7 +111,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
MeetingDomainSupport meetingDomainSupport,
MeetingRuntimeProfileResolver meetingRuntimeProfileResolver,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService,
RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService,
RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService, RealtimeMeetingProxyWebSocketHandler realtimeMeetingProxyWebSocketHandler,
MeetingProgressService meetingProgressService,
MeetingPointsService meetingPointsService,
MeetingSummaryPromptAssembler meetingSummaryPromptAssembler,
@ -112,7 +120,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
AndroidPushMessageService androidPushMessageService,
AndroidPendingMeetingDraftService androidPendingMeetingDraftService,
MeetingLockCache meetingLockCache,
MeetingAsrPermitCache meetingAsrPermitCache) {
MeetingAsrPermitCache meetingAsrPermitCache,
SysParamService sysParamService) {
this.meetingService = meetingService;
this.aiTaskService = aiTaskService;
this.hotWordService = hotWordService;
@ -124,7 +133,8 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
this.meetingDomainSupport = meetingDomainSupport;
this.meetingRuntimeProfileResolver = meetingRuntimeProfileResolver;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService;
this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService;
this.realtimeMeetingAudioStorageService = realtimeMeetingAudioStorageService;
this.realtimeMeetingProxyWebSocketHandler = realtimeMeetingProxyWebSocketHandler;
this.meetingProgressService = meetingProgressService;
this.meetingPointsService = meetingPointsService;
this.meetingSummaryPromptAssembler = meetingSummaryPromptAssembler;
@ -134,6 +144,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
this.androidPendingMeetingDraftService = androidPendingMeetingDraftService;
this.meetingLockCache = meetingLockCache;
this.meetingAsrPermitCache = meetingAsrPermitCache;
this.sysParamService = sysParamService;
}
@Override
@ -176,7 +187,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
aiTaskService.save(asrTask);
Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId();
meetingDomainSupport.createChapterTask(
createChapterTaskIfEnabled(
meeting.getId(),
runtimeProfile.getResolvedSummaryModelId(),
chapterModelId,
@ -228,7 +239,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
hostUserId, hostName, runtimeProfile.getResolvedSummaryModelId(), runtimeProfile.getResolvedPromptId(), summaryDetailLevel, 0);
meetingService.save(meeting);
Long chapterModelId = command.getChapterModelId() != null ? command.getChapterModelId() : runtimeProfile.getResolvedSummaryModelId();
meetingDomainSupport.createChapterTask(
createChapterTaskIfEnabled(
meeting.getId(),
runtimeProfile.getResolvedSummaryModelId(),
chapterModelId,
@ -339,58 +350,6 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
deleteMeetingArtifactsAfterCommit(id);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void appendRealtimeTranscripts(Long meetingId, List<RealtimeTranscriptItemDTO> items) {
if (items == null || items.isEmpty()) {
return;
}
Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.orderByDesc(MeetingTranscript::getSortOrder)
.last("LIMIT 1"))
.stream()
.findFirst()
.map(MeetingTranscript::getSortOrder)
.orElse(0);
int nextSortOrder = maxSortOrder == null ? 0 : maxSortOrder + 1;
boolean inserted = false;
for (RealtimeTranscriptItemDTO item : items) {
if (item.getContent() == null || item.getContent().isBlank()) {
continue;
}
MeetingTranscript existing = transcriptMapper.selectOne(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.eq(MeetingTranscript::getContent, item.getContent().trim())
.eq(item.getSpeakerId() != null && !item.getSpeakerId().isBlank(), MeetingTranscript::getSpeakerId, item.getSpeakerId())
.eq(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime())
.eq(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime())
.last("LIMIT 1"));
if (existing != null) {
continue;
}
MeetingTranscript transcript = new MeetingTranscript();
transcript.setMeetingId(meetingId);
transcript.setSpeakerId(meetingDomainSupport.resolveSpeakerId(item.getSpeakerId()));
transcript.setSpeakerName(meetingDomainSupport.resolveSpeakerName(item.getSpeakerId(), item.getSpeakerName()));
transcript.setContent(item.getContent().trim());
transcript.setStartTime(item.getStartTime());
transcript.setEndTime(item.getEndTime());
transcript.setSortOrder(nextSortOrder++);
transcriptMapper.insert(transcript);
inserted = true;
}
if (inserted) {
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
realtimeMeetingSessionStateService.refreshAfterTranscript(meetingId);
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public void saveRealtimeTranscriptSnapshot(Long meetingId, RealtimeTranscriptItemDTO item, boolean finalResult) {
@ -449,11 +408,11 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
if (meeting == null) {
throw new RuntimeException("会议不存在");
}
realtimeMeetingProxyWebSocketHandler.closeMeetingSession(meetingId);
RealtimeMeetingSessionStatusVO currentStatus = realtimeMeetingSessionStateService.getStatus(meetingId);
if (overwriteAudio) {
if (audioUrl == null || audioUrl.isBlank()) {
throw new RuntimeException("overwriteAudio 为 true 时必须提供音频地址");
throw new RuntimeException("overwriteAudio=true requires audioUrl");
}
meetingDomainSupport.applyMeetingAudioMetadata(
meeting,
@ -494,10 +453,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
realtimeMeetingSessionStateService.clear(meetingId);
meeting.setStatus(MeetingStatusEnum.SUMMARIZING.getCode());
meetingService.updateById(meeting);
updateMeetingProgress(meetingId, 85, "正在生成 AI 目录与总结...", 0);
updateMeetingProgress(meetingId, resolveAiCatalogEnabled() ? 85 : 90, resolveAiCatalogEnabled() ? "正在生成 AI 目录与总结..." : "正在生成会议总结...", 0);
meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl());
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
if (!resolveAiCatalogEnabled()) {
aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
} else if (isParallelDispatchMode()) {
aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
} else {
aiTaskService.dispatchChapterTask(meetingId, meeting.getTenantId(), meeting.getCreatorId());
}
}
@Override
@ -522,6 +487,22 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
meetingService.updateById(meeting);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void failOfflineTranscription(Long meetingId, String failureMessage) {
Meeting meeting = meetingService.getById(meetingId);
if (meeting == null) {
throw new BusinessException(BusinessErrorCodeEnum.MEETING_NOT_FOUND.getCode(),"会议不存在");
}
if (!MeetingConstants.TYPE_OFFLINE.equals(meeting.getMeetingType())) {
throw new RuntimeException("会议不是离线会议");
}
meeting.setStatus(MeetingStatusEnum.FAILED.getCode());
meeting.setOfflineRecordingStatus(MeetingConstants.OFFLINE_RECORDING_UPLOAD_FINISHED);
markAudioSaveFailure(meeting, failureMessage);
meetingService.updateById(meeting);
}
private void applyRealtimeAudioFinalizeResult(Meeting meeting, RealtimeMeetingAudioStorageService.FinalizeResult result) {
if (result == null) {
markAudioSaveFailure(meeting, RealtimeMeetingAudioStorageService.DEFAULT_FAILURE_MESSAGE);
@ -767,6 +748,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
@Transactional(rollbackFor = Exception.class)
public MeetingTranscriptChapterImportResultVO importTranscriptChapters(MeetingTranscriptChapterImportDTO command) {
ensureExternalSummaryModeEnabled();
ensureAiCatalogEnabled();
if (command == null || command.getMeetingId() == null) {
throw new RuntimeException("缺少会议ID无法导入章节");
}
@ -787,6 +769,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
.last("LIMIT 1"));
if (latestChapterTask == null) {
ensureAiCatalogEnabled();
Long summaryModelId = resolveSummaryModelId(command, latestSummaryTask);
Long chapterModelId = resolveChapterModelId(command, latestSummaryTask, summaryModelId);
Long promptId = resolvePromptId(command, latestSummaryTask);
@ -1120,7 +1103,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
.eq(AiTask::getTaskType, "CHAPTER")
.orderByDesc(AiTask::getId)
.last("LIMIT 1"));
if (chapterTask == null) {
if (resolveAiCatalogEnabled() && chapterTask == null) {
chapterTask = meetingDomainSupport.createChapterTask(
meetingId,
effectiveSummaryModelId,
@ -1129,7 +1112,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
effectiveUserPrompt,
effectiveSummaryDetailLevel
);
} else {
} else if (resolveAiCatalogEnabled()) {
resetAiTask(chapterTask, meetingSummaryPromptAssembler.buildTaskConfig(
effectiveSummaryModelId,
effectiveChapterModelId,
@ -1185,6 +1168,16 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
if (!Integer.valueOf(3).equals(summaryTask.getStatus())) {
throw new RuntimeException("当前总结环节未失败,无需重试");
}
if (resolveAiCatalogEnabled() && isSerialDispatchMode()) {
AiTask chapterTask = aiTaskService.getOne(new LambdaQueryWrapper<AiTask>()
.eq(AiTask::getMeetingId, meetingId)
.eq(AiTask::getTaskType, "CHAPTER")
.orderByDesc(AiTask::getId)
.last("LIMIT 1"));
if (chapterTask == null || !Integer.valueOf(2).equals(chapterTask.getStatus())) {
throw new RuntimeException("串行模式下缺少成功的 AI 目录产物,无法重试总结");
}
}
Long effectiveSummaryModelId = resolveMeetingSummaryModelId(meeting, summaryTask);
resetAiTask(summaryTask, buildSummaryTaskConfigForRetry(
summaryTask,
@ -1207,6 +1200,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
@Override
@Transactional(rollbackFor = Exception.class)
public void retryChapter(Long meetingId) {
ensureAiCatalogEnabled();
Meeting meeting = meetingService.getById(meetingId);
if (meeting == null) {
throw new RuntimeException("会议不存在");
@ -1248,6 +1242,25 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
dispatchChapterTaskAfterCommit(meetingId, meeting.getTenantId(), meeting.getCreatorId());
}
private AiTask createChapterTaskIfEnabled(Long meetingId,
Long summaryModelId,
Long chapterModelId,
Long promptId,
String userPrompt,
String summaryDetailLevel) {
if (!resolveAiCatalogEnabled()) {
return null;
}
return meetingDomainSupport.createChapterTask(
meetingId,
summaryModelId,
chapterModelId,
promptId,
userPrompt,
summaryDetailLevel
);
}
private void clearLegacyDispatchState(Long meetingId) {
if (meetingId == null) {
return;
@ -1586,4 +1599,45 @@ public class MeetingCommandServiceImpl implements MeetingCommandService {
}
return MeetingConstants.SUMMARY_DETAIL_STANDARD;
}
private void ensureAiCatalogEnabled() {
if (!resolveAiCatalogEnabled()) {
throw new RuntimeException("AI目录功能未开启");
}
}
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private boolean isSerialDispatchMode() {
return "SERIAL".equals(resolveSummaryDispatchMode());
}
private boolean isParallelDispatchMode() {
return "PARALLEL".equals(resolveSummaryDispatchMode());
}
private String resolveSummaryDispatchMode() {
if (sysParamService == null) {
return "PARALLEL";
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_SUMMARY_DISPATCH_MODE, "PARALLEL");
if (rawValue == null || rawValue.isBlank()) {
return "PARALLEL";
}
String normalized = rawValue.trim().toUpperCase();
return "SERIAL".equals(normalized) ? "SERIAL" : "PARALLEL";
}
}

View File

@ -1,7 +1,9 @@
package com.imeeting.service.biz.impl;
import cn.hutool.core.date.StopWatch;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.common.MeetingConstants;
import com.imeeting.common.SysParamKeys;
import com.imeeting.entity.biz.AiTask;
import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript;
@ -13,6 +15,7 @@ import com.imeeting.service.biz.MeetingSummaryFileService;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.unisbase.entity.SysUser;
import com.unisbase.mapper.SysUserMapper;
import com.unisbase.service.SysParamService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@ -54,6 +57,7 @@ public class MeetingDomainSupport {
private final ApplicationEventPublisher eventPublisher;
private final MeetingSummaryFileService meetingSummaryFileService;
private final MeetingPlaybackAudioResolver meetingPlaybackAudioResolver;
private final SysParamService sysParamService;
@Value("${unisbase.app.upload-path}")
private String uploadPath;
@ -399,6 +403,7 @@ public class MeetingDomainSupport {
vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus());
vo.setSummaryModelId(meeting.getSummaryModelId());
vo.setPromptId(meeting.getPromptId());
vo.setAiCatalogEnabled(resolveAiCatalogEnabled());
vo.setSummaryDetailLevel(normalizeSummaryDetailLevel(meeting.getSummaryDetailLevel()));
vo.setAudioSaveStatus(meeting.getAudioSaveStatus());
vo.setAudioSaveMessage(meeting.getAudioSaveMessage());
@ -648,6 +653,21 @@ public class MeetingDomainSupport {
return path != null && path.contains("\\");
}
public boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private Path resolvePublicAudioPath(String audioUrl) {
if (audioUrl == null || audioUrl.isBlank()) {
return null;

View File

@ -485,7 +485,7 @@ public class MeetingPlaybackAudioResolver {
"-vn",
"-ar", String.valueOf(BROWSER_SAMPLE_RATE),
"-c:a", "aac",
"-f", "mp4",
"-ac", "1",
targetPath.toString()
);
executeCommand(command, targetPath);

View File

@ -241,6 +241,14 @@ public class MeetingProgressServiceImpl implements MeetingProgressService {
if (latestChapter != null && Integer.valueOf(1).equals(latestChapter.getStatus())) {
return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0);
}
if (MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.SUMMARIZING)) {
if (latestSummary != null && Integer.valueOf(2).equals(latestSummary.getStatus())) {
return buildSnapshot(meetingId, latestSummary, meeting.getStatus(), MeetingProgressStage.SUMMARY_RUNNING, 90, "正在生成会议总结...", 0);
}
if (latestChapter != null && Integer.valueOf(0).equals(latestChapter.getStatus())) {
return buildSnapshot(meetingId, latestChapter, meeting.getStatus(), MeetingProgressStage.CHAPTER_RUNNING, 85, "正在生成会议章节...", 0);
}
}
AiTask latestAsr = findLatestTask(meetingId, "ASR");
if (latestAsr != null) {
if (Integer.valueOf(1).equals(latestAsr.getStatus())) {

View File

@ -1,5 +1,6 @@
package com.imeeting.service.biz.impl;
import cn.hutool.core.date.StopWatch;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.imeeting.dto.biz.MeetingSummaryPromptContextRequestDTO;
@ -19,6 +20,7 @@ import com.imeeting.service.biz.MeetingService;
import com.imeeting.service.biz.MeetingTranscriptChapterService;
import com.unisbase.dto.PageResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
@ -27,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
@Service
@RequiredArgsConstructor
public class MeetingQueryServiceImpl implements MeetingQueryService {
@ -83,10 +86,12 @@ public class MeetingQueryServiceImpl implements MeetingQueryService {
return meeting != null ? toVO(meeting, true) : null;
}
@Override
public MeetingVO getDetailIgnoreTenant(Long id) {
public MeetingVO getDetailIgnoreTenant(Long id, Boolean includeAudio) {
Meeting meeting = meetingMapper.selectByIdIgnoreTenant(id);
return meeting != null ? toVO(meeting, true) : null;
return meeting != null ? toVO(meeting, includeAudio) : null;
}
@Override

View File

@ -369,28 +369,29 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha
private String buildChapterSystemPrompt() {
return """
JSON
chapters
JSONchapters ,
chapterNo,title,summary,keywords,startTranscriptId,endTranscriptId,confidence
transcript
- startTranscriptId transcriptId
- startTranscriptId endTranscriptId + 1
- transcript
- transcript
- transcript
-
-
transcript .
transcript
JSON
{
"chapters": [
{
"chapterNo": number,
"title": string,
"summary": string,
"keywords": [string],
"startTranscriptId": number,
"endTranscriptId": number,
"confidence": number
}
]
}
1.
2. transcript
3.
4. title
5. JSON
""";
}
@ -1017,9 +1018,9 @@ public class MeetingTranscriptChapterServiceImpl implements MeetingTranscriptCha
Long startTranscriptId = longValue(item.path("startTranscriptId").asText(null));
Long endTranscriptId = longValue(item.path("endTranscriptId").asText(null));
Integer chapterNo = item.path("chapterNo").isInt() ? item.path("chapterNo").asInt() : null;
if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) {
throw new RuntimeException("章节模型返回了不完整的章节边界");
}
// if (chapterNo == null || startTranscriptId == null || endTranscriptId == null) {
// throw new RuntimeException("章节模型返回了不完整的章节边界");
// }
List<String> keywords = new ArrayList<>();
if (item.path("keywords").isArray()) {
for (JsonNode keyword : item.path("keywords")) {

View File

@ -2,6 +2,7 @@ package com.imeeting.service.biz.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.common.MeetingConstants;
import com.imeeting.common.SysParamKeys;
import com.imeeting.dto.biz.MeetingProgressSnapshot;
import com.imeeting.dto.biz.MeetingVO;
import com.imeeting.dto.biz.UnifiedMeetingStatusStage;
@ -17,6 +18,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptChapterVersionMapper;
import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.MeetingUnifiedStatusService;
import com.imeeting.support.redis.MeetingProgressCache;
import com.unisbase.service.SysParamService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@ -31,6 +33,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
private final MeetingTranscriptMapper meetingTranscriptMapper;
private final MeetingTranscriptChapterVersionMapper chapterVersionMapper;
private final MeetingProgressCache meetingProgressCache;
private final SysParamService sysParamService;
@Override
public UnifiedMeetingStatusVO resolve(MeetingVO meeting, MeetingProgressSnapshot snapshot) {
@ -87,12 +90,12 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
if (isAndroidOfflineMeetingWaitingUpload(meeting)) {
return UnifiedMeetingStatusStage.WAITING_UPLOAD;
}
UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot);
MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot);
UnifiedMeetingStatusStage stageFromSnapshot = resolveStageFromSnapshot(snapshot, context);
if (stageFromSnapshot != null) {
return stageFromSnapshot;
}
MeetingUnifiedStageContext context = buildStageContext(meeting.getId(), snapshot);
if (isTranscribing(context)) {
return UnifiedMeetingStatusStage.TRANSCRIBING;
}
@ -103,7 +106,8 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return UnifiedMeetingStatusStage.INITIALIZING;
}
private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot) {
private UnifiedMeetingStatusStage resolveStageFromSnapshot(MeetingProgressSnapshot snapshot,
MeetingUnifiedStageContext context) {
if (snapshot == null || snapshot.getStage() == null || snapshot.getStage().isBlank()) {
return null;
}
@ -112,15 +116,31 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
case "completed" -> UnifiedMeetingStatusStage.COMPLETED;
case "summary_running", "chapter_running" -> UnifiedMeetingStatusStage.SUMMARIZING;
case "asr_running", "asr_completed", "asr_submitted" -> UnifiedMeetingStatusStage.TRANSCRIBING;
case "queued" -> UnifiedMeetingStatusStage.INITIALIZING;
case "queued" -> resolveQueuedSnapshotStage(context);
default -> null;
};
}
private UnifiedMeetingStatusStage resolveQueuedSnapshotStage(MeetingUnifiedStageContext context) {
if (context == null) {
return UnifiedMeetingStatusStage.INITIALIZING;
}
if (isSummarizing(context)) {
return UnifiedMeetingStatusStage.SUMMARIZING;
}
if (isTranscribing(context)) {
return UnifiedMeetingStatusStage.TRANSCRIBING;
}
return UnifiedMeetingStatusStage.INITIALIZING;
}
private UnifiedMeetingStatusStage resolveFailedStage(MeetingVO meeting) {
if (meeting == null || !MeetingStatusEnum.isCode(meeting.getStatus(), MeetingStatusEnum.FAILED)) {
return null;
}
if (isAndroidOfflineEmptyUploadFailure(meeting)) {
return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING;
}
AiTask asrTask = findLatestTask(meeting.getId(), "ASR");
if (isTaskFailed(asrTask)) {
return UnifiedMeetingStatusStage.FAILED_TRANSCRIBING;
@ -140,6 +160,13 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return UnifiedMeetingStatusStage.FAILED_INITIALIZING;
}
private boolean isAndroidOfflineEmptyUploadFailure(MeetingVO meeting) {
return meeting != null
&& MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType())
&& MeetingConstants.SOURCE_ANDROID.equalsIgnoreCase(meeting.getMeetingSource())
&& "FAILED".equalsIgnoreCase(meeting.getAudioSaveStatus());
}
private boolean isAndroidOfflineMeetingWaitingUpload(MeetingVO meeting) {
return meeting != null
&& MeetingConstants.TYPE_OFFLINE.equalsIgnoreCase(meeting.getMeetingType())
@ -162,6 +189,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
}
private Integer resolvePercent(MeetingProgressSnapshot snapshot, UnifiedMeetingStatusStage stage) {
if (stage != null && stage.getCode().startsWith("FAILED_")) {
return -1;
}
if (snapshot != null && snapshot.getPercent() != null) {
return snapshot.getPercent();
}
@ -203,6 +233,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
if (meeting == null) {
return "处理失败";
}
if (meeting.getAudioSaveMessage() != null && !meeting.getAudioSaveMessage().isBlank()) {
return meeting.getAudioSaveMessage();
}
if (meeting.getLatestSummaryAttemptErrorMsg() != null && !meeting.getLatestSummaryAttemptErrorMsg().isBlank()) {
return meeting.getLatestSummaryAttemptErrorMsg();
}
@ -222,6 +255,9 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
}
private boolean canViewAiChapters(Long meetingId) {
if (!resolveAiCatalogEnabled()) {
return false;
}
return meetingId != null && chapterVersionMapper.selectCount(new LambdaQueryWrapper<MeetingTranscriptChapterVersion>()
.eq(MeetingTranscriptChapterVersion::getMeetingId, meetingId)
.eq(MeetingTranscriptChapterVersion::getIsCurrent, 1)
@ -274,6 +310,7 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
vo.setSourceDeviceCode(meeting.getSourceDeviceCode());
vo.setSourceDeviceMode(meeting.getSourceDeviceMode());
vo.setOfflineRecordingStatus(meeting.getOfflineRecordingStatus());
vo.setAiCatalogEnabled(resolveAiCatalogEnabled());
vo.setSummaryDetailLevel(meeting.getSummaryDetailLevel());
vo.setAudioSaveStatus(meeting.getAudioSaveStatus());
vo.setAudioSaveMessage(meeting.getAudioSaveMessage());
@ -284,6 +321,21 @@ public class MeetingUnifiedStatusServiceImpl implements MeetingUnifiedStatusServ
return vo;
}
private boolean resolveAiCatalogEnabled() {
if (sysParamService == null) {
return false;
}
String rawValue = sysParamService.getCachedParamValue(SysParamKeys.MEETING_AI_CATALOG_ENABLED, "false");
if (rawValue == null || rawValue.isBlank()) {
return false;
}
String normalized = rawValue.trim().toLowerCase();
return "1".equals(normalized)
|| "true".equals(normalized)
|| "yes".equals(normalized)
|| "on".equals(normalized);
}
private record MeetingUnifiedStageContext(AiTask asrTask,
AiTask chapterTask,
AiTask summaryTask,

View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.imeeting.dto.biz.RealtimeMeetingResumeConfig;
import com.imeeting.dto.biz.RealtimeMeetingSessionState;
import com.imeeting.dto.biz.RealtimeMeetingSessionStatusVO;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.entity.biz.Meeting;
import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.enums.MeetingStatusEnum;
@ -12,6 +13,7 @@ import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.support.redis.MeetingLockCache;
import com.imeeting.support.redis.RealtimeMeetingSessionCache;
import com.imeeting.support.redis.RealtimeMeetingTranscriptCache;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@ -31,6 +33,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
private final MeetingLockCache meetingLockCache;
private final MeetingTranscriptMapper transcriptMapper;
private final MeetingMapper meetingMapper;
private final RealtimeMeetingTranscriptCache realtimeMeetingTranscriptCache;
@Value("${imeeting.realtime.resume-window-minutes:30}")
private String resumeWindowMinutesValue;
@ -49,8 +52,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
next.setTenantId(tenantId);
next.setUserId(userId);
next.setStatus("IDLE");
next.setHasTranscript(countTranscripts(meetingId) > 0);
next.setTranscriptCountSnapshot(countTranscripts(meetingId));
long transcriptCount = countCapturedTranscripts(meetingId);
next.setHasTranscript(transcriptCount > 0);
next.setTranscriptCountSnapshot(transcriptCount);
next.setUpdatedAt(System.currentTimeMillis());
writeState(next);
}
@ -59,6 +63,22 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
public void rememberResumeConfig(Long meetingId, RealtimeMeetingResumeConfig resumeConfig) {
RealtimeMeetingSessionState state = getOrCreateState(meetingId);
state.setResumeConfig(resumeConfig);
state.setUpdatedAt(System.currentTimeMillis());
writeState(state);
}
@Override
public void rememberSpeakerContext(Long meetingId, String speakerContextId) {
if (meetingId == null || speakerContextId == null || speakerContextId.isBlank()) {
return;
}
RealtimeMeetingSessionState state = getOrCreateState(meetingId);
RealtimeMeetingResumeConfig resumeConfig = state.getResumeConfig();
if (resumeConfig == null) {
resumeConfig = new RealtimeMeetingResumeConfig();
state.setResumeConfig(resumeConfig);
}
resumeConfig.setSpeakerContextId(speakerContextId.trim());
state.setUpdatedAt(System.currentTimeMillis());
writeState(state);
}
@ -72,16 +92,16 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
String currentStatus = status.getStatus();
if ("COMPLETING".equals(currentStatus)) {
throw new RuntimeException("实时会议正在结束处理中");
throw new RuntimeException("Realtime meeting is completing");
}
if ("COMPLETED".equals(currentStatus)) {
throw new RuntimeException("实时会议已结束");
throw new RuntimeException("Realtime meeting is already completed");
}
if ("ACTIVE".equals(currentStatus) || Boolean.TRUE.equals(status.getActiveConnection())) {
throw new RuntimeException("实时会议已存在活动连接");
throw new RuntimeException("Realtime meeting already has an active connection");
}
if ("PAUSED_RESUMABLE".equals(currentStatus) && !Boolean.TRUE.equals(status.getCanResume())) {
throw new RuntimeException("实时会议恢复窗口已过期");
throw new RuntimeException("Realtime meeting resume window has expired");
}
}
@ -102,7 +122,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
}
long now = System.currentTimeMillis();
long transcriptCount = countTranscripts(meetingId);
long transcriptCount = countCapturedTranscripts(meetingId);
state.setStatus("ACTIVE");
state.setHasTranscript(transcriptCount > 0);
state.setTranscriptCountSnapshot(transcriptCount);
@ -182,9 +202,13 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
@Override
public void refreshAfterTranscript(Long meetingId) {
refreshAfterTranscriptCapture(meetingId, countTranscripts(meetingId));
}
@Override
public void refreshAfterTranscriptCapture(Long meetingId, long transcriptCount) {
RealtimeMeetingSessionState state = getOrCreateState(meetingId);
long now = System.currentTimeMillis();
long transcriptCount = countTranscripts(meetingId);
state.setHasTranscript(transcriptCount > 0);
state.setTranscriptCountSnapshot(transcriptCount);
@ -214,7 +238,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
return false;
}
long transcriptCount = countTranscripts(meetingId);
long transcriptCount = countCapturedTranscripts(meetingId);
if (transcriptCount <= 0) {
clear(meetingId);
return false;
@ -256,7 +280,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
}
private RealtimeMeetingSessionStatusVO pauseState(Long meetingId, RealtimeMeetingSessionState state) {
long transcriptCount = countTranscripts(meetingId);
long transcriptCount = countCapturedTranscripts(meetingId);
long now = System.currentTimeMillis();
state.setHasTranscript(transcriptCount > 0);
@ -307,7 +331,7 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
} else {
vo.setStatus("IDLE");
}
vo.setHasTranscript(countTranscripts(meetingId) > 0);
vo.setHasTranscript(countCapturedTranscripts(meetingId) > 0);
vo.setCanResume(false);
vo.setRemainingSeconds(0L);
vo.setActiveConnection(false);
@ -362,8 +386,9 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
RealtimeMeetingSessionState next = new RealtimeMeetingSessionState();
next.setMeetingId(meetingId);
next.setStatus("IDLE");
next.setHasTranscript(countTranscripts(meetingId) > 0);
next.setTranscriptCountSnapshot(countTranscripts(meetingId));
long transcriptCount = countCapturedTranscripts(meetingId);
next.setHasTranscript(transcriptCount > 0);
next.setTranscriptCountSnapshot(transcriptCount);
next.setUpdatedAt(System.currentTimeMillis());
return next;
}
@ -400,4 +425,21 @@ public class RealtimeMeetingSessionStateServiceImpl implements RealtimeMeetingSe
return transcriptMapper.selectCount(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId));
}
private long countCapturedTranscripts(Long meetingId) {
return Math.max(countTranscripts(meetingId), countCachedTranscripts(meetingId));
}
private long countCachedTranscripts(Long meetingId) {
if (meetingId == null) {
return 0L;
}
RealtimeMeetingTranscriptCacheState state = realtimeMeetingTranscriptCache.getState(meetingId);
if (state == null || state.getItems() == null || state.getItems().isEmpty()) {
return 0L;
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.count();
}
}

View File

@ -10,12 +10,13 @@ import com.imeeting.service.biz.AiModelService;
import com.imeeting.service.biz.MeetingAccessService;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.biz.RealtimeMeetingSocketSessionService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelFactory;
import com.imeeting.support.redis.RealtimeMeetingSocketSessionCache;
import com.unisbase.security.LoginUser;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -30,6 +31,7 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
private final MeetingAccessService meetingAccessService;
private final AiModelService aiModelService;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeAsrChannelFactory realtimeAsrChannelFactory;
@Override
public RealtimeSocketSessionVO createSession(Long meetingId, Long asrModelId, String mode, String language,
@ -37,10 +39,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
Boolean enableTextRefine, Boolean saveAudio,
List<Map<String, Object>> hotwords, LoginUser loginUser) {
if (meetingId == null) {
throw new RuntimeException("会议 ID 不能为空");
throw new RuntimeException("会议 ID 不能为空");
}
if (asrModelId == null) {
throw new RuntimeException("ASR 模型 ID 不能为空");
throw new RuntimeException("ASR 模型 ID 不能为空");
}
Meeting meeting = meetingAccessService.requireMeeting(meetingId);
@ -51,12 +53,13 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
AiModelVO asrModel = aiModelService.getModelById(asrModelId, "ASR");
if (asrModel == null) {
throw new RuntimeException("ASR 模型不存在");
throw new RuntimeException("ASR 模型不存在");
}
String targetWsUrl = resolveWsUrl(asrModel);
RealtimeAsrChannel realtimeAsrChannel = realtimeAsrChannelFactory.getRequired(asrModel.getProvider());
String targetWsUrl = realtimeAsrChannel.resolveTargetWsUrl(asrModel);
if (targetWsUrl == null || targetWsUrl.isBlank()) {
throw new RuntimeException("ASR 模型未配置 WebSocket 地址");
throw new RuntimeException("ASR 模型未配置 WebSocket 地址");
}
RealtimeMeetingResumeConfig resumeConfig = new RealtimeMeetingResumeConfig();
@ -82,7 +85,10 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
sessionData.setUserId(loginUser.getUserId());
sessionData.setTenantId(loginUser.getTenantId());
sessionData.setAsrModelId(asrModelId);
sessionData.setProvider(realtimeAsrChannelFactory.normalizeProvider(asrModel.getProvider()));
sessionData.setTargetWsUrl(targetWsUrl);
sessionData.setModelCode(asrModel.getModelCode());
sessionData.setMediaConfig(asrModel.getMediaConfig());
String sessionToken = UUID.randomUUID().toString().replace("-", "");
socketSessionCache.save(sessionToken, sessionData);
@ -91,9 +97,8 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
vo.setSessionToken(sessionToken);
vo.setPath(WS_PATH);
vo.setExpiresInSeconds(socketSessionCache.getSessionTtlSeconds());
vo.setStartMessage(buildStartMessage(
vo.setStartMessage(realtimeAsrChannel.buildStartMessage(
asrModel,
meetingId,
mode,
language,
useSpkId,
@ -110,74 +115,4 @@ public class RealtimeMeetingSocketSessionServiceImpl implements RealtimeMeetingS
public RealtimeSocketSessionData getSessionData(String sessionToken) {
return socketSessionCache.get(sessionToken);
}
private String resolveWsUrl(AiModelVO model) {
if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) {
return model.getWsUrl();
}
if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) {
return "";
}
return model.getBaseUrl()
.replaceFirst("^http://", "ws://")
.replaceFirst("^https://", "wss://");
}
private Map<String, Object> buildStartMessage(AiModelVO model, Long meetingId, String mode, String language,
Integer useSpkId, Boolean enablePunctuation, Boolean enableItn,
Boolean enableTextRefine, Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
root.put("request_id", "web_" + System.currentTimeMillis() + "_" + meetingId);
root.put("authorization", buildAuthorization(model.getApiKey()));
Map<String, Object> config = new HashMap<>();
Map<String, Object> audio = new HashMap<>();
audio.put("format", "pcm");
audio.put("sample_rate", 16000);
audio.put("channels", 1);
config.put("audio", audio);
Map<String, Object> recognition = new HashMap<>();
recognition.put("language", normalizeLanguage(language));
recognition.put("enable_punctuation", boolOrDefault(enablePunctuation, true));
recognition.put("enable_itn", boolOrDefault(enableItn, true));
recognition.put("enable_speaker", Integer.valueOf(1).equals(useSpkId));
recognition.put("enable_two_pass", !"online".equalsIgnoreCase(mode));
recognition.put("enable_text_refine", boolOrDefault(enableTextRefine, false));
recognition.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig()));
recognition.put("hotwords", hotwords == null ? List.of() : hotwords);
config.put("recognition", recognition);
config.put("model", model.getModelCode());
config.put("save_audio", boolOrDefault(saveAudio, false));
root.put("config", config);
return root;
}
private String buildAuthorization(String apiKey) {
if (apiKey == null || apiKey.isBlank()) {
return "";
}
return apiKey.startsWith("Bearer ") ? apiKey : "Bearer " + apiKey;
}
private Object readSpeakerThreshold(Map<String, Object> mediaConfig) {
if (mediaConfig == null) {
return null;
}
return mediaConfig.get("svThreshold");
}
private String normalizeLanguage(String language) {
if (language == null || language.isBlank()) {
return "auto";
}
return language.trim();
}
private boolean boolOrDefault(Boolean value, boolean defaultValue) {
return value != null ? value : defaultValue;
}
}

View File

@ -0,0 +1,36 @@
package com.imeeting.service.realtime;
import com.imeeting.dto.biz.AiModelVO;
import java.util.List;
import java.util.Map;
public interface RealtimeAsrChannel {
boolean supports(String provider);
String resolveTargetWsUrl(AiModelVO model);
Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords);
void connect(RealtimeAsrChannelContext context) throws Exception;
void handleFrontendText(RealtimeAsrChannelContext context, String payload);
void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload);
default void onFrontendDetached(RealtimeAsrChannelContext context) {
// default no-op
}
void closeMeeting(RealtimeAsrChannelContext context);
boolean isOpen(RealtimeAsrChannelContext context);
}

View File

@ -0,0 +1,17 @@
package com.imeeting.service.realtime;
import org.springframework.web.socket.CloseStatus;
public interface RealtimeAsrChannelCallback {
void onChannelOpen(Long meetingId) throws Exception;
void sendFrontendText(Long meetingId, String payload) throws Exception;
void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception;
void sendFrontendError(Long meetingId, String code, String message);
void removeMeetingSession(Long meetingId);
void closeFrontend(Long meetingId, CloseStatus status);
}

View File

@ -0,0 +1,26 @@
package com.imeeting.service.realtime;
import lombok.Data;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Data
public class RealtimeAsrChannelContext {
private Long meetingId;
private String provider;
private String targetWsUrl;
private WebSocketSession rawSession;
private ConcurrentWebSocketSessionDecorator frontendSession;
private RealtimeAsrChannelCallback callback;
private final ConcurrentMap<String, Object> channelState = new ConcurrentHashMap<>();
private volatile ConcurrentMap<String, Object> frontendState = new ConcurrentHashMap<>();
public void bindFrontendSession(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) {
this.rawSession = rawSession;
this.frontendSession = frontendSession;
this.frontendState = new ConcurrentHashMap<>();
}
}

View File

@ -0,0 +1,29 @@
package com.imeeting.service.realtime;
import com.imeeting.enums.ModelProviderEnum;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@RequiredArgsConstructor
public class RealtimeAsrChannelFactory {
private final List<RealtimeAsrChannel> channels;
public RealtimeAsrChannel getRequired(String provider) {
String normalizedProvider = normalizeProvider(provider);
return channels.stream()
.filter(channel -> channel.supports(normalizedProvider))
.findFirst()
.orElseThrow(() -> new RuntimeException("暂不支持的实时 ASR 渠道: " + provider));
}
public String normalizeProvider(String provider) {
if (provider == null || provider.isBlank()) {
return ModelProviderEnum.LOCAL.getCode();
}
return provider.trim().toLowerCase();
}
}

View File

@ -0,0 +1,13 @@
package com.imeeting.service.realtime;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import java.util.List;
public interface RealtimeMeetingTranscriptCacheService {
void mergeUpstreamMessage(Long meetingId, String payload);
List<RealtimeMeetingTranscriptCacheItem> listOrderedItems(Long meetingId);
void clear(Long meetingId);
}

View File

@ -0,0 +1,538 @@
package com.imeeting.service.realtime.impl;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.CloseStatus;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@Slf4j
@Component
@RequiredArgsConstructor
public class LocalRealtimeAsrChannel implements RealtimeAsrChannel {
private static final String STATE_UPSTREAM_SOCKET = "upstreamSocket";
private static final String STATE_CLOSE_AFTER_END = "closeAfterEnd";
private static final String STATE_START_MESSAGE_FORWARDED = "startMessageForwarded";
private static final String STATE_UPSTREAM_SEND_CHAIN = "upstreamSendChain";
private static final String STATE_START_MESSAGE_SENT = "startMessageSent";
private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final CompletableFuture<Void> COMPLETED = CompletableFuture.completedFuture(null);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String STOP_MESSAGE = "{\"type\":\"stop\"}";
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
@Override
public boolean supports(String provider) {
return ModelProviderEnum.LOCAL.getCode().equalsIgnoreCase(provider);
}
@Override
public String resolveTargetWsUrl(AiModelVO model) {
if (model.getWsUrl() != null && !model.getWsUrl().isBlank()) {
return model.getWsUrl();
}
if (model.getBaseUrl() == null || model.getBaseUrl().isBlank()) {
return "";
}
return model.getBaseUrl()
.replaceFirst("^http://", "ws://")
.replaceFirst("^https://", "wss://");
}
@Override
public Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
Map<String, Object> payload = new HashMap<>();
payload.put("format", "pcm");
payload.put("sample_rate", 16000);
payload.put("language", normalizeProtocolLanguage(language));
payload.put("context", "");
payload.put("enable_inverse_text_normalization", boolOrDefault(enableItn, true));
payload.put("unfixed_token_num", 3);
payload.put("silence_duration_ms", 800);
payload.put("min_partial_sec", 0.3D);
payload.put("pre_roll_ms", 240);
payload.put("max_sentence_count", 8);
payload.put("partial_holdback_chars", 2);
payload.put("enable_native_partial_stream", false);
payload.put("enable_speaker", Integer.valueOf(1).equals(useSpkId));
payload.put("match_speaker_registry", false);
payload.put("speaker_threshold", readSpeakerThreshold(model.getMediaConfig()));
payload.put("enable_realtime_longform", false);
payload.put("enable_realtime_vad_split", false);
payload.put("force_stable_segment_sec", 6);
payload.put("force_stable_min_chars", 24);
payload.put("max_segment_sec", 12);
payload.put("hotwords", hotwords == null ? List.of() : hotwords);
root.put("payload", payload);
return root;
}
@Override
public void connect(RealtimeAsrChannelContext context) throws Exception {
initializeFrontendState(context);
java.net.http.WebSocket upstreamSocket = java.net.http.HttpClient.newHttpClient()
.newWebSocketBuilder()
.buildAsync(URI.create(context.getTargetWsUrl()), new UpstreamListener(context))
.get();
context.getChannelState().put(STATE_UPSTREAM_SOCKET, upstreamSocket);
}
@Override
public void handleFrontendText(RealtimeAsrChannelContext context, String payload) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
initializeFrontendState(context);
if (looksLikeStartMessage(payload)) {
context.getFrontendState().put(STATE_START_MESSAGE_SENT, Boolean.TRUE);
if (!Boolean.TRUE.equals(context.getChannelState().get(STATE_START_MESSAGE_FORWARDED))) {
context.getChannelState().put(STATE_START_MESSAGE_FORWARDED, Boolean.TRUE);
sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text-start");
}
flushPendingAudioFrames(context, upstreamSocket);
return;
}
if (looksLikeStopMessage(payload)) {
context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE);
}
sendUpstreamOrdered(context, () -> upstreamSocket.sendText(payload, true), "text");
}
@Override
public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
initializeFrontendState(context);
if (!Boolean.TRUE.equals(context.getFrontendState().get(STATE_START_MESSAGE_SENT))) {
queuePendingAudioFrame(context, payload);
return;
}
sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary");
}
@Override
public void closeMeeting(RealtimeAsrChannelContext context) {
if (context == null) {
return;
}
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(context);
if (upstreamSocket == null) {
return;
}
context.getChannelState().put(STATE_CLOSE_AFTER_END, Boolean.TRUE);
upstreamSocket.sendText(STOP_MESSAGE, true);
}
@Override
public boolean isOpen(RealtimeAsrChannelContext context) {
return getUpstreamSocket(context) != null;
}
public static String buildFrontendTranscriptMessage(RealtimeMeetingTranscriptCacheItem item) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
boolean isFinal = Boolean.TRUE.equals(item.getFinalResult());
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", item.getContent());
data.put("is_final", isFinal);
if (item.getSentenceId() != null) {
data.put("sentence_id", item.getSentenceId());
}
if (item.getStartTime() != null) {
data.put("start", item.getStartTime() / 1000D);
}
if (item.getEndTime() != null) {
data.put("end", item.getEndTime() / 1000D);
}
if (item.getSpeakerId() != null && !item.getSpeakerId().isBlank()) {
data.put("speaker_id", item.getSpeakerId());
}
if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) {
data.put("speaker_name", item.getSpeakerName());
}
if (item.getUserId() != null && !item.getUserId().isBlank()) {
data.put("user_id", item.getUserId());
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private Object readSpeakerThreshold(Map<String, Object> mediaConfig) {
if (mediaConfig == null) {
return null;
}
return mediaConfig.get("svThreshold");
}
private String normalizeProtocolLanguage(String language) {
if (language == null || language.isBlank()) {
return null;
}
String normalized = language.trim();
if ("auto".equalsIgnoreCase(normalized)) {
return null;
}
return normalized;
}
private boolean boolOrDefault(Boolean value, boolean defaultValue) {
return value != null ? value : defaultValue;
}
private void initializeFrontendState(RealtimeAsrChannelContext context) {
context.getFrontendState().putIfAbsent(STATE_UPSTREAM_SEND_CHAIN, COMPLETED);
context.getFrontendState().putIfAbsent(STATE_START_MESSAGE_SENT, Boolean.FALSE);
context.getFrontendState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
private java.net.http.WebSocket getUpstreamSocket(RealtimeAsrChannelContext context) {
Object value = context.getChannelState().get(STATE_UPSTREAM_SOCKET);
return value instanceof java.net.http.WebSocket socket ? socket : null;
}
@SuppressWarnings("unchecked")
private void sendUpstreamOrdered(RealtimeAsrChannelContext context,
Supplier<CompletableFuture<?>> sendAction,
String messageType) {
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
CompletableFuture<Void> chain = (CompletableFuture<Void>) frontendState.getOrDefault(STATE_UPSTREAM_SEND_CHAIN, COMPLETED);
CompletableFuture<Void> nextChain = chain
.exceptionally(ex -> null)
.thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null));
nextChain = nextChain.whenComplete((ignored, ex) -> {
if (ex != null) {
log.error("顺序发送上游消息失败meetingId={}, sessionId={}, type={}",
context.getMeetingId(), currentConnectionId(context), messageType, ex);
}
});
frontendState.put(STATE_UPSTREAM_SEND_CHAIN, nextChain);
}
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) {
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
List<byte[]> pendingFrames = (List<byte[]>) frontendState.get(STATE_PENDING_AUDIO_FRAMES);
if (pendingFrames == null) {
pendingFrames = new ArrayList<>();
frontendState.put(STATE_PENDING_AUDIO_FRAMES, pendingFrames);
}
pendingFrames.add(payload);
}
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(RealtimeAsrChannelContext context, java.net.http.WebSocket upstreamSocket) {
List<byte[]> pendingFrames;
ConcurrentMap<String, Object> frontendState = context.getFrontendState();
synchronized (frontendState) {
pendingFrames = (List<byte[]>) frontendState.get(STATE_PENDING_AUDIO_FRAMES);
if (pendingFrames == null || pendingFrames.isEmpty()) {
return;
}
frontendState.put(STATE_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
log.info("start 后开始补发排队音频帧meetingId={}, sessionId={}, frameCount={}",
context.getMeetingId(), currentConnectionId(context), pendingFrames.size());
for (byte[] frame : pendingFrames) {
sendUpstreamOrdered(context, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush");
}
}
private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId();
}
private static ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
private static boolean shouldLogBinaryFrame(int count) {
return count <= 3 || count % 25 == 0;
}
private static String summarizeText(String payload) {
if (payload == null) {
return "";
}
String normalized = payload.replaceAll("\\s+", " ").trim();
if (normalized.length() <= 240) {
return normalized;
}
return normalized.substring(0, 240) + "...";
}
private static boolean looksLikeStartMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private static boolean looksLikeStopMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"stop\"");
}
private static boolean looksLikeEndMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
try {
JsonNode root = OBJECT_MAPPER.readTree(payload);
return "end".equals(root.path("type").asText(""));
} catch (Exception ex) {
return false;
}
}
private static List<String> normalizeFrontendMessages(String upstreamPayload) {
try {
JsonNode root = OBJECT_MAPPER.readTree(upstreamPayload);
String type = root.path("type").asText("");
if (!"sentences".equals(type) && !"end".equals(type)) {
return List.of(upstreamPayload);
}
List<String> normalizedMessages = new ArrayList<>();
JsonNode sentences = root.path("sentences");
if (sentences.isArray()) {
for (JsonNode sentence : sentences) {
String text = sentence.path("sentence").asText("").trim();
if (text.isEmpty()) {
continue;
}
boolean isFinal = sentence.path("sentence_type").asInt(0) != 0 || "end".equals(type);
normalizedMessages.add(buildFrontendTranscriptMessage(sentence, text, isFinal));
}
}
if (normalizedMessages.isEmpty()) {
String fallbackText = root.path("result").path("voice_text_str").asText("").trim();
if (!fallbackText.isEmpty()) {
normalizedMessages.add(buildFrontendTranscriptMessage((JsonNode) null, fallbackText, true));
}
}
return normalizedMessages.isEmpty() ? List.of(upstreamPayload) : normalizedMessages;
} catch (Exception ex) {
return List.of(upstreamPayload);
}
}
private static String buildFrontendTranscriptMessage(JsonNode sentence, String text, boolean isFinal) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", text);
data.put("is_final", isFinal);
if (sentence != null && sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt()) {
data.put("sentence_id", sentence.get("sentence_id").asInt());
}
if (sentence != null) {
copyOptionalTimeSeconds(sentence, data, "start_time", "start");
copyOptionalTimeSeconds(sentence, data, "end_time", "end");
copyOptionalText(sentence, data, "speaker_id");
copyOptionalText(sentence, data, "speaker_name");
copyOptionalText(sentence, data, "user_id");
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private static void copyOptionalTimeSeconds(JsonNode source, ObjectNode target, String sourceFieldName, String targetFieldName) {
if (source == null) {
return;
}
if (source.has(sourceFieldName) && source.get(sourceFieldName).isNumber()) {
target.put(targetFieldName, source.get(sourceFieldName).asDouble() / 1000D);
return;
}
if (source.has(targetFieldName) && source.get(targetFieldName).isNumber()) {
target.put(targetFieldName, source.get(targetFieldName).asDouble());
}
}
private static void copyOptionalText(JsonNode source, ObjectNode target, String fieldName) {
if (source == null || !source.has(fieldName) || source.get(fieldName).isNull()) {
return;
}
String value = source.get(fieldName).asText("").trim();
if (!value.isEmpty()) {
target.put(fieldName, value);
}
}
private final class UpstreamListener implements java.net.http.WebSocket.Listener {
private final RealtimeAsrChannelContext context;
private final StringBuilder textBuffer = new StringBuilder();
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
private final AtomicInteger upstreamTextCount = new AtomicInteger();
private final AtomicInteger upstreamBinaryCount = new AtomicInteger();
private UpstreamListener(RealtimeAsrChannelContext context) {
this.context = context;
}
@Override
public void onOpen(java.net.http.WebSocket webSocket) {
context.getChannelState().put(STATE_UPSTREAM_SOCKET, webSocket);
log.info("上游 ASR websocket 已打开meetingId={}, sessionId={}, upstream={}",
context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl());
String connectionId = currentConnectionId(context);
if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) {
context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接");
webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "当前会议无法激活这条前端连接");
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接"));
return;
}
try {
context.getCallback().onChannelOpen(context.getMeetingId());
} catch (Exception ex) {
log.error("通知前端上游就绪失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
return;
}
webSocket.request(1);
}
@Override
public java.util.concurrent.CompletionStage<?> onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) {
textBuffer.append(data);
if (last) {
int count = upstreamTextCount.incrementAndGet();
String upstreamPayload = textBuffer.toString();
realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), upstreamPayload);
try {
for (String frontendPayload : normalizeFrontendMessages(upstreamPayload)) {
context.getCallback().sendFrontendText(context.getMeetingId(), frontendPayload);
}
log.info("上游 ASR 文本 -> 前端meetingId={}, sessionId={}, count={}, payload={}",
context.getMeetingId(), currentConnectionId(context), count, summarizeText(upstreamPayload));
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_CLOSE_AFTER_END))
&& looksLikeEndMessage(upstreamPayload)) {
webSocket.sendClose(CloseStatus.NORMAL.getCode(), "meeting-complete");
}
} catch (Exception ex) {
log.error("转发上游 ASR 文本失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
} finally {
textBuffer.setLength(0);
}
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) {
byte[] chunk = new byte[data.remaining()];
data.get(chunk);
binaryBuffer.writeBytes(chunk);
if (last) {
int count = upstreamBinaryCount.incrementAndGet();
try {
context.getCallback().sendFrontendBinary(context.getMeetingId(), binaryBuffer.toByteArray());
if (shouldLogBinaryFrame(count)) {
log.info("上游 ASR 二进制消息 -> 前端meetingId={}, sessionId={}, count={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), count, binaryBuffer.size());
}
} catch (Exception ex) {
log.error("转发上游 ASR 二进制消息失败meetingId={}, sessionId={}", context.getMeetingId(), currentConnectionId(context), ex);
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
} finally {
binaryBuffer.reset();
}
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPing(java.net.http.WebSocket webSocket, ByteBuffer message) {
webSocket.sendPong(copyBuffer(message));
log.info("上游 ASR ping 已本地响应meetingId={}, sessionId={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), message.remaining());
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPong(java.net.http.WebSocket webSocket, ByteBuffer message) {
log.debug("上游 ASR pong 已本地忽略meetingId={}, sessionId={}, bytes={}",
context.getMeetingId(), currentConnectionId(context), message.remaining());
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) {
log.info("上游 ASR websocket 已关闭meetingId={}, sessionId={}, code={}, reason={}",
context.getMeetingId(), currentConnectionId(context), statusCode, reason);
context.getChannelState().remove(STATE_UPSTREAM_SOCKET);
context.getCallback().removeMeetingSession(context.getMeetingId());
context.getCallback().sendFrontendError(context.getMeetingId(),
"REALTIME_UPSTREAM_CLOSED",
reason == null || reason.isBlank() ? "上游 ASR WebSocket 已断开" : "上游 ASR WebSocket 已断开: " + reason);
context.getCallback().closeFrontend(context.getMeetingId(), new CloseStatus(statusCode, reason));
return COMPLETED;
}
@Override
public void onError(java.net.http.WebSocket webSocket, Throwable error) {
log.error("上游 ASR websocket 异常meetingId={}, sessionId={}, upstream={}",
context.getMeetingId(), currentConnectionId(context), context.getTargetWsUrl(), error);
context.getChannelState().remove(STATE_UPSTREAM_SOCKET);
context.getCallback().removeMeetingSession(context.getMeetingId());
context.getCallback().sendFrontendError(context.getMeetingId(),
"REALTIME_UPSTREAM_ERROR",
error == null || error.getMessage() == null || error.getMessage().isBlank()
? "上游 ASR WebSocket 连接异常"
: "上游 ASR WebSocket 连接异常: " + error.getMessage());
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR);
}
}
}

View File

@ -0,0 +1,291 @@
package com.imeeting.service.realtime.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.entity.biz.MeetingTranscript;
import com.imeeting.mapper.biz.MeetingTranscriptMapper;
import com.imeeting.service.biz.MeetingTranscriptFileService;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.imeeting.support.redis.RealtimeMeetingTranscriptCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@Service
public class RealtimeMeetingTranscriptCacheServiceImpl implements RealtimeMeetingTranscriptCacheService {
private final RealtimeMeetingTranscriptCache transcriptCache;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final MeetingTranscriptMapper transcriptMapper;
private final MeetingTranscriptFileService meetingTranscriptFileService;
private final ObjectMapper objectMapper = new ObjectMapper();
private final Map<Long, Object> meetingLocks = new ConcurrentHashMap<>();
@Autowired
public RealtimeMeetingTranscriptCacheServiceImpl(RealtimeMeetingTranscriptCache transcriptCache,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService,
MeetingTranscriptMapper transcriptMapper,
MeetingTranscriptFileService meetingTranscriptFileService) {
this.transcriptCache = transcriptCache;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService;
this.transcriptMapper = transcriptMapper;
this.meetingTranscriptFileService = meetingTranscriptFileService;
}
@Override
public void mergeUpstreamMessage(Long meetingId, String payload) {
if (meetingId == null || payload == null || payload.isBlank()) {
return;
}
synchronized (lockForMeeting(meetingId)) {
RealtimeMeetingTranscriptCacheState state = getOrCreateState(meetingId);
try {
JsonNode root = objectMapper.readTree(payload);
String type = root.path("type").asText("");
if (!"sentences".equals(type) && !"end".equals(type)) {
return;
}
JsonNode sentences = root.path("sentences");
if (!sentences.isArray()) {
return;
}
for (JsonNode sentence : sentences) {
RealtimeMeetingTranscriptCacheItem item = mergeSentenceNode(state, sentence, "end".equals(type));
persistFinalSentence(meetingId, item);
}
state.setUpdatedAt(System.currentTimeMillis());
transcriptCache.saveState(state);
realtimeMeetingSessionStateService.refreshAfterTranscriptCapture(meetingId, countNonEmptyItems(state));
} catch (Exception ignored) {
// ignore malformed upstream payload
}
}
}
@Override
public List<RealtimeMeetingTranscriptCacheItem> listOrderedItems(Long meetingId) {
RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId);
if (state == null || state.getItems() == null || state.getItems().isEmpty()) {
return List.of();
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.sorted(Comparator.comparing(item -> item.getSortOrder() == null ? Integer.MAX_VALUE : item.getSortOrder()))
.toList();
}
@Override
public void clear(Long meetingId) {
transcriptCache.clear(meetingId);
}
private RealtimeMeetingTranscriptCacheItem mergeSentenceNode(RealtimeMeetingTranscriptCacheState state,
JsonNode sentence,
boolean fromEndMessage) {
String text = sentence.path("sentence").asText("").trim();
if (text.isEmpty()) {
return null;
}
Integer sentenceId = sentence.has("sentence_id") && sentence.get("sentence_id").canConvertToInt()
? sentence.get("sentence_id").asInt()
: null;
String upstreamSentenceKey = readText(sentence, "sentence_key");
String sentenceKey = upstreamSentenceKey != null && !upstreamSentenceKey.isBlank()
? upstreamSentenceKey
: sentenceId == null ? "sentence-" + nextLegacySequence(state) : "sentence-" + sentenceId;
RealtimeMeetingTranscriptCacheItem item = findBySentenceKey(state, sentenceKey);
long now = System.currentTimeMillis();
if (item == null) {
item = new RealtimeMeetingTranscriptCacheItem();
item.setSentenceKey(sentenceKey);
item.setSentenceGroupKey(upstreamSentenceKey);
item.setSentenceId(sentenceId);
item.setSortOrder(nextSortOrder(state));
item.setFirstReceivedAt(now);
state.getItems().add(item);
}
item.setSentenceGroupKey(upstreamSentenceKey);
item.setSentenceType(readInteger(sentence, "sentence_type"));
item.setSpeakerId(resolveSpeakerId(sentence));
item.setSpeakerName(readText(sentence, "speaker_name"));
item.setUserId(readText(sentence, "user_id"));
item.setStartTime(readTimeMilliseconds(sentence, "start_time", "start"));
item.setEndTime(readTimeMilliseconds(sentence, "end_time", "end"));
item.setContent(text);
item.setFinalResult(fromEndMessage || Objects.equals(item.getSentenceType(), 1));
item.setUpdatedAt(now);
return item;
}
private void persistFinalSentence(Long meetingId, RealtimeMeetingTranscriptCacheItem item) {
if (meetingId == null || item == null || !Boolean.TRUE.equals(item.getFinalResult())) {
return;
}
String content = item.getContent() == null ? null : item.getContent().trim();
if (content == null || content.isBlank()) {
return;
}
String speakerId = resolveSpeakerId(item);
String speakerName = resolveSpeakerName(item);
if (item.getTranscriptId() != null) {
transcriptMapper.update(null, new LambdaUpdateWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getId, item.getTranscriptId())
.set(MeetingTranscript::getSpeakerId, speakerId)
.set(MeetingTranscript::getSpeakerName, speakerName)
.set(MeetingTranscript::getContent, content)
.set(item.getStartTime() != null, MeetingTranscript::getStartTime, item.getStartTime())
.set(item.getEndTime() != null, MeetingTranscript::getEndTime, item.getEndTime()));
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
return;
}
MeetingTranscript transcript = new MeetingTranscript();
transcript.setMeetingId(meetingId);
transcript.setSpeakerId(speakerId);
transcript.setSpeakerName(speakerName);
transcript.setContent(content);
transcript.setStartTime(item.getStartTime());
transcript.setEndTime(item.getEndTime());
transcript.setSortOrder(nextPersistedSortOrder(meetingId));
transcriptMapper.insert(transcript);
item.setTranscriptId(transcript.getId());
meetingTranscriptFileService.initializeTranscriptFileIfAbsent(meetingId);
}
private int nextPersistedSortOrder(Long meetingId) {
Integer maxSortOrder = transcriptMapper.selectList(new LambdaQueryWrapper<MeetingTranscript>()
.eq(MeetingTranscript::getMeetingId, meetingId)
.orderByDesc(MeetingTranscript::getSortOrder)
.last("LIMIT 1"))
.stream()
.findFirst()
.map(MeetingTranscript::getSortOrder)
.orElse(0);
return maxSortOrder == null ? 0 : maxSortOrder + 1;
}
private RealtimeMeetingTranscriptCacheItem findBySentenceKey(RealtimeMeetingTranscriptCacheState state, String sentenceKey) {
if (state.getItems() == null || state.getItems().isEmpty()) {
return null;
}
return state.getItems().stream()
.filter(item -> sentenceKey.equals(item.getSentenceKey()))
.findFirst()
.orElse(null);
}
private String resolveSpeakerId(RealtimeMeetingTranscriptCacheItem item) {
if (item == null) {
return null;
}
if (item.getUserId() != null && !item.getUserId().isBlank()) {
return item.getUserId().trim();
}
if (item.getSpeakerId() == null || item.getSpeakerId().isBlank() || "-1".equals(item.getSpeakerId().trim())) {
return null;
}
return item.getSpeakerId().trim();
}
private String resolveSpeakerName(RealtimeMeetingTranscriptCacheItem item) {
if (item == null) {
return null;
}
if (item.getSpeakerName() != null && !item.getSpeakerName().isBlank()) {
return item.getSpeakerName().trim();
}
String speakerId = resolveSpeakerId(item);
return speakerId == null || speakerId.isBlank() ? null : speakerId;
}
private String resolveSpeakerId(JsonNode sentence) {
String userId = readText(sentence, "user_id");
if (userId != null && !userId.isBlank()) {
return userId;
}
return readText(sentence, "speaker_id");
}
private Integer readInteger(JsonNode node, String fieldName) {
if (node == null || !node.has(fieldName) || !node.get(fieldName).canConvertToInt()) {
return null;
}
return node.get(fieldName).asInt();
}
private Integer readTimeMilliseconds(JsonNode node, String primaryField, String fallbackField) {
if (node == null) {
return null;
}
if (node.has(primaryField) && node.get(primaryField).canConvertToInt()) {
return node.get(primaryField).asInt();
}
if (node.has(fallbackField) && node.get(fallbackField).isNumber()) {
return Math.round((float) (node.get(fallbackField).asDouble() * 1000));
}
return null;
}
private String readText(JsonNode node, String fieldName) {
if (node == null || !node.has(fieldName) || node.get(fieldName).isNull()) {
return null;
}
String value = node.get(fieldName).asText("");
return value == null ? null : value.trim();
}
private long countNonEmptyItems(RealtimeMeetingTranscriptCacheState state) {
if (state.getItems() == null || state.getItems().isEmpty()) {
return 0L;
}
return state.getItems().stream()
.filter(item -> item.getContent() != null && !item.getContent().isBlank())
.count();
}
private int nextSortOrder(RealtimeMeetingTranscriptCacheState state) {
Integer current = state.getNextSortOrder();
int next = current == null ? 0 : current;
state.setNextSortOrder(next + 1);
return next;
}
private int nextLegacySequence(RealtimeMeetingTranscriptCacheState state) {
Integer current = state.getNextLegacySequence();
int next = current == null ? 0 : current;
state.setNextLegacySequence(next + 1);
return next;
}
private RealtimeMeetingTranscriptCacheState getOrCreateState(Long meetingId) {
RealtimeMeetingTranscriptCacheState state = transcriptCache.getState(meetingId);
if (state != null) {
if (state.getItems() == null) {
state.setItems(new ArrayList<>());
}
return state;
}
RealtimeMeetingTranscriptCacheState next = new RealtimeMeetingTranscriptCacheState();
next.setMeetingId(meetingId);
next.setItems(new ArrayList<>());
next.setNextSortOrder(0);
next.setNextLegacySequence(0);
next.setUpdatedAt(System.currentTimeMillis());
return next;
}
private Object lockForMeeting(Long meetingId) {
return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object());
}
}

View File

@ -0,0 +1,506 @@
package com.imeeting.service.realtime.impl;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.enums.ModelProviderEnum;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.tencent.asrspeaker.SpeakerConstant;
import com.tencent.asrspeaker.SpeakerRecognitionListener;
import com.tencent.asrspeaker.SpeakerRecognitionResponse;
import com.tencent.asrspeaker.SpeakerRecognizer;
import com.tencent.asrspeaker.SpeakerRecognizerRequest;
import com.tencent.asrspeaker.SpeakerSentenceItem;
import com.tencent.core.ws.Credential;
import com.tencent.core.ws.SpeechClient;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.CloseStatus;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
@RequiredArgsConstructor
public class TencentRealtimeAsrChannel implements RealtimeAsrChannel {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String TARGET_WS_URL = "tencent-sdk://speaker-recognizer";
private static final String MEDIA_TENCENT_APP_ID = "tencentAppId";
private static final String MEDIA_TENCENT_SECRET_ID = "tencentSecretId";
private static final String MEDIA_TENCENT_SECRET_KEY = "tencentSecretKey";
private static final String STATE_CONNECTED = "tencentConnected";
private static final String STATE_STARTED = "tencentStarted";
private static final String STATE_RECOGNIZER = "tencentRecognizer";
private static final String STATE_SPEECH_CLIENT = "tencentSpeechClient";
private static final String STATE_STOP_REQUESTED = "tencentStopRequested";
private static final String STATE_MEETING_COMPLETE_REQUESTED = "tencentMeetingCompleteRequested";
private static final String STATE_FRONTEND_DETACHED = "tencentFrontendDetached";
private static final String STATE_SPEAKER_CONTEXT_ID = "speakerContextId";
private static final String STATE_VOICE_ID = "voiceId";
private static final String STATE_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final String STATE_MODEL_CODE = "modelCode";
private static final String STATE_MEDIA_CONFIG = "mediaConfig";
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
@Override
public boolean supports(String provider) {
return ModelProviderEnum.TENCENT.getCode().equalsIgnoreCase(provider);
}
@Override
public String resolveTargetWsUrl(AiModelVO model) {
return TARGET_WS_URL;
}
@Override
public Map<String, Object> buildStartMessage(AiModelVO model,
String mode,
String language,
Integer useSpkId,
Boolean enablePunctuation,
Boolean enableItn,
Boolean enableTextRefine,
Boolean saveAudio,
List<Map<String, Object>> hotwords) {
Map<String, Object> payload = new HashMap<>();
payload.put("provider", ModelProviderEnum.TENCENT.getCode());
payload.put("engine_model_type", model.getModelCode());
payload.put("language", language);
Map<String, Object> root = new HashMap<>();
root.put("type", "start");
root.put("payload", payload);
return root;
}
@Override
public void connect(RealtimeAsrChannelContext context) throws Exception {
String connectionId = currentConnectionId(context);
if (connectionId == null || !realtimeMeetingSessionStateService.activate(context.getMeetingId(), connectionId)) {
context.getCallback().sendFrontendError(context.getMeetingId(), "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议无法激活这条前端连接");
context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.POLICY_VIOLATION.withReason("当前会议无法激活这条前端连接"));
return;
}
context.getChannelState().put(STATE_CONNECTED, Boolean.TRUE);
context.getChannelState().put(STATE_STARTED, Boolean.FALSE);
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.FALSE);
context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.FALSE);
context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.FALSE);
context.getChannelState().putIfAbsent(STATE_PENDING_AUDIO_FRAMES, new java.util.ArrayList<byte[]>());
context.getCallback().onChannelOpen(context.getMeetingId());
}
@Override
public void handleFrontendText(RealtimeAsrChannelContext context, String payload) {
if (looksLikeStartMessage(payload)) {
startRecognizerIfNecessary(context);
return;
}
if (looksLikeStopMessage(payload)) {
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
}
@Override
public void handleFrontendBinary(RealtimeAsrChannelContext context, byte[] payload) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (payload == null || payload.length == 0) {
return;
}
if (recognizer == null) {
queuePendingAudioFrame(context, payload);
return;
}
try {
recognizer.write(payload);
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 音频发送失败", ex);
}
}
@Override
public void closeMeeting(RealtimeAsrChannelContext context) {
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
context.getChannelState().put(STATE_MEETING_COMPLETE_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
@Override
public boolean isOpen(RealtimeAsrChannelContext context) {
return !Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED));
}
@Override
public void onFrontendDetached(RealtimeAsrChannelContext context) {
context.getChannelState().put(STATE_FRONTEND_DETACHED, Boolean.TRUE);
context.getChannelState().put(STATE_STOP_REQUESTED, Boolean.TRUE);
stopRecognizer(context);
}
static String buildFrontendTranscriptMessage(String sentenceKey,
String text,
boolean isFinal,
Integer sentenceId,
Long startTime,
Long endTime,
Integer speakerId) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", isFinal ? "segment" : "partial");
ObjectNode data = root.putObject("data");
data.put("text", text);
data.put("is_final", isFinal);
if (sentenceId != null) {
data.put("sentence_id", sentenceId);
}
if (sentenceKey != null && !sentenceKey.isBlank()) {
data.put("sentence_key", sentenceKey);
}
if (startTime != null) {
data.put("start", startTime / 1000D);
}
if (endTime != null) {
data.put("end", endTime / 1000D);
}
if (speakerId != null) {
data.put("speaker_id", String.valueOf(speakerId));
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private void startRecognizerIfNecessary(RealtimeAsrChannelContext context) {
synchronized (context.getChannelState()) {
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_STARTED))) {
return;
}
try {
SpeechClient speechClient = createSpeechClient();
SpeakerRecognizerRequest request = createRecognizerRequest(context);
SpeakerRecognizer recognizer = createRecognizer(context, speechClient, request);
context.getChannelState().put(STATE_SPEECH_CLIENT, speechClient);
context.getChannelState().put(STATE_RECOGNIZER, recognizer);
context.getChannelState().put(STATE_VOICE_ID, request.getVoiceId());
recognizer.start();
context.getChannelState().put(STATE_STARTED, Boolean.TRUE);
flushPendingAudioFrames(context, recognizer);
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_CONNECT_FAILED", "腾讯实时 ASR 启动失败", ex);
}
}
}
protected SpeechClient createSpeechClient() {
return new SpeechClient(SpeakerConstant.DEFAULT_RT_REQ_URL);
}
protected SpeakerRecognizer createRecognizer(RealtimeAsrChannelContext context,
SpeechClient speechClient,
SpeakerRecognizerRequest request) {
return new SpeakerRecognizer(
speechClient,
buildCredential(context),
request,
new TencentRecognitionListener(context)
);
}
private Credential buildCredential(RealtimeAsrChannelContext context) {
Map<String, Object> mediaConfig = getMediaConfig(context);
String appId = readConfigString(mediaConfig, MEDIA_TENCENT_APP_ID);
String secretId = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_ID);
String secretKey = readConfigString(mediaConfig, MEDIA_TENCENT_SECRET_KEY);
if (appId == null || secretId == null || secretKey == null) {
throw new RuntimeException("腾讯实时 ASR 会话缺少鉴权配置");
}
return new Credential(appId, secretId, secretKey);
}
SpeakerRecognizerRequest createRecognizerRequest(RealtimeAsrChannelContext context) {
SpeakerRecognizerRequest request = SpeakerRecognizerRequest.init();
request.setEngineModelType(resolveEngineModelType(context));
request.setVoiceFormat(SpeakerConstant.AUDIO_FORMAT_PCM);
request.setVoiceId(UUID.randomUUID().toString());
//是否需要vad
request.setNeedVad(1);
//vad静默时间
request.setVadSilenceTime(1000);
// 分句策略参数 0小1大
request.setSentenceStrategy(0);
//是否进行阿拉伯数字智能转换 0否1智能 23:打开数学相关转化
request.setConvertNumMode(1);
request.setSpeakerDiarization(1);
//启动断点续传
request.setEnableSpeakerContext(1);
String speakerContextId = resolveSpeakerContextId(context);
if (speakerContextId != null) {
request.setSpeakerContextId(speakerContextId);
}
return request;
}
private String resolveEngineModelType(RealtimeAsrChannelContext context) {
Object modelCode = context.getChannelState().get(STATE_MODEL_CODE);
if (modelCode instanceof String value && !value.isBlank()) {
return value;
}
return "16k_zh";
}
private String resolveSpeakerContextId(RealtimeAsrChannelContext context) {
Object speakerContextId = context.getChannelState().get(STATE_SPEAKER_CONTEXT_ID);
if (speakerContextId instanceof String value && !value.isBlank()) {
return value;
}
var status = realtimeMeetingSessionStateService.getStatus(context.getMeetingId());
if (status == null || status.getResumeConfig() == null) {
return null;
}
String value = status.getResumeConfig().getSpeakerContextId();
if (value == null || value.isBlank()) {
return null;
}
context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, value);
return value;
}
@SuppressWarnings("unchecked")
private Map<String, Object> getMediaConfig(RealtimeAsrChannelContext context) {
Object mediaConfig = context.getChannelState().get(STATE_MEDIA_CONFIG);
if (mediaConfig instanceof Map<?, ?> map) {
return (Map<String, Object>) map;
}
return Map.of();
}
private String readConfigString(Map<String, Object> mediaConfig, String key) {
Object value = mediaConfig.get(key);
if (value == null) {
return null;
}
String text = String.valueOf(value).trim();
return text.isEmpty() ? null : text;
}
private SpeakerRecognizer getRecognizer(RealtimeAsrChannelContext context) {
Object recognizer = context.getChannelState().get(STATE_RECOGNIZER);
return recognizer instanceof SpeakerRecognizer value ? value : null;
}
private SpeechClient getSpeechClient(RealtimeAsrChannelContext context) {
Object speechClient = context.getChannelState().get(STATE_SPEECH_CLIENT);
return speechClient instanceof SpeechClient value ? value : null;
}
private void stopRecognizer(RealtimeAsrChannelContext context) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (recognizer == null) {
shutdownSdkResources(context);
return;
}
try {
recognizer.stop();
} catch (Exception ex) {
log.warn("Tencent realtime ASR stop failed, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context), ex);
shutdownSdkResources(context);
}
}
private void shutdownSdkResources(RealtimeAsrChannelContext context) {
SpeakerRecognizer recognizer = getRecognizer(context);
if (recognizer != null) {
try {
recognizer.close();
} catch (Exception ignored) {
// ignore
}
}
SpeechClient speechClient = getSpeechClient(context);
if (speechClient != null) {
try {
speechClient.shutdown();
} catch (Exception ignored) {
// ignore
}
}
context.getChannelState().remove(STATE_RECOGNIZER);
context.getChannelState().remove(STATE_SPEECH_CLIENT);
context.getChannelState().put(STATE_STARTED, Boolean.FALSE);
}
private void forwardResponse(RealtimeAsrChannelContext context,
SpeakerRecognitionResponse response,
boolean forceFinal) {
if (response == null || response.getSentences() == null || response.getSentences().getSentenceList() == null) {
return;
}
try {
rememberSpeakerContext(context, response);
String cachePayload = buildCachePayload(response, forceFinal);
realtimeMeetingTranscriptCacheService.mergeUpstreamMessage(context.getMeetingId(), cachePayload);
for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) {
if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) {
continue;
}
boolean isFinal = forceFinal || item.getSentenceType() == 1;
context.getCallback().sendFrontendText(
context.getMeetingId(),
buildFrontendTranscriptMessage(
buildSentenceKey(context, item.getSentenceId()),
item.getSentence().trim(),
isFinal,
item.getSentenceId(),
item.getStartTime(),
item.getEndTime(),
item.getSpeakerId()
)
);
}
} catch (Exception ex) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 结果转发失败", ex);
}
}
private void rememberSpeakerContext(RealtimeAsrChannelContext context, SpeakerRecognitionResponse response) {
if (response == null || response.getSpeakerContextId() == null || response.getSpeakerContextId().isBlank()) {
return;
}
String speakerContextId = response.getSpeakerContextId().trim();
context.getChannelState().put(STATE_SPEAKER_CONTEXT_ID, speakerContextId);
realtimeMeetingSessionStateService.rememberSpeakerContext(context.getMeetingId(), speakerContextId);
}
private String buildSentenceKey(RealtimeAsrChannelContext context, Integer sentenceId) {
Object voiceId = context.getChannelState().get(STATE_VOICE_ID);
if (!(voiceId instanceof String value) || value.isBlank() || sentenceId == null) {
return null;
}
return value + "-" + sentenceId;
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(RealtimeAsrChannelContext context, byte[] payload) {
Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES);
if (frames instanceof List<?> list) {
((List<byte[]>) list).add(payload.clone());
return;
}
List<byte[]> next = new java.util.ArrayList<>();
next.add(payload.clone());
context.getChannelState().put(STATE_PENDING_AUDIO_FRAMES, next);
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(RealtimeAsrChannelContext context, SpeakerRecognizer recognizer) {
Object frames = context.getChannelState().get(STATE_PENDING_AUDIO_FRAMES);
if (!(frames instanceof List<?> list) || list.isEmpty()) {
return;
}
List<byte[]> pendingFrames = (List<byte[]>) list;
for (byte[] frame : pendingFrames) {
if (frame != null && frame.length > 0) {
recognizer.write(frame);
}
}
pendingFrames.clear();
}
private String buildCachePayload(SpeakerRecognitionResponse response, boolean forceFinal) throws JsonProcessingException {
ObjectNode root = OBJECT_MAPPER.createObjectNode();
root.put("type", forceFinal ? "end" : "sentences");
ArrayNode sentences = root.putArray("sentences");
for (SpeakerSentenceItem item : response.getSentences().getSentenceList()) {
if (item == null || item.getSentence() == null || item.getSentence().trim().isEmpty()) {
continue;
}
ObjectNode sentenceNode = sentences.addObject();
sentenceNode.put("sentence", item.getSentence().trim());
sentenceNode.put("sentence_type", forceFinal ? 1 : item.getSentenceType());
sentenceNode.put("sentence_id", item.getSentenceId());
sentenceNode.put("speaker_id", String.valueOf(item.getSpeakerId()));
sentenceNode.put("start_time", item.getStartTime());
sentenceNode.put("end_time", item.getEndTime());
}
return OBJECT_MAPPER.writeValueAsString(root);
}
private void handleChannelFailure(RealtimeAsrChannelContext context, String code, String message, Exception ex) {
log.error("Tencent realtime ASR channel failed, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context), ex);
shutdownSdkResources(context);
context.getCallback().sendFrontendError(context.getMeetingId(), code, message);
CompletableFuture.delayedExecutor(200, TimeUnit.MILLISECONDS).execute(
() -> context.getCallback().closeFrontend(context.getMeetingId(), CloseStatus.SERVER_ERROR)
);
}
private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId();
}
private static boolean looksLikeStartMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private static boolean looksLikeStopMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"stop\"");
}
private final class TencentRecognitionListener extends SpeakerRecognitionListener {
private final RealtimeAsrChannelContext context;
private TencentRecognitionListener(RealtimeAsrChannelContext context) {
this.context = context;
}
@Override
public void onRecognitionStart(SpeakerRecognitionResponse response) {
rememberSpeakerContext(context, response);
log.info("Tencent realtime ASR started, meetingId={}, sessionId={}",
context.getMeetingId(), currentConnectionId(context));
}
@Override
public void onRecognitionSentences(SpeakerRecognitionResponse response) {
forwardResponse(context, response, false);
}
@Override
public void onSentenceEnd(SpeakerRecognitionResponse response) {
forwardResponse(context, response, true);
shutdownSdkResources(context);
if (Boolean.TRUE.equals(context.getChannelState().get(STATE_MEETING_COMPLETE_REQUESTED))) {
context.getCallback().removeMeetingSession(context.getMeetingId());
}
}
@Override
public void onFail(SpeakerRecognitionResponse response, Exception error) {
handleChannelFailure(context, "REALTIME_UPSTREAM_ERROR", "腾讯实时 ASR 识别失败", error);
}
}
}

View File

@ -0,0 +1,39 @@
package com.imeeting.support.redis;
import com.imeeting.common.RedisKeys;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheState;
import com.imeeting.support.RedisSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.time.Duration;
@Component
@RequiredArgsConstructor
public class RealtimeMeetingTranscriptCache {
private static final Duration CACHE_TTL = Duration.ofHours(12);
private final RedisSupport redisSupport;
public RealtimeMeetingTranscriptCacheState getState(Long meetingId) {
if (meetingId == null) {
return null;
}
return redisSupport.getJsonQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId), RealtimeMeetingTranscriptCacheState.class);
}
public void saveState(RealtimeMeetingTranscriptCacheState state) {
if (state == null || state.getMeetingId() == null) {
return;
}
redisSupport.setJson(RedisKeys.realtimeMeetingTranscriptCacheKey(state.getMeetingId()), state, CACHE_TTL);
}
public void clear(Long meetingId) {
if (meetingId == null) {
return;
}
redisSupport.deleteQuietly(RedisKeys.realtimeMeetingTranscriptCacheKey(meetingId));
}
}

View File

@ -1,173 +1,133 @@
package com.imeeting.websocket;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.dto.biz.RealtimeMeetingTranscriptCacheItem;
import com.imeeting.dto.biz.RealtimeSocketSessionData;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.biz.RealtimeMeetingSocketSessionService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelCallback;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeAsrChannelFactory;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import com.imeeting.service.realtime.RealtimeMeetingTranscriptCacheService;
import com.imeeting.service.realtime.impl.LocalRealtimeAsrChannel;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.BinaryMessage;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.PingMessage;
import org.springframework.web.socket.PongMessage;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.AbstractWebSocketHandler;
import org.springframework.web.socket.handler.ConcurrentWebSocketSessionDecorator;
import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@Slf4j
@Component
@RequiredArgsConstructor
public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandler {
private static final String ATTR_FRONTEND_SESSION = "frontendSession";
private static final String ATTR_UPSTREAM_SOCKET = "upstreamSocket";
private static final String ATTR_MEETING_ID = "meetingId";
private static final String ATTR_TARGET_WS_URL = "targetWsUrl";
private static final String ATTR_PROVIDER = "provider";
private static final String ATTR_FRONTEND_TEXT_COUNT = "frontendTextCount";
private static final String ATTR_FRONTEND_BINARY_COUNT = "frontendBinaryCount";
private static final String ATTR_UPSTREAM_SEND_CHAIN = "upstreamSendChain";
private static final String ATTR_START_MESSAGE_SENT = "startMessageSent";
private static final String ATTR_PENDING_AUDIO_FRAMES = "pendingAudioFrames";
private static final CompletableFuture<Void> COMPLETED = CompletableFuture.completedFuture(null);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final RealtimeMeetingSocketSessionService realtimeMeetingSocketSessionService;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final RealtimeMeetingAudioStorageService realtimeMeetingAudioStorageService;
private final RealtimeMeetingTranscriptCacheService realtimeMeetingTranscriptCacheService;
private final RealtimeAsrChannelFactory realtimeAsrChannelFactory;
private final ConcurrentMap<Long, MeetingChannelSession> meetingSessions = new ConcurrentHashMap<>();
private final ConcurrentMap<Long, Object> meetingLocks = new ConcurrentHashMap<>();
@Override
public void afterConnectionEstablished(WebSocketSession session) throws Exception {
String sessionToken = extractQueryParam(session.getUri(), "sessionToken");
RealtimeSocketSessionData sessionData = realtimeMeetingSocketSessionService.getSessionData(sessionToken);
if (sessionData == null) {
log.warn("Realtime websocket rejected: invalid session token, sessionId={}", session.getId());
log.warn("实时会议 websocket 拒绝连接:会话令牌无效,sessionId={}", session.getId());
session.close(CloseStatus.POLICY_VIOLATION.withReason("实时 Socket 会话无效"));
return;
}
ConcurrentWebSocketSessionDecorator frontendSession =
new ConcurrentWebSocketSessionDecorator(session, (int) Duration.ofSeconds(15).toMillis(), 1024 * 1024);
session.getAttributes().put(ATTR_FRONTEND_SESSION, frontendSession);
session.getAttributes().put(ATTR_MEETING_ID, sessionData.getMeetingId());
session.getAttributes().put(ATTR_TARGET_WS_URL, sessionData.getTargetWsUrl());
session.getAttributes().put(ATTR_PROVIDER, sessionData.getProvider());
session.getAttributes().put(ATTR_FRONTEND_TEXT_COUNT, new AtomicInteger());
session.getAttributes().put(ATTR_FRONTEND_BINARY_COUNT, new AtomicInteger());
session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED);
session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.FALSE);
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
realtimeMeetingAudioStorageService.openSession(sessionData.getMeetingId(), session.getId());
log.info("Realtime websocket accepted: meetingId={}, sessionId={}, upstream={}",
sessionData.getMeetingId(), session.getId(), sessionData.getTargetWsUrl());
log.info("实时会议 websocket 已接入meetingId={}, sessionId={}, provider={}, upstream={}",
sessionData.getMeetingId(), session.getId(), sessionData.getProvider(), sessionData.getTargetWsUrl());
java.net.http.WebSocket upstreamSocket;
try {
upstreamSocket = java.net.http.HttpClient.newHttpClient()
.newWebSocketBuilder()
.buildAsync(URI.create(sessionData.getTargetWsUrl()),
new UpstreamListener(
frontendSession,
session,
sessionData.getMeetingId(),
sessionData.getTargetWsUrl(),
realtimeMeetingSessionStateService
))
.get();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.error("Realtime websocket upstream connect interrupted: meetingId={}, sessionId={}",
sessionData.getMeetingId(), session.getId(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接第三方识别服务时被中断");
realtimeMeetingAudioStorageService.closeSession(session.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断"));
return;
} catch (ExecutionException | CompletionException ex) {
log.warn("Failed to connect upstream websocket, meetingId={}, target={}", sessionData.getMeetingId(), sessionData.getTargetWsUrl(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接第三方识别服务失败,请检查模型 WebSocket 配置或服务状态");
realtimeMeetingAudioStorageService.closeSession(session.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败"));
return;
}
session.getAttributes().put(ATTR_UPSTREAM_SOCKET, upstreamSocket);
attachFrontendSession(sessionData, session, frontendSession);
}
@Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session);
if (upstreamSocket == null) {
log.warn("Frontend text ignored because upstream socket is unavailable, meetingId={}, sessionId={}",
MeetingChannelSession meetingSession = getMeetingSession(session);
if (meetingSession == null || !meetingSession.isChannelOpen()) {
log.warn("前端文本消息已忽略:上游 ASR 连接不可用meetingId={}, sessionId={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId());
return;
}
int count = nextCount(session, ATTR_FRONTEND_TEXT_COUNT);
log.info("Frontend text -> upstream: meetingId={}, sessionId={}, count={}, payload={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, summarizeText(message.getPayload()));
sendUpstreamOrdered(session, () -> upstreamSocket.sendText(message.getPayload(), true), "text");
if (looksLikeStartMessage(message.getPayload())) {
session.getAttributes().put(ATTR_START_MESSAGE_SENT, Boolean.TRUE);
flushPendingAudioFrames(session, upstreamSocket);
}
String payload = message.getPayload();
log.info("前端文本 -> ASR 渠道meetingId={}, sessionId={}, provider={}, count={}, payload={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, summarizeText(payload));
meetingSession.channel.handleFrontendText(meetingSession.context, payload);
}
@Override
protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session);
if (upstreamSocket == null) {
log.warn("Frontend binary ignored because upstream socket is unavailable, meetingId={}, sessionId={}",
MeetingChannelSession meetingSession = getMeetingSession(session);
if (meetingSession == null || !meetingSession.isChannelOpen()) {
log.warn("前端音频帧已忽略:上游 ASR 连接不可用meetingId={}, sessionId={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId());
return;
}
int count = nextCount(session, ATTR_FRONTEND_BINARY_COUNT);
int bytes = message.getPayloadLength();
if (shouldLogBinaryFrame(count)) {
log.info("Frontend binary -> upstream: meetingId={}, sessionId={}, count={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes);
log.info("前端音频帧 -> ASR 渠道meetingId={}, sessionId={}, provider={}, count={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_PROVIDER), count, bytes);
}
byte[] payload = toByteArray(message.getPayload());
realtimeMeetingAudioStorageService.append(session.getId(), payload);
if (!Boolean.TRUE.equals(session.getAttributes().get(ATTR_START_MESSAGE_SENT))) {
queuePendingAudioFrame(session, payload);
if (shouldLogBinaryFrame(count)) {
log.warn("Frontend binary queued before start message: meetingId={}, sessionId={}, count={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), count, bytes);
}
return;
}
sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(payload), true), "binary");
meetingSession.channel.handleFrontendBinary(meetingSession.context, payload);
}
@Override
protected void handlePongMessage(WebSocketSession session, PongMessage message) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session);
if (upstreamSocket == null) {
if (getMeetingSession(session) == null) {
return;
}
sendUpstreamOrdered(session, () -> upstreamSocket.sendPong(copyBuffer(message.getPayload())), "pong");
log.debug("前端 pong 已在本地忽略meetingId={}, sessionId={}, bytes={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), message.getPayloadLength());
}
@Override
public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception {
log.error("Realtime websocket transport error: meetingId={}, sessionId={}, upstream={}",
log.error("实时会议 websocket 传输异常:meetingId={}, sessionId={}, upstream={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), session.getAttributes().get(ATTR_TARGET_WS_URL), exception);
closeUpstreamSocket(session, CloseStatus.SERVER_ERROR);
detachFrontend(session);
realtimeMeetingAudioStorageService.closeSession(session.getId());
if (session.isOpen()) {
session.close(CloseStatus.SERVER_ERROR);
}
@ -175,32 +135,160 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
@Override
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
log.info("Realtime websocket closed: meetingId={}, sessionId={}, code={}, reason={}",
log.info("实时会议 websocket 已关闭:meetingId={}, sessionId={}, code={}, reason={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), status.getCode(), status.getReason());
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (meetingIdValue instanceof Long meetingId) {
detachFrontend(meetingId, session.getId());
realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, session.getId());
}
realtimeMeetingAudioStorageService.closeSession(session.getId());
closeUpstreamSocket(session, status);
}
private java.net.http.WebSocket getUpstreamSocket(WebSocketSession session) {
Object socket = session.getAttributes().get(ATTR_UPSTREAM_SOCKET);
if (socket instanceof java.net.http.WebSocket webSocket) {
return webSocket;
public void closeMeetingSession(Long meetingId) {
if (meetingId == null) {
return;
}
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null || meetingSession.channel == null) {
return;
}
meetingSession.channel.closeMeeting(meetingSession.context);
}
private void attachFrontendSession(RealtimeSocketSessionData sessionData,
WebSocketSession rawSession,
ConcurrentWebSocketSessionDecorator frontendSession) throws Exception {
Long meetingId = sessionData.getMeetingId();
MeetingChannelSession meetingSession;
boolean reused = false;
synchronized (lockForMeeting(meetingId)) {
meetingSession = meetingSessions.get(meetingId);
if (meetingSession != null && meetingSession.isChannelOpen()) {
String previousSessionId = meetingSession.context.getRawSession() == null
? null
: meetingSession.context.getRawSession().getId();
meetingSession.clearFrontendIfClosed();
if (previousSessionId != null && !meetingSession.hasOpenFrontend()) {
realtimeMeetingSessionStateService.pauseByDisconnect(meetingId, previousSessionId);
}
return null;
if (meetingSession.hasOpenFrontend()) {
sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃前端连接");
frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("已存在活跃的前端连接"));
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
return;
}
if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) {
sendFrontendError(frontendSession, "REALTIME_ACTIVE_CONNECTION_REJECTED", "当前状态下无法继续会议");
frontendSession.close(CloseStatus.POLICY_VIOLATION.withReason("当前状态下无法继续会议"));
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
return;
}
meetingSession.bindFrontend(rawSession, frontendSession);
reused = true;
} else {
RealtimeAsrChannel channel = realtimeAsrChannelFactory.getRequired(sessionData.getProvider());
RealtimeAsrChannelContext context = new RealtimeAsrChannelContext();
context.setMeetingId(meetingId);
context.setProvider(realtimeAsrChannelFactory.normalizeProvider(sessionData.getProvider()));
context.setTargetWsUrl(sessionData.getTargetWsUrl());
context.setCallback(new HandlerChannelCallback());
context.bindFrontendSession(rawSession, frontendSession);
context.getChannelState().put("modelCode", sessionData.getModelCode());
context.getChannelState().put("mediaConfig", sessionData.getMediaConfig());
meetingSession = new MeetingChannelSession(meetingId, channel, context);
meetingSessions.put(meetingId, meetingSession);
}
}
private void closeUpstreamSocket(WebSocketSession session, CloseStatus status) {
java.net.http.WebSocket upstreamSocket = getUpstreamSocket(session);
if (upstreamSocket != null) {
upstreamSocket.sendClose(status.getCode(), status.getReason() == null ? "" : status.getReason());
session.getAttributes().remove(ATTR_UPSTREAM_SOCKET);
if (reused) {
sendProxyReady(frontendSession);
replayCachedMessages(meetingId, frontendSession);
return;
}
try {
meetingSession.channel.connect(meetingSession.context);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
removeMeetingSession(meetingId, meetingSession);
log.error("连接上游 ASR websocket 时被中断meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_INTERRUPTED", "连接上游 ASR 服务时被中断");
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接上游服务时被中断"));
} catch (Exception ex) {
removeMeetingSession(meetingId, meetingSession);
log.warn("连接上游 ASR websocket 失败meetingId={}, provider={}, target={}",
meetingId, sessionData.getProvider(), sessionData.getTargetWsUrl(), ex);
sendFrontendError(frontendSession, "REALTIME_UPSTREAM_CONNECT_FAILED", "连接上游 ASR 服务失败");
realtimeMeetingAudioStorageService.closeSession(rawSession.getId());
frontendSession.close(CloseStatus.SERVER_ERROR.withReason("连接 ASR WebSocket 失败"));
}
}
private void replayCachedMessages(Long meetingId, ConcurrentWebSocketSessionDecorator frontendSession) {
try {
if (!frontendSession.isOpen()) {
return;
}
for (RealtimeMeetingTranscriptCacheItem item : realtimeMeetingTranscriptCacheService.listOrderedItems(meetingId)) {
frontendSession.sendMessage(new TextMessage(LocalRealtimeAsrChannel.buildFrontendTranscriptMessage(item)));
}
} catch (Exception ex) {
log.warn("回放缓存转写消息失败meetingId={}", meetingId, ex);
}
}
private void sendProxyReady(ConcurrentWebSocketSessionDecorator frontendSession) throws Exception {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}"));
}
}
private void detachFrontend(WebSocketSession session) {
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (meetingIdValue instanceof Long meetingId) {
detachFrontend(meetingId, session.getId());
}
}
private void detachFrontend(Long meetingId, String sessionId) {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
synchronized (lockForMeeting(meetingId)) {
if (meetingSession.context.getRawSession() != null && meetingSession.context.getRawSession().getId().equals(sessionId)) {
meetingSession.channel.onFrontendDetached(meetingSession.context);
}
meetingSession.detachFrontend(sessionId);
}
}
private MeetingChannelSession getMeetingSession(WebSocketSession session) {
Object meetingIdValue = session.getAttributes().get(ATTR_MEETING_ID);
if (!(meetingIdValue instanceof Long meetingId)) {
return null;
}
return meetingSessions.get(meetingId);
}
void removeMeetingSession(Long meetingId) {
synchronized (lockForMeeting(meetingId)) {
meetingSessions.remove(meetingId);
}
}
void removeMeetingSession(Long meetingId, MeetingChannelSession meetingSession) {
synchronized (lockForMeeting(meetingId)) {
meetingSessions.remove(meetingId, meetingSession);
}
}
private Object lockForMeeting(Long meetingId) {
return meetingLocks.computeIfAbsent(meetingId, ignored -> new Object());
}
private String extractQueryParam(URI uri, String key) {
if (uri == null || uri.getQuery() == null || uri.getQuery().isBlank()) {
return null;
@ -213,13 +301,6 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
.orElse(null);
}
private ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
private byte[] toByteArray(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
@ -235,21 +316,18 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
return 0;
}
@SuppressWarnings("unchecked")
private void sendUpstreamOrdered(WebSocketSession session, Supplier<CompletableFuture<?>> sendAction, String messageType) {
synchronized (session) {
CompletableFuture<Void> chain = (CompletableFuture<Void>) session.getAttributes()
.getOrDefault(ATTR_UPSTREAM_SEND_CHAIN, COMPLETED);
CompletableFuture<Void> nextChain = chain
.exceptionally(ex -> null)
.thenCompose(ignored -> sendAction.get().thenApply(ignoredResult -> null));
nextChain = nextChain.whenComplete((ignored, ex) -> {
if (ex != null) {
log.error("Ordered upstream send failed: meetingId={}, sessionId={}, type={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), messageType, ex);
}
});
session.getAttributes().put(ATTR_UPSTREAM_SEND_CHAIN, nextChain);
private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) {
try {
if (!frontendSession.isOpen()) {
return;
}
Map<String, Object> payload = new HashMap<>();
payload.put("type", "error");
payload.put("code", code);
payload.put("message", message);
frontendSession.sendMessage(new TextMessage(OBJECT_MAPPER.writeValueAsString(payload)));
} catch (Exception ex) {
log.warn("向前端发送实时代理错误消息失败code={}", code, ex);
}
}
@ -268,237 +346,113 @@ public class RealtimeMeetingProxyWebSocketHandler extends AbstractWebSocketHandl
return normalized.substring(0, 240) + "...";
}
private boolean looksLikeStartMessage(String payload) {
if (payload == null || payload.isBlank()) {
return false;
}
String normalized = payload.replaceAll("\\s+", "");
return normalized.contains("\"type\":\"start\"");
}
private void sendFrontendError(ConcurrentWebSocketSessionDecorator frontendSession, String code, String message) {
try {
if (!frontendSession.isOpen()) {
return;
}
Map<String, Object> payload = new HashMap<>();
payload.put("type", "error");
payload.put("code", code);
payload.put("message", message);
frontendSession.sendMessage(new TextMessage(new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(payload)));
} catch (Exception ex) {
log.warn("Failed to send realtime proxy error to frontend: code={}", code, ex);
}
}
@SuppressWarnings("unchecked")
private void queuePendingAudioFrame(WebSocketSession session, byte[] payload) {
synchronized (session) {
List<byte[]> pendingFrames = (List<byte[]>) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES);
if (pendingFrames == null) {
pendingFrames = new ArrayList<>();
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, pendingFrames);
}
pendingFrames.add(payload);
}
}
@SuppressWarnings("unchecked")
private void flushPendingAudioFrames(WebSocketSession session, java.net.http.WebSocket upstreamSocket) {
List<byte[]> pendingFrames;
synchronized (session) {
pendingFrames = (List<byte[]>) session.getAttributes().get(ATTR_PENDING_AUDIO_FRAMES);
if (pendingFrames == null || pendingFrames.isEmpty()) {
return;
}
session.getAttributes().put(ATTR_PENDING_AUDIO_FRAMES, new ArrayList<byte[]>());
}
log.info("Flushing queued audio frames after start message: meetingId={}, sessionId={}, frameCount={}",
session.getAttributes().get(ATTR_MEETING_ID), session.getId(), pendingFrames.size());
for (byte[] frame : pendingFrames) {
sendUpstreamOrdered(session, () -> upstreamSocket.sendBinary(ByteBuffer.wrap(frame), true), "binary-flush");
}
}
private static final class UpstreamListener implements java.net.http.WebSocket.Listener {
private final ConcurrentWebSocketSessionDecorator frontendSession;
private final WebSocketSession rawSession;
static final class MeetingChannelSession {
private final Long meetingId;
private final String targetWsUrl;
private final RealtimeMeetingSessionStateService realtimeMeetingSessionStateService;
private final StringBuilder textBuffer = new StringBuilder();
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
private final AtomicInteger upstreamTextCount = new AtomicInteger();
private final AtomicInteger upstreamBinaryCount = new AtomicInteger();
private final RealtimeAsrChannel channel;
private final RealtimeAsrChannelContext context;
private UpstreamListener(ConcurrentWebSocketSessionDecorator frontendSession, WebSocketSession rawSession,
Long meetingId, String targetWsUrl,
RealtimeMeetingSessionStateService realtimeMeetingSessionStateService) {
this.frontendSession = frontendSession;
this.rawSession = rawSession;
private MeetingChannelSession(Long meetingId, RealtimeAsrChannel channel, RealtimeAsrChannelContext context) {
this.meetingId = meetingId;
this.targetWsUrl = targetWsUrl;
this.realtimeMeetingSessionStateService = realtimeMeetingSessionStateService;
this.channel = channel;
this.context = context;
}
private void bindFrontend(WebSocketSession rawSession, ConcurrentWebSocketSessionDecorator frontendSession) {
context.bindFrontendSession(rawSession, frontendSession);
}
private void detachFrontend(String sessionId) {
if (context.getRawSession() != null && context.getRawSession().getId().equals(sessionId)) {
context.bindFrontendSession(null, null);
}
}
private void clearFrontendIfClosed() {
if (context.getRawSession() != null && !context.getRawSession().isOpen()) {
context.bindFrontendSession(null, null);
}
}
private boolean hasOpenFrontend() {
return context.getFrontendSession() != null
&& context.getFrontendSession().isOpen()
&& context.getRawSession() != null
&& context.getRawSession().isOpen();
}
private boolean isChannelOpen() {
return channel != null && channel.isOpen(context);
}
}
private final class HandlerChannelCallback implements RealtimeAsrChannelCallback {
@Override
public void onOpen(java.net.http.WebSocket webSocket) {
log.info("Upstream websocket opened: meetingId={}, sessionId={}, upstream={}",
meetingId, rawSession.getId(), targetWsUrl);
if (!realtimeMeetingSessionStateService.activate(meetingId, rawSession.getId())) {
sendFrontendError("REALTIME_ACTIVE_CONNECTION_EXISTS", "当前会议已有活跃实时连接,请先关闭旧连接后再继续");
webSocket.sendClose(CloseStatus.POLICY_VIOLATION.getCode(), "Active realtime connection already exists");
closeFrontend(CloseStatus.POLICY_VIOLATION.withReason("已存在活动的实时连接"));
public void onChannelOpen(Long meetingId) throws Exception {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage("{\"type\":\"proxy_ready\"}"));
}
} catch (Exception ex) {
log.error("Failed to notify frontend that upstream websocket is ready: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession != null && frontendSession.isOpen()) {
sendProxyReady(frontendSession);
}
}
@Override
public void sendFrontendText(Long meetingId, String payload) throws Exception {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
webSocket.request(1);
ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession != null && frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage(payload));
}
}
@Override
public java.util.concurrent.CompletionStage<?> onText(java.net.http.WebSocket webSocket, CharSequence data, boolean last) {
textBuffer.append(data);
if (last) {
int count = upstreamTextCount.incrementAndGet();
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new TextMessage(textBuffer.toString()));
}
log.info("Upstream text -> frontend: meetingId={}, sessionId={}, count={}, payload={}",
meetingId, rawSession.getId(), count, summarizeText(textBuffer.toString()));
} catch (Exception ex) {
log.error("Failed to forward upstream text: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
} finally {
textBuffer.setLength(0);
}
}
webSocket.request(1);
return COMPLETED;
public void sendFrontendBinary(Long meetingId, byte[] payload) throws Exception {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession != null && frontendSession.isOpen()) {
frontendSession.sendMessage(new BinaryMessage(payload));
}
}
@Override
public java.util.concurrent.CompletionStage<?> onBinary(java.net.http.WebSocket webSocket, ByteBuffer data, boolean last) {
byte[] chunk = new byte[data.remaining()];
data.get(chunk);
binaryBuffer.writeBytes(chunk);
if (last) {
int count = upstreamBinaryCount.incrementAndGet();
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new BinaryMessage(binaryBuffer.toByteArray()));
}
if (shouldLogBinaryFrame(count)) {
log.info("Upstream binary -> frontend: meetingId={}, sessionId={}, count={}, bytes={}",
meetingId, rawSession.getId(), count, binaryBuffer.size());
}
} catch (Exception ex) {
log.error("Failed to forward upstream binary: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
} finally {
binaryBuffer.reset();
}
}
webSocket.request(1);
return COMPLETED;
public void sendFrontendError(Long meetingId, String code, String message) {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
ConcurrentWebSocketSessionDecorator frontendSession = meetingSession.context.getFrontendSession();
if (frontendSession != null) {
RealtimeMeetingProxyWebSocketHandler.this.sendFrontendError(frontendSession, code, message);
}
}
@Override
public java.util.concurrent.CompletionStage<?> onPing(java.net.http.WebSocket webSocket, ByteBuffer message) {
public void removeMeetingSession(Long meetingId) {
RealtimeMeetingProxyWebSocketHandler.this.removeMeetingSession(meetingId);
}
@Override
public void closeFrontend(Long meetingId, CloseStatus status) {
MeetingChannelSession meetingSession = meetingSessions.get(meetingId);
if (meetingSession == null) {
return;
}
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new PingMessage(copyBuffer(message)));
}
log.info("Upstream ping -> frontend: meetingId={}, sessionId={}, bytes={}",
meetingId, rawSession.getId(), message.remaining());
} catch (Exception ex) {
log.error("Failed to forward upstream ping: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onPong(java.net.http.WebSocket webSocket, ByteBuffer message) {
try {
if (frontendSession.isOpen()) {
frontendSession.sendMessage(new PongMessage(copyBuffer(message)));
}
log.info("Upstream pong -> frontend: meetingId={}, sessionId={}, bytes={}",
meetingId, rawSession.getId(), message.remaining());
} catch (Exception ex) {
log.error("Failed to forward upstream pong: meetingId={}, sessionId={}", meetingId, rawSession.getId(), ex);
closeFrontend(CloseStatus.SERVER_ERROR);
}
webSocket.request(1);
return COMPLETED;
}
@Override
public java.util.concurrent.CompletionStage<?> onClose(java.net.http.WebSocket webSocket, int statusCode, String reason) {
log.info("Upstream websocket closed: meetingId={}, sessionId={}, code={}, reason={}",
meetingId, rawSession.getId(), statusCode, reason);
sendFrontendError("REALTIME_UPSTREAM_CLOSED", reason == null || reason.isBlank() ? "第三方识别服务已断开连接" : "第三方识别服务已断开: " + reason);
closeFrontend(new CloseStatus(statusCode, reason));
return COMPLETED;
}
@Override
public void onError(java.net.http.WebSocket webSocket, Throwable error) {
log.error("Upstream websocket error: meetingId={}, sessionId={}, upstream={}",
meetingId, rawSession.getId(), targetWsUrl, error);
sendFrontendError("REALTIME_UPSTREAM_ERROR", error == null || error.getMessage() == null || error.getMessage().isBlank()
? "第三方识别服务连接异常"
: "第三方识别服务连接异常: " + error.getMessage());
closeFrontend(CloseStatus.SERVER_ERROR);
}
private void sendFrontendError(String code, String message) {
try {
if (!frontendSession.isOpen()) {
return;
}
frontendSession.sendMessage(new TextMessage("{\"type\":\"error\",\"code\":\"" + code + "\",\"message\":\"" + escapeJson(message) + "\"}"));
} catch (Exception ex) {
log.warn("Failed to send upstream error to frontend: meetingId={}, sessionId={}, code={}", meetingId, rawSession.getId(), code, ex);
}
}
private String escapeJson(String value) {
if (value == null) {
return "";
}
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\r", "\\r")
.replace("\n", "\\n");
}
private void closeFrontend(CloseStatus status) {
try {
if (rawSession.isOpen()) {
WebSocketSession rawSession = meetingSession.context.getRawSession();
if (rawSession != null && rawSession.isOpen()) {
rawSession.close(status);
}
} catch (Exception ignored) {
// ignore close failure
}
}
private ByteBuffer copyBuffer(ByteBuffer source) {
ByteBuffer duplicate = source.asReadOnlyBuffer();
byte[] bytes = new byte[duplicate.remaining()];
duplicate.get(bytes);
return ByteBuffer.wrap(bytes);
}
}
}

View File

@ -11,8 +11,23 @@ option java_outer_classname = "PushProto";
// =========================
enum Platform {
PLATFORM_UNKNOWN = 0;
// Mobile
ANDROID = 1;
IOS = 2;
HARMONY_MOBILE = 3;
// Desktop
WINDOWS = 10;
MACOS = 11;
LINUX = 12;
// Linux
KYLIN = 20;
UOS = 21;
// Harmony PC
HARMONY_PC = 30;
}
// =========================

View File

@ -349,6 +349,64 @@ class AiModelServiceImplTest {
assertNull(captor.getValue().getApiKey());
}
@Test
void saveModelShouldRejectTencentAsrWithoutSecretKey() {
AiModelServiceImpl service = new AiModelServiceImpl(
objectMapper,
mock(AsrModelMapper.class),
mock(LlmModelMapper.class)
);
AiModelDTO dto = new AiModelDTO();
dto.setModelType("ASR");
dto.setModelName("tencent-asr");
dto.setProvider("tencent");
dto.setModelCode("16k_zh");
dto.setIsDefault(0);
dto.setStatus(1);
dto.setMediaConfig(Map.of(
"tencentAppId", "app-id",
"tencentSecretId", "secret-id"
));
RuntimeException ex = assertThrows(RuntimeException.class, () -> service.saveModel(dto));
assertEquals("腾讯实时 ASR 模型必须配置 mediaConfig.tencentSecretKey", ex.getMessage());
}
@Test
void saveModelShouldPersistTencentAsrWithoutBaseUrl() {
AsrModelMapper asrModelMapper = mock(AsrModelMapper.class);
when(asrModelMapper.insert(any(AsrModel.class))).thenReturn(1);
AiModelServiceImpl service = new AiModelServiceImpl(
objectMapper,
asrModelMapper,
mock(LlmModelMapper.class)
);
AiModelDTO dto = new AiModelDTO();
dto.setModelType("ASR");
dto.setModelName("tencent-asr");
dto.setProvider("tencent");
dto.setModelCode("16k_zh");
dto.setIsDefault(0);
dto.setStatus(1);
dto.setMediaConfig(Map.of(
"tencentAppId", "app-id",
"tencentSecretId", "secret-id",
"tencentSecretKey", "secret-key"
));
service.saveModel(dto);
ArgumentCaptor<AsrModel> captor = ArgumentCaptor.forClass(AsrModel.class);
verify(asrModelMapper, times(1)).insert(captor.capture());
assertEquals("tencent", captor.getValue().getProvider());
assertEquals("16k_zh", captor.getValue().getModelCode());
assertEquals("secret-key", captor.getValue().getMediaConfig().get("tencentSecretKey"));
assertNull(captor.getValue().getBaseUrl());
}
private void captureRequest(HttpExchange exchange,
AtomicReference<String> requestPath,
AtomicReference<String> authorization,

View File

@ -9,6 +9,7 @@ export type SummaryDetailLevel = "DETAILED" | "STANDARD" | "BRIEF";
export interface MeetingCreateConfig {
offlineEnabled: boolean;
realtimeEnabled: boolean;
aiCatalogEnabled?: boolean;
offlineAudioMaxSizeMb: number;
chunkUploadEnabled?: boolean;
chunkDurationSeconds?: number;
@ -35,6 +36,7 @@ export interface MeetingVO {
summaryDetailLevel?: SummaryDetailLevel;
summaryModelId: number;
promptId?: number;
aiCatalogEnabled?: boolean;
audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED";
audioSaveMessage?: string;
accessPassword?: string;
@ -247,12 +249,6 @@ export const createRealtimeMeeting = (data: CreateRealtimeMeetingCommand) => {
);
};
export const appendRealtimeTranscripts = (meetingId: number, data: RealtimeTranscriptItemDTO[]) => {
return http.post<{ code: string; data: boolean; msg: string }>(
`/api/biz/meeting/${meetingId}/realtime/transcripts`,
data
);
};
export const getRealtimeMeetingSessionStatus = (meetingId: number) => {
return http.get<{ code: string; data: RealtimeMeetingSessionStatus; msg: string }>(

View File

@ -138,6 +138,7 @@ export default function Users() {
const [filterTenantId, setFilterTenantId] = useState<number | undefined>(undefined);
const [drawerOpen, setDrawerOpen] = useState(false);
const [editing, setEditing] = useState<SysUser | null>(null);
const [roleSelectOpen, setRoleSelectOpen] = useState(false);
const [form] = Form.useForm();
const [resetPasswordOpen, setResetPasswordOpen] = useState(false);
const [resetPasswordLoading, setResetPasswordLoading] = useState(false);
@ -264,6 +265,7 @@ export default function Users() {
const openCreate = () => {
setEditing(null);
setRoleSelectOpen(false);
form.resetFields();
form.setFieldsValue({
status: 1,
@ -277,6 +279,7 @@ export default function Users() {
const openEdit = async (record: SysUser) => {
setEditing(record);
setRoleSelectOpen(false);
try {
const detail = await getUserDetail(record.userId);
const roleIds = await listUserRoles(record.userId);
@ -594,10 +597,13 @@ export default function Users() {
</Form.Item>
) : null}
<Form.Item label={t("users.roles")} name="roleIds"
rules={[{required: true, message: t("users.roles")}]}><Select mode="multiple"
placeholder={t("users.roles")}
options={roleOptions}
optionFilterProp={isPlatformMode ? "searchText" : "label"}/></Form.Item>
rules={[{required: true, message: t("users.roles")}]}><Select mode="multiple"
placeholder={t("users.roles")}
options={roleOptions}
open={roleSelectOpen}
onDropdownVisibleChange={setRoleSelectOpen}
onChange={() => setRoleSelectOpen(false)}
optionFilterProp={isPlatformMode ? "searchText" : "label"}/></Form.Item>
{!isPlatformMode && <Form.Item label={t("users.orgNode")} name="orgId"><TreeSelect
placeholder={t("usersExt.selectOrgPlaceholder")} allowClear treeData={orgTreeData}/></Form.Item>}
<Row gutter={16}>

View File

@ -70,6 +70,7 @@ const AiModels: React.FC = () => {
const provider = Form.useWatch("provider", form);
const isDefaultChecked = Form.useWatch("isDefaultChecked", form);
const isLocalProvider = String(provider || "").toLowerCase() === "custom";
const isTencentProvider = String(provider || "").toLowerCase() === "tencent";
const isPlatformAdmin = useMemo(() => {
const profileStr = sessionStorage.getItem("userProfile");
@ -133,11 +134,17 @@ const AiModels: React.FC = () => {
setEditingId(record.id);
const speakerModel = record.mediaConfig?.speakerModel;
const svThreshold = record.mediaConfig?.svThreshold;
const tencentAppId = record.mediaConfig?.tencentAppId;
const tencentSecretId = record.mediaConfig?.tencentSecretId;
const tencentSecretKey = record.mediaConfig?.tencentSecretKey;
form.setFieldsValue({
...record,
modelType: record.modelType,
speakerModel,
svThreshold,
tencentAppId,
tencentSecretId,
tencentSecretKey,
isDefaultChecked: record.isDefault === 1,
statusChecked: record.status === 1,
});
@ -262,6 +269,12 @@ const AiModels: React.FC = () => {
speakerModel: values.speakerModel,
svThreshold: values.svThreshold,
}
: activeType === "ASR" && isTencentProvider
? {
tencentAppId: values.tencentAppId,
tencentSecretId: values.tencentSecretId,
tencentSecretKey: values.tencentSecretKey,
}
: undefined,
temperature: values.temperature,
topP: values.topP,
@ -524,16 +537,20 @@ const AiModels: React.FC = () => {
</Col>
</Row>
<Form.Item name="baseUrl" label="Base URL" rules={[{ required: true, message: "请输入 Base URL" }]}>
<Input placeholder="https://api.example.com" />
</Form.Item>
{!isTencentProvider && (
<>
<Form.Item name="baseUrl" label="Base URL" rules={[{required: true, message: "请输入 Base URL"}]}>
<Input placeholder="https://api.example.com"/>
</Form.Item>
<Form.Item
name="apiKey"
label="API Key"
>
<Input.Password />
</Form.Item>
<Form.Item
name="apiKey"
label="API Key"
>
<Input.Password/>
</Form.Item>
</>
)}
{(activeType === "LLM" || isLocalProvider) && (
<Form.Item label="连通性测试">
@ -556,7 +573,10 @@ const AiModels: React.FC = () => {
<Form.Item
name="modelCode"
noStyle
rules={activeType === "LLM" ? [{ required: true, message: "请输入或选择模型名称" }] : []}
rules={activeType === "LLM" || isTencentProvider ? [{
required: true,
message: "请输入或选择模型名称"
}] : []}
>
<AutoComplete
style={{ width: "calc(100% - 100px)" }}
@ -574,9 +594,11 @@ const AiModels: React.FC = () => {
<Input allowClear placeholder="可选择或自定义输入模型名称" />
</AutoComplete>
</Form.Item>
<Button icon={<SyncOutlined spin={fetchLoading} />} onClick={handleFetchRemote} style={{ width: 100 }}>
</Button>
{!isTencentProvider && (
<Button icon={<SyncOutlined spin={fetchLoading}/>} onClick={handleFetchRemote} style={{width: 100}}>
</Button>
)}
</Space.Compact>
</Form.Item>
@ -609,6 +631,38 @@ const AiModels: React.FC = () => {
</Row>
)}
{activeType === "ASR" && isTencentProvider && (
<Row gutter={16}>
<Col span={12}>
<Form.Item
name="tencentAppId"
label="App ID"
rules={[{required: true, message: "请输入 App ID"}]}
>
<Input/>
</Form.Item>
</Col>
<Col span={12}>
<Form.Item
name="tencentSecretId"
label="Secret ID"
rules={[{required: true, message: "请输入 Secret ID"}]}
>
<Input/>
</Form.Item>
</Col>
<Col span={24}>
<Form.Item
name="tencentSecretKey"
label="Secret Key"
rules={[{required: true, message: "请输入 Secret Key"}]}
>
<Input.Password/>
</Form.Item>
</Col>
</Row>
)}
{activeType === "LLM" && (
<>
<Form.Item name="apiPath" label="API 路径" initialValue="/v1/chat/completions">

View File

@ -1226,7 +1226,7 @@ const MeetingDetail: React.FC = () => {
const [expandKeywords, setExpandKeywords] = useState(false);
const [expandSummary, setExpandSummary] = useState(false);
const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
const [workspaceTab, setWorkspaceTab] = useState<WorkspaceTab>('catalog');
const [workspaceTab, setWorkspaceTab] = useState<WorkspaceTab>('transcript');
const [addingHotwords, setAddingHotwords] = useState(false);
const [editingTranscriptId, setEditingTranscriptId] = useState<number | null>(null);
const [savingTranscriptId, setSavingTranscriptId] = useState<number | null>(null);
@ -1402,15 +1402,16 @@ const MeetingDetail: React.FC = () => {
return false;
}, [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const canRetrySummary = isOwner
&& transcripts.length > 0
&& meeting?.status !== 1
&& meeting?.status !== 2
&& meeting?.latestSummaryAttemptStatus !== 3
&& meeting?.latestChapterAttemptStatus !== 3;
&& (!aiCatalogEnabled || meeting?.latestChapterAttemptStatus !== 3);
const canRetryTranscription = isOwner && meeting?.status === 4 && transcripts.length === 0 && !!meeting?.audioUrl;
const canRetryFailedSummaryTask = isOwner && meeting?.latestSummaryAttemptStatus === 3 && meeting?.status !== 2;
const canRetryFailedChapterTask = isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2;
const canRetryFailedChapterTask = aiCatalogEnabled && isOwner && meeting?.latestChapterAttemptStatus === 3 && meeting?.status !== 2;
const canRetrySchedule = isOwner && meeting?.status === 0 && (!generationProgress || generationProgress.percent <= 0) && !!generationProgress?.queuedAt && dayjs().diff(dayjs(generationProgress.queuedAt)) >= QUEUED_RETRY_THRESHOLD_MS;
@ -1433,14 +1434,14 @@ const MeetingDetail: React.FC = () => {
};
const hasSummaryContent = Boolean(meeting?.summaryContent?.trim());
const hasCatalogContent = catalogChapterLinks.length > 0;
const hasCatalogContent = aiCatalogEnabled && catalogChapterLinks.length > 0;
const generationFailureNotice = useMemo<MeetingStateNotice | null>(() => {
if (!meeting || meeting.status !== 4) {
return null;
}
const hasFallbackContent = hasSummaryContent || hasCatalogContent;
if (meeting.latestChapterAttemptStatus === 3) {
if (aiCatalogEnabled && meeting.latestChapterAttemptStatus === 3) {
const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败';
return {
title: hasFallbackContent ? '历史内容仍可查看' : '本次 AI 目录生成失败',
@ -1467,45 +1468,13 @@ const MeetingDetail: React.FC = () => {
}
return {
title: hasFallbackContent ? '历史内容仍可查看' : '会议处理异常',
description: hasFallbackContent
? '最近一次处理未成功,当前展示的是最近一次成功生成的内容。你可以继续查看,或重新发起识别/总结。'
: '会议在处理中遇到问题,暂时没有可展示的总结内容。你可以重新发起识别或总结。',
title: '会议处理异常',
description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。',
type: hasFallbackContent ? 'info' : 'warning',
hasFallbackContent,
scope: 'global',
};
if (meeting.latestChapterAttemptStatus === 3) {
const detail = meeting.latestChapterAttemptErrorMsg || '章节生成失败';
return {
key: `chapter-${meeting.latestChapterAttemptTaskId ?? 'latest'}`,
title: '本次重新总结失败',
description: hasFallbackContent
? `章节生成失败,当前展示的是上一次成功的摘要和 AI 目录。失败原因:${detail}`
: `章节生成失败,且当前没有可展示的历史摘要或 AI 目录。失败原因:${detail}`,
hasFallbackContent,
};
}
if (meeting.latestSummaryAttemptStatus === 3) {
const detail = meeting.latestSummaryAttemptErrorMsg || '总结生成失败';
return {
key: `summary-${meeting.latestSummaryAttemptTaskId ?? 'latest'}`,
title: '本次重新总结失败',
description: hasFallbackContent
? `总结生成失败,当前展示的是上一次成功的摘要和 AI 目录。失败原因:${detail}`
: `总结生成失败,且当前没有可展示的历史摘要或 AI 目录。失败原因:${detail}`,
hasFallbackContent,
};
}
return {
key: 'general-failure',
title: '会议处理异常',
description: '会议在处理过程中遇到了问题。您可以尝试重新发起识别或总结。',
hasFallbackContent,
};
}, [hasCatalogContent, hasSummaryContent, meeting]);
}, [aiCatalogEnabled, hasCatalogContent, hasSummaryContent, meeting]);
const summaryPanelNotice = useMemo<MeetingStateNotice | null>(() => {
if (!meeting || !hasSummaryContent) {
return null;
@ -1522,7 +1491,7 @@ const MeetingDetail: React.FC = () => {
return null;
}, [generationFailureNotice, hasSummaryContent, meeting]);
const catalogPanelNotice = useMemo<MeetingStateNotice | null>(() => {
if (!generationFailureNotice || generationFailureNotice.scope !== 'catalog') {
if (!aiCatalogEnabled || !generationFailureNotice || generationFailureNotice.scope !== 'catalog') {
return null;
}
if (generationFailureNotice.hasFallbackContent && hasCatalogContent) {
@ -1535,7 +1504,7 @@ const MeetingDetail: React.FC = () => {
};
}
return generationFailureNotice;
}, [generationFailureNotice, hasCatalogContent]);
}, [aiCatalogEnabled, generationFailureNotice, hasCatalogContent]);
const emptyTranscriptFailureNotice = useMemo(() => {
if (!meeting || meeting.status !== 4 || transcripts.length > 0) {
return null;
@ -1557,12 +1526,18 @@ const MeetingDetail: React.FC = () => {
}
}, [meeting?.id, meeting?.status]);
useEffect(() => {
if (!aiCatalogEnabled && workspaceTab === 'catalog') {
setWorkspaceTab('transcript');
}
}, [aiCatalogEnabled, workspaceTab]);
useEffect(() => {
const attemptKey = String(meeting?.latestChapterAttemptTaskId ?? meeting?.latestSummaryAttemptTaskId ?? '');
if (meeting?.status !== 2 || !attemptKey) {
return;
}
if ((generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) {
if (!aiCatalogEnabled || (generationProgress?.percent ?? 0) < 88 || catalogChapterLinks.length === 0) {
return;
}
if (autoOpenedCatalogAttemptRef.current === attemptKey) {
@ -1571,6 +1546,7 @@ const MeetingDetail: React.FC = () => {
autoOpenedCatalogAttemptRef.current = attemptKey;
setWorkspaceTab('catalog');
}, [
aiCatalogEnabled,
catalogChapterLinks.length,
generationProgress?.percent,
meeting?.latestChapterAttemptTaskId,
@ -2766,13 +2742,15 @@ const MeetingDetail: React.FC = () => {
)}
<div className="transcript-stage-tabs">
<button
type="button"
className={workspaceTab === 'catalog' ? 'active' : ''}
onClick={() => setWorkspaceTab('catalog')}
>
AI
</button>
{aiCatalogEnabled && (
<button
type="button"
className={workspaceTab === 'catalog' ? 'active' : ''}
onClick={() => setWorkspaceTab('catalog')}
>
AI
</button>
)}
<button
type="button"
className={workspaceTab === 'transcript' ? 'active' : ''}
@ -2783,7 +2761,7 @@ const MeetingDetail: React.FC = () => {
</div>
<div className="transcript-scroll-shell">
{workspaceTab === 'catalog' ? (
{aiCatalogEnabled && workspaceTab === 'catalog' ? (
<div className="catalog-list">
{catalogPanelNotice && (
<Alert

View File

@ -339,6 +339,7 @@ export default function MeetingPreview() {
const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]);
const keywords = useMemo(() => analysis.keywords || [], [analysis.keywords]);
const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const statusMeta = STATUS_META[meeting?.status || 0] || {
label: TEXT.statusPending,
className: "is-warning",
@ -416,6 +417,12 @@ export default function MeetingPreview() {
});
}, [analysis.chapters, meetingChapters, transcripts]);
useEffect(() => {
if (!aiCatalogEnabled && pageTab === "catalog") {
setPageTab("summary");
}
}, [aiCatalogEnabled, pageTab]);
useEffect(() => {
if (!activeTranscriptId) {
return;
@ -967,7 +974,7 @@ export default function MeetingPreview() {
onChange={(key) => setPageTab(key as PreviewPageTab)}
items={[
{ key: "summary", label: TEXT.pageSummary },
{ key: "catalog", label: TEXT.pageCatalog },
...(aiCatalogEnabled ? [{ key: "catalog", label: TEXT.pageCatalog }] : []),
{ key: "transcript", label: TEXT.pageTranscript },
]}
/>
@ -975,7 +982,7 @@ export default function MeetingPreview() {
<div className="meeting-preview-tab-content">
{pageTab === "summary" ? summaryTabContent : null}
{pageTab === "catalog" ? catalogTabContent : null}
{aiCatalogEnabled && pageTab === "catalog" ? catalogTabContent : null}
{pageTab === "transcript" ? transcriptTabContent : null}
</div>
</div>

View File

@ -10,7 +10,6 @@
InfoCircleOutlined,
PauseCircleOutlined,
PlusOutlined,
RightOutlined,
SearchOutlined,
SettingOutlined,
SyncOutlined,
@ -136,6 +135,9 @@ const shouldPollMeetingCard = (item: MeetingVO) =>
const getUnifiedStatusCode = (progress: MeetingProgress | null | undefined) =>
progress?.unifiedStatus?.statusCode;
const isWaitingUploadProgress = (progress: MeetingProgress | null | undefined) =>
getUnifiedStatusCode(progress) === "WAITING_UPLOAD";
const getEffectiveStatus = (item: MeetingVO, progress: MeetingProgress | null) => {
const unifiedStatusCode = getUnifiedStatusCode(progress);
if (unifiedStatusCode === "WAITING_UPLOAD") {
@ -285,12 +287,17 @@ const MeetingCardItem: React.FC<{
}> = ({ item, config, progress, onOpenMeeting, onRetrySchedule, onDelete, retrying }) => {
const effectiveStatus = getEffectiveStatus(item, progress);
const isProcessing = shouldTrackGenerationProgress(item);
const isWaitingUpload = isWaitingUploadProgress(progress) || effectiveStatus === 8;
const isPaused = effectiveStatus === PAUSED_DISPLAY_STATUS;
const isRealtimeActive = effectiveStatus === REALTIME_ACTIVE_DISPLAY_STATUS;
const isRealtimeIdle = effectiveStatus === REALTIME_IDLE_DISPLAY_STATUS;
const isCrossPlatformRealtime = (isPaused || isRealtimeActive || isRealtimeIdle) && !canControlRealtimeFromCurrentPlatform(item);
const crossPlatformHint = `${getRealtimeSourceLabel(item)} 继续`;
const canRetry = canRetryQueuedMeeting(item, progress);
const ownerName = item.creatorName || "未知";
const processingMessage = isWaitingUpload
? (progress?.message || progress?.unifiedStatus?.message || config.text)
: (progress?.unifiedStatus?.message || progress?.message || "深度分析中...");
const sourceColor = item.meetingSource === "ANDROID" ? "#10b981" : "#3b82f6";
@ -375,26 +382,28 @@ const MeetingCardItem: React.FC<{
fontSize: "12px"
}}
>
<div style={{ display: "flex", alignItems: "center", gap: "6px", color: config.color, fontWeight: 600 }}>
{isProcessing ? <SyncOutlined spin /> : <InfoCircleOutlined />}
<span style={{ fontSize: "12px" }}>
{isProcessing ? (progress?.message || "深度分析中...") : (isCrossPlatformRealtime ? crossPlatformHint : config.text)}
</span>
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: "10px" }}>
<div style={{ display: "flex", alignItems: "center", gap: "6px", minWidth: 0, color: config.color, fontWeight: 600 }}>
{isProcessing ? <SyncOutlined spin /> : <InfoCircleOutlined />}
<span style={{ fontSize: "12px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
{isProcessing ? processingMessage : (isCrossPlatformRealtime ? crossPlatformHint : config.text)}
</span>
</div>
{canRetry && (
<Button
type="link"
size="small"
loading={retrying}
onClick={(event) => {
event.stopPropagation();
onRetrySchedule(item);
}}
style={{ paddingInline: 0, height: 18, lineHeight: "18px", flexShrink: 0, fontSize: "12px", fontWeight: 600 }}
>
</Button>
)}
</div>
{canRetry && (
<Button
type="link"
size="small"
loading={retrying}
onClick={(event) => {
event.stopPropagation();
onRetrySchedule(item);
}}
style={{ paddingInline: 0, height: "auto" }}
>
</Button>
)}
</div>
) : (
<div style={{ display: "flex", flexWrap: "wrap", gap: "12px", color: "#8c8c8c", fontSize: "12px" }}>
@ -402,10 +411,6 @@ const MeetingCardItem: React.FC<{
<CalendarOutlined style={{ fontSize: "13px", opacity: 0.7 }} />
<span>{dayjs(item.meetingTime).format("MM-DD HH:mm")}</span>
</div>
<div style={{ display: "flex", alignItems: "center", gap: "4px" }}>
<UserOutlined style={{ fontSize: "13px", opacity: 0.7 }} />
<span style={{ maxWidth: "80px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{item.creatorName || "未知"}</span>
</div>
</div>
)}
</div>
@ -433,34 +438,30 @@ const MeetingCardItem: React.FC<{
</span>
)) || <span style={{ fontSize: "11px", color: "#bfbfbf" }}></span>}
</div>
<div
style={{
height: "28px",
borderRadius: "8px",
background: "rgba(95, 81, 255, 0.1)",
color: "var(--primary-blue)",
display: "flex",
alignItems: "center",
justifyContent: "center",
transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)",
<div
style={{
height: "28px",
maxWidth: "112px",
padding: "0 10px",
borderRadius: "8px",
background: "rgba(15, 23, 42, 0.04)",
color: "#6b7280",
display: "flex",
alignItems: "center",
justifyContent: "center",
gap: "5px",
overflow: "hidden",
whiteSpace: "nowrap",
minWidth: "28px"
flexShrink: 0,
fontSize: "11px",
fontWeight: 600
}}
className="card-arrow"
title={`所属用户:${ownerName}`}
>
<RightOutlined style={{ fontSize: "10px", flexShrink: 0 }} />
<span className="view-detail-text" style={{
fontSize: "11px",
fontWeight: 700,
maxWidth: 0,
opacity: 0,
transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)",
marginLeft: 0,
overflow: 'hidden',
display: 'inline-block'
}}></span>
<UserOutlined style={{ fontSize: "12px", opacity: 0.75, flexShrink: 0 }} />
<span style={{ overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>
{ownerName}
</span>
</div>
</div>
</Card>
@ -1001,8 +1002,6 @@ const Meetings: React.FC = () => {
<style>{`
.meeting-card-v2:hover { transform: translateY(-3px); box-shadow: 0 12px 24px rgba(95, 81, 255, 0.08) !important; border-color: rgba(95, 81, 255, 0.15) !important; }
.meeting-card-v2:hover .card-arrow { background: var(--primary-blue) !important; color: white !important; padding: 0 12px; }
.meeting-card-v2:hover .view-detail-text { max-width: 60px; opacity: 1; margin-left: 6px; }
.status-bar-active { animation: statusBreathing 2s infinite ease-in-out; }
@keyframes statusBreathing { 0% { opacity: 1; } 50% { opacity: 0.4; } 100% { opacity: 1; } }
.meetings-list-table.ant-table-wrapper,

View File

@ -18,7 +18,6 @@ import dayjs from "dayjs";
import PageHeader from "../../components/shared/PageHeader";
import PageContainer from "@/components/shared/PageContainer";
import {
appendRealtimeTranscripts,
completeRealtimeMeeting,
getMeetingDetail,
getRealtimeMeetingSessionStatus,
@ -28,7 +27,6 @@ import {
type MeetingTranscriptVO,
type MeetingVO,
type RealtimeMeetingSessionStatus,
type RealtimeTranscriptItemDTO,
type RealtimeSocketSessionVO,
} from "../../api/business/meeting";
const { Text, Title } = Typography;
@ -44,6 +42,8 @@ type WsMessage = {
data?: {
text?: string;
is_final?: boolean;
sentence_id?: number;
sentence_key?: string;
start?: number;
end?: number;
speaker_id?: string;
@ -58,6 +58,8 @@ type WsMessage = {
type TranscriptCard = {
id: string;
sentenceKey?: string;
sentenceId?: number;
speakerName: string;
userId?: string | number;
text: string;
@ -66,6 +68,16 @@ type TranscriptCard = {
final: boolean;
};
type NormalizedWsMessage = {
sentenceKey?: string;
text: string;
isFinal: boolean;
sentenceId?: number;
speaker?: WsSpeaker;
startTime?: number;
endTime?: number;
};
type RealtimeMeetingSessionDraft = {
meetingId: number;
meetingTitle: string;
@ -123,12 +135,18 @@ function resolveSpeaker(speaker?: WsSpeaker) {
return { speakerId: "spk_0", speakerName: "Unknown", userId: undefined };
}
if (typeof speaker === "string") {
return { speakerId: speaker, speakerName: speaker, userId: undefined };
const normalized = speaker.trim();
if (!normalized || normalized === "-1") {
return {speakerId: "spk_0", speakerName: "Unknown", userId: undefined};
}
return {speakerId: normalized, speakerName: normalized, userId: undefined};
}
const rawUserId = speaker.user_id === null || speaker.user_id === undefined ? undefined : String(speaker.user_id).trim();
const speakerId = rawUserId && rawUserId !== "-1" ? rawUserId : "spk_0";
return {
speakerId: speaker.user_id ? String(speaker.user_id) : "spk_0",
speakerName: speaker.name || (speaker.user_id ? String(speaker.user_id) : "Unknown"),
userId: speaker.user_id,
speakerId,
speakerName: speaker.name?.trim() || (speakerId !== "spk_0" ? speakerId : "Unknown"),
userId: speakerId === "spk_0" ? undefined : rawUserId,
};
}
@ -159,17 +177,29 @@ function toMs(value?: number) {
return Math.round(value * 1000);
}
function buildTranscriptCardId(sentenceKey?: string, sentenceId?: number) {
if (sentenceKey) {
return sentenceKey;
}
if (sentenceId === undefined || sentenceId === null) {
return `live-${Date.now()}-${Math.random()}`;
}
return `sentence-${sentenceId}`;
}
function buildRealtimeProxyWsUrl(socketSession: RealtimeSocketSessionVO) {
const protocol = window.location.protocol === "https:" ? "wss" : "ws";
return `${protocol}://${window.location.host}${socketSession.path}?sessionToken=${encodeURIComponent(socketSession.sessionToken)}`;
}
function normalizeWsMessage(payload: WsMessage) {
function normalizeWsMessage(payload: WsMessage): NormalizedWsMessage | null {
if (payload.type === "partial" || payload.type === "segment") {
const data = payload.data || {};
return {
text: data.text || "",
isFinal: payload.type === "segment" || !!data.is_final,
sentenceKey: data.sentence_key,
sentenceId: data.sentence_id,
speaker: {
name: data.speaker_name,
user_id: data.user_id ?? data.speaker_id,
@ -212,7 +242,6 @@ export function RealtimeAsrSession() {
const [audioLevel, setAudioLevel] = useState(0);
const [elapsedSeconds, setElapsedSeconds] = useState(0);
const [sessionStatus, setSessionStatus] = useState<RealtimeMeetingSessionStatus | null>(null);
const transcriptRef = useRef<HTMLDivElement | null>(null);
const wsRef = useRef<WebSocket | null>(null);
const audioContextRef = useRef<AudioContext | null>(null);
@ -223,6 +252,7 @@ export function RealtimeAsrSession() {
const completeOnceRef = useRef(false);
const startedAtRef = useRef<number | null>(null);
const sessionStartedRef = useRef(false);
const elapsedOffsetRef = useRef(0);
const finalTranscriptCount = transcripts.length;
const totalTranscriptChars = useMemo(
@ -281,7 +311,7 @@ export function RealtimeAsrSession() {
}
setTranscripts(
(transcriptRes.data.data || []).map((item: MeetingTranscriptVO) => ({
id: String(item.id),
id: `persisted-${item.id}`,
speakerName: item.speakerName || item.speakerId || "发言人",
text: item.content,
startTime: item.startTime,
@ -301,12 +331,12 @@ export function RealtimeAsrSession() {
useEffect(() => {
if (!recording) {
setElapsedSeconds(0);
setElapsedSeconds(elapsedOffsetRef.current);
return;
}
const timer = window.setInterval(() => {
if (startedAtRef.current) {
setElapsedSeconds(Math.floor((Date.now() - startedAtRef.current) / 1000));
setElapsedSeconds(elapsedOffsetRef.current + Math.floor((Date.now() - startedAtRef.current) / 1000));
}
}, 1000);
return () => window.clearInterval(timer);
@ -326,7 +356,7 @@ export function RealtimeAsrSession() {
}
const token = localStorage.getItem("accessToken");
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ is_speaking: false }));
wsRef.current.close();
}
fetch(`/api/biz/meeting/${meetingId}/realtime/pause`, {
method: "POST",
@ -360,6 +390,44 @@ export function RealtimeAsrSession() {
setAudioLevel(0);
};
const closeFrontendSocket = async (sendStop: boolean) => {
const socket = wsRef.current;
if (sendStop && socket?.readyState === WebSocket.OPEN) {
socket.send(JSON.stringify({type: "stop"}));
await new Promise((resolve) => window.setTimeout(resolve, 150));
}
socket?.close();
wsRef.current = null;
sessionStartedRef.current = false;
};
const upsertTranscriptCard = (normalized: NormalizedWsMessage, speaker: ReturnType<typeof resolveSpeaker>) => {
setTranscripts((prev) => {
const next = [...prev];
const cardId = buildTranscriptCardId(normalized.sentenceKey, normalized.sentenceId);
const nextCard: TranscriptCard = {
id: cardId,
sentenceKey: normalized.sentenceKey,
sentenceId: normalized.sentenceId,
speakerName: speaker.speakerName,
userId: speaker.userId,
text: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
final: true,
};
if (normalized.sentenceKey || (normalized.sentenceId !== undefined && normalized.sentenceId !== null)) {
const index = next.findIndex((item) => item.id === cardId);
if (index >= 0) {
next[index] = {...next[index], ...nextCard};
return next;
}
}
next.push(nextCard);
return next;
});
};
const handleFatalRealtimeError = async (errorMessage: string) => {
setConnecting(false);
setRecording(false);
@ -369,6 +437,8 @@ export function RealtimeAsrSession() {
wsRef.current = null;
await shutdownAudioPipeline();
startedAtRef.current = null;
elapsedOffsetRef.current = 0;
setElapsedSeconds(0);
message.error(errorMessage);
};
@ -419,26 +489,6 @@ export function RealtimeAsrSession() {
processor.connect(audioContext.destination);
};
const saveFinalTranscript = async (normalized: {
text: string;
speaker?: WsSpeaker;
startTime?: number;
endTime?: number;
}) => {
if (!normalized.text || !meetingId) {
return;
}
const speaker = resolveSpeaker(normalized.speaker);
const item: RealtimeTranscriptItemDTO = {
speakerId: speaker.speakerId,
speakerName: speaker.speakerName,
content: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
};
await appendRealtimeTranscripts(meetingId, [item]);
};
const handlePause = async () => {
if (!meetingId || pausing || finishing || (!recording && !connecting)) {
return;
@ -451,20 +501,19 @@ export function RealtimeAsrSession() {
setPausing(true);
setStatusText("暂停识别中...");
try {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ is_speaking: false }));
if (recording && startedAtRef.current) {
elapsedOffsetRef.current += Math.floor((Date.now() - startedAtRef.current) / 1000);
}
wsRef.current?.close();
wsRef.current = null;
sessionStartedRef.current = false;
await shutdownAudioPipeline();
const pauseRes = await pauseRealtimeMeeting(meetingId);
await closeFrontendSocket(false);
await shutdownAudioPipeline();
setSessionStatus(pauseRes.data.data);
setRecording(false);
setConnecting(false);
startedAtRef.current = null;
setStatusText(pauseRes.data.data?.hasTranscript ? "已暂停,可继续识别" : "已暂停,当前还没有转录内容");
message.success("实时识别已暂停");
setElapsedSeconds(elapsedOffsetRef.current);
setStatusText("已暂停,可继续会议并等待说话人修正");
message.success("实时会议已暂停");
} catch (error) {
setStatusText("暂停失败");
message.error(error instanceof Error ? error.message : "暂停实时识别失败");
@ -547,21 +596,9 @@ export function RealtimeAsrSession() {
const speaker = resolveSpeaker(normalized.speaker);
if (normalized.isFinal) {
setTranscripts((prev) => [
...prev,
{
id: `${Date.now()}-${Math.random()}`,
speakerName: speaker.speakerName,
userId: speaker.userId,
text: normalized.text,
startTime: normalized.startTime,
endTime: normalized.endTime,
final: true,
},
]);
upsertTranscriptCard(normalized, speaker);
setStreamingText("");
setStreamingSpeaker("Unknown");
void saveFinalTranscript(normalized);
} else {
setStreamingText(normalized.text);
setStreamingSpeaker(speaker.speakerName);
@ -602,12 +639,7 @@ export function RealtimeAsrSession() {
setFinishing(true);
setStatusText("结束会议中...");
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ is_speaking: false }));
}
wsRef.current?.close();
wsRef.current = null;
sessionStartedRef.current = false;
await closeFrontendSocket(true);
await shutdownAudioPipeline();

View File

@ -7,9 +7,10 @@ interface PageHeaderProps {
title: string;
extra?: ReactNode;
back?: boolean;
onBack?: () => void;
}
export default function PageHeader({ title, extra, back = false }: PageHeaderProps) {
export default function PageHeader({ title, extra, back = false, onBack }: PageHeaderProps) {
const navigate = useNavigate();
return (
@ -20,7 +21,7 @@ export default function PageHeader({ title, extra, back = false }: PageHeaderPro
type="text"
shape="circle"
icon={<ArrowLeftOutlined />}
onClick={() => navigate(-1)}
onClick={() => (onBack ? onBack() : navigate(-1))}
className="page-header__back"
/>
) : null}

View File

@ -1,4 +1,4 @@
import {useMemo, useRef, useState} from "react";
import {useEffect, useMemo, useRef, useState} from "react";
import {
AudioOutlined,
CalendarOutlined,
@ -195,6 +195,7 @@ export default function MeetingPreviewView({
}, [meeting?.participants, transcripts]);
const tags = useMemo(() => splitDisplayItems(meeting?.tags), [meeting?.tags]);
const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]);
const aiCatalogEnabled = meeting?.aiCatalogEnabled !== false;
const statusMeta = STATUS_META[meeting?.status || 0] || {
label: TEXT.statusPending,
className: "is-warning",
@ -267,6 +268,12 @@ export default function MeetingPreviewView({
});
}, [analysis.chapters, meetingChapters, transcripts]);
useEffect(() => {
if (!aiCatalogEnabled && pageTab === "catalog") {
setPageTab("summary");
}
}, [aiCatalogEnabled, pageTab]);
const handleTranscriptSeek = (item: MeetingTranscriptVO) => {
if (!audioRef.current) return;
audioRef.current.currentTime = Math.max(0, (item.startTime || 0) / 1000);
@ -459,7 +466,7 @@ export default function MeetingPreviewView({
onChange={(key) => setPageTab(key as PreviewPageTab)}
items={[
{key: "summary", label: TEXT.pageSummary},
{key: "catalog", label: TEXT.pageCatalog},
...(aiCatalogEnabled ? [{key: "catalog", label: TEXT.pageCatalog}] : []),
{key: "transcript", label: TEXT.pageTranscript},
]}
/>
@ -489,7 +496,7 @@ export default function MeetingPreviewView({
</>
) : null}
{pageTab === "catalog" ? (
{aiCatalogEnabled && pageTab === "catalog" ? (
<div className="meeting-preview-catalog-list">
{catalogChapterLinks.length ? (
catalogChapterLinks.map((chapter, index) => (

View File

@ -1,18 +1,21 @@
import { App, Button, Card, Input, Space, Typography } from "antd";
import { useEffect, useMemo, useState } from "react";
import { useParams, useSearchParams } from "react-router-dom";
import { useNavigate, useParams, useSearchParams } from "react-router-dom";
import { getMeetingPreviewAccess, getPublicMeetingPreview } from "@/api/meeting";
import LoadingScreen from "@/components/LoadingScreen";
import PageHeader from "@/components/PageHeader";
import MeetingPreviewView from "@/components/preview/MeetingPreviewView";
import usePageTitle from "@/hooks/usePageTitle";
import type { MeetingChapterVO, MeetingTranscriptVO, MeetingVO } from "@/types";
import { hasAccessToken } from "@/utils/auth";
import { buildMeetingPreviewUrl } from "@/utils/meeting";
const { Paragraph, Title } = Typography;
export default function MeetingPreviewPage() {
const { message } = App.useApp();
const navigate = useNavigate();
const { id } = useParams<{ id: string }>();
const [searchParams] = useSearchParams();
const [loading, setLoading] = useState(true);
@ -27,6 +30,10 @@ export default function MeetingPreviewPage() {
const presetAccessPassword = useMemo(() => (searchParams.get("accessPassword") || "").trim(), [searchParams]);
usePageTitle(meeting?.title || "会议预览");
const handleBack = () => {
navigate(hasAccessToken() ? "/meetings" : "/login", { replace: true });
};
const loadPreview = async (password?: string) => {
const previewResp = await getPublicMeetingPreview(meetingId, password);
setMeeting(previewResp.data.data.meeting);
@ -69,7 +76,7 @@ export default function MeetingPreviewPage() {
};
void run();
}, [meetingId, presetAccessPassword]);
}, [meetingId, presetAccessPassword, message]);
const handleSubmitPassword = async () => {
if (!accessPassword.trim()) {
@ -93,21 +100,24 @@ export default function MeetingPreviewPage() {
if (passwordRequired && !passwordVerified) {
return (
<div className="preview-page">
<Card className="surface-card preview-password-card">
<Title level={3}></Title>
<Paragraph type="secondary">访访</Paragraph>
<Space direction="vertical" size={16} style={{ width: "100%" }}>
<Input.Password
value={accessPassword}
placeholder="请输入访问密码"
onChange={(event) => setAccessPassword(event.target.value)}
onPressEnter={() => void handleSubmitPassword()}
/>
<Button type="primary" block loading={loading} onClick={() => void handleSubmitPassword()}>
</Button>
</Space>
</Card>
<div className="preview-page__inner">
<PageHeader title="分享预览" back onBack={handleBack} />
<Card className="surface-card preview-password-card">
<Title level={3}></Title>
<Paragraph type="secondary">访访</Paragraph>
<Space direction="vertical" size={16} style={{ width: "100%" }}>
<Input.Password
value={accessPassword}
placeholder="请输入访问密码"
onChange={(event) => setAccessPassword(event.target.value)}
onPressEnter={() => void handleSubmitPassword()}
/>
<Button type="primary" block loading={loading} onClick={() => void handleSubmitPassword()}>
</Button>
</Space>
</Card>
</div>
</div>
);
}
@ -119,9 +129,7 @@ export default function MeetingPreviewPage() {
return (
<div className="preview-page">
<div className="preview-page__inner">
<div className="preview-page__header">
<span className="login-page__badge">iMeeting </span>
</div>
<PageHeader title="分享预览" back onBack={handleBack} />
<MeetingPreviewView
meeting={meeting}
transcripts={transcripts}

View File

@ -47,6 +47,7 @@ export interface MeetingVO {
summaryDetailLevel?: "DETAILED" | "STANDARD" | "BRIEF";
summaryModelId?: number;
promptId?: number;
aiCatalogEnabled?: boolean;
audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED";
audioSaveMessage?: string;
accessPassword?: string | null;