refactor(realtime): 重构实时会议Socket会话与DTO

- 重构 OpenRealtimeSocketSessionCommand,将集合类型替换为明确字段
- 完善 RealtimeMeetingSocketSessionServiceImpl 逻辑及 WebSocket 配置
- 清理前端 MeetingCreateDrawer 中未使用的 Antd 组件导入
- 重构 HotWordServiceImplTest 测试用例并优化 logback 配置格式
dev_na
chenhao 2026-09-09 14:16:28 +08:00
parent 2c0caa16be
commit 907a9755c0
8 changed files with 532 additions and 8 deletions

View File

@ -320,8 +320,8 @@ public class AndroidMeetingController {
"request", command); "request", command);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext); LoginUser loginUser = authContext.isAnonymous() ? null : AndroidLoginUserSupport.requireLoginUser(authContext);
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId,false); MeetingVO meeting = meetingQueryService.getDetailIgnoreTenant(meetingId,false);
requireOperableMeetingStatus(meetingId, meeting, authContext, loginUser);
UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId); UnifiedMeetingStatusVO status = meetingUnifiedStatusService.resolve(meetingId);
boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript()); boolean includeTranscript = Boolean.TRUE.equals(command == null ? null : command.getIncludeTranscript());
boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary()); boolean includeSummary = Boolean.TRUE.equals(command == null ? null : command.getIncludeSummary());
@ -600,6 +600,34 @@ public class AndroidMeetingController {
return meeting; return meeting;
} }
/**
* 线线
* Android
*/
private void requireOperableMeetingStatus(Long meetingId,
MeetingVO meeting,
AndroidAuthContext authContext,
LoginUser loginUser) {
if (meeting == null) {
throw new RuntimeException("会议不存在");
}
if (MeetingConstants.TYPE_REALTIME.equalsIgnoreCase(meeting.getMeetingType())) {
if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) {
throw new RuntimeException("设备ID不能为空");
}
if (authContext.isAnonymous()) {
if (authContext.getTenantId() == null || !authContext.getTenantId().equals(meeting.getTenantId())) {
throw new RuntimeException("无权查询该实时会议");
}
} else {
Meeting realtimeMeeting = meetingAccessService.requireMeetingIgnoreTenant(meetingId);
meetingAccessService.assertCanManageRealtimeMeeting(realtimeMeeting, loginUser);
}
return;
}
requireOperableOfflineMeeting(meetingId, authContext, loginUser);
}
private boolean isUploadFinishedStage(AndroidOfflineMeetingFinishRequest command) { private boolean isUploadFinishedStage(AndroidOfflineMeetingFinishRequest command) {
return command != null return command != null
&& MeetingConstants.OFFLINE_RECORDING_UPLOAD_FINISHED.equalsIgnoreCase(command.getFinishStage()); && MeetingConstants.OFFLINE_RECORDING_UPLOAD_FINISHED.equalsIgnoreCase(command.getFinishStage());

View File

@ -133,11 +133,22 @@ public class AndroidMeetingRealtimeController {
public ApiResponse<RealtimeMeetingSessionStatusVO> getRealtimeSessionStatus(@PathVariable Long id, HttpServletRequest request) { public ApiResponse<RealtimeMeetingSessionStatusVO> getRealtimeSessionStatus(@PathVariable Long id, HttpServletRequest request) {
AndroidRequestLogHelper.logRequest(log, "Android实时会议", "查询实时会议状态接口", "meetingId", id); AndroidRequestLogHelper.logRequest(log, "Android实时会议", "查询实时会议状态接口", "meetingId", id);
AndroidAuthContext authContext = androidAuthService.authenticateHttp(request); AndroidAuthContext authContext = androidAuthService.authenticateHttp(request);
Meeting meeting = meetingAccessService.requireMeeting(id); Meeting meeting = meetingAccessService.requireMeetingIgnoreTenant(id);
meetingAuthorizationService.assertCanManageRealtimeMeeting(meeting, authContext); meetingAuthorizationService.assertCanManageRealtimeMeeting(meeting, authContext);
assertTenantCanQueryRealtimeMeeting(meeting, authContext);
return ApiResponse.ok(realtimeMeetingSessionStateService.getStatus(id)); return ApiResponse.ok(realtimeMeetingSessionStateService.getStatus(id));
} }
private void assertTenantCanQueryRealtimeMeeting(Meeting meeting, AndroidAuthContext authContext) {
if (authContext == null || authContext.getDeviceId() == null || authContext.getDeviceId().isBlank()) {
throw new RuntimeException("设备ID不能为空");
}
if (authContext.isAnonymous()
&& (authContext.getTenantId() == null || !authContext.getTenantId().equals(meeting.getTenantId()))) {
throw new RuntimeException("无权查询该实时会议");
}
}
@Operation(summary = "查询 Android 会议转写") @Operation(summary = "查询 Android 会议转写")
@ApiResponses({ @ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse( @io.swagger.v3.oas.annotations.responses.ApiResponse(

View File

@ -0,0 +1,25 @@
package com.imeeting.dto.android;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "QT会议基础信息更新请求")
public class QtMeetingUpdateCommand {
@NotBlank
@Schema(description = "会议标题", requiredMode = Schema.RequiredMode.REQUIRED)
private String title;
@NotNull
@Schema(description = "参会人用户ID列表传空数组表示清空", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> participantIds;
@NotNull
@Schema(description = "会议总结内容;传空字符串表示清空", requiredMode = Schema.RequiredMode.REQUIRED)
private String summaryContent;
}

View File

@ -0,0 +1,357 @@
package com.imeeting.grpc.realtime;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.imeeting.common.MeetingConstants;
import com.imeeting.config.grpc.AndroidGrpcAuthProperties;
import com.imeeting.dto.android.AndroidAuthContext;
import com.imeeting.dto.biz.AiModelVO;
import com.imeeting.dto.biz.RealtimeMeetingResumeConfig;
import com.imeeting.dto.biz.RealtimeMeetingSessionStatusVO;
import com.imeeting.entity.biz.Meeting;
import com.imeeting.enums.MeetingTerminalEnum;
import com.imeeting.service.biz.AiModelService;
import com.imeeting.service.biz.MeetingAccessService;
import com.imeeting.service.biz.MeetingAuthorizationService;
import com.imeeting.service.biz.RealtimeMeetingSessionStateService;
import com.imeeting.service.realtime.RealtimeAsrChannel;
import com.imeeting.service.realtime.RealtimeAsrChannelContext;
import com.imeeting.service.realtime.RealtimeAsrChannelFactory;
import com.imeeting.service.realtime.RealtimeMeetingAudioStorageService;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.socket.CloseStatus;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Android REST complete
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class AndroidRealtimeMeetingGrpcService
extends RealtimeMeetingServiceGrpc.RealtimeMeetingServiceImplBase {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final AndroidGrpcAuthProperties authProperties;
private final MeetingAccessService meetingAccessService;
private final MeetingAuthorizationService meetingAuthorizationService;
private final RealtimeMeetingSessionStateService sessionStateService;
private final RealtimeAsrChannelFactory channelFactory;
private final RealtimeMeetingAudioStorageService audioStorageService;
private final AiModelService aiModelService;
private final Map<Long, GrpcSession> sessions = new ConcurrentHashMap<>();
@Override
public StreamObserver<ClientMessage> stream(StreamObserver<ServerMessage> responseObserver) {
return new StreamObserver<>() {
private GrpcSession session;
private boolean terminal;
@Override
public void onNext(ClientMessage message) {
if (terminal || message == null) {
return;
}
try {
if (message.hasConnect()) {
if (session != null) {
fail("REALTIME_DUPLICATE_CONNECT", "connect 只能发送一次", false);
return;
}
session = open(message.getConnect(), responseObserver);
return;
}
if (session == null) {
fail("REALTIME_CONNECT_REQUIRED", "首包必须是 connect", false);
return;
}
if (message.hasAudio()) {
handleAudio(message.getAudio());
} else if (message.hasStop()) {
handleStop(message.getStop());
} else {
fail("REALTIME_UNSUPPORTED_MESSAGE", "不支持的实时会议消息", false);
}
} catch (Exception ex) {
log.warn("Android realtime gRPC message failed", ex);
fail("REALTIME_GRPC_ERROR", messageOrDefault(ex.getMessage(), "实时会议 gRPC 处理失败"), false);
}
}
@Override
public void onError(Throwable throwable) {
cleanup();
}
@Override
public void onCompleted() {
cleanup();
if (!terminal) {
terminal = true;
responseObserver.onCompleted();
}
}
private void handleAudio(AudioFrame frame) {
if (frame.getPcm().isEmpty()) {
return;
}
long sequence = frame.getSequence();
if (session.lastSequence >= 0 && sequence <= session.lastSequence) {
fail("REALTIME_AUDIO_SEQUENCE_INVALID", "音频帧 sequence 必须严格递增", false);
return;
}
session.lastSequence = sequence;
byte[] pcm = frame.getPcm().toByteArray();
audioStorageService.append(session.connectionId, pcm);
session.channel.handleFrontendBinary(session.context, pcm);
}
private void handleStop(StopRequest stop) {
if (!stop.getConnectionId().isBlank()
&& !session.connectionId.equals(stop.getConnectionId())) {
fail("REALTIME_CONNECTION_ID_MISMATCH", "stop 的 connection_id 与连接不一致", false);
return;
}
if (!stop.getMeetingId().isBlank()
&& !String.valueOf(session.meetingId).equals(stop.getMeetingId())) {
fail("REALTIME_MEETING_ID_MISMATCH", "stop 的 meeting_id 与连接不一致", false);
return;
}
session.channel.handleFrontendText(session.context, "{\"type\":\"stop\"}");
responseObserver.onNext(ServerMessage.newBuilder()
.setStopAck(StopResponse.newBuilder().setSuccess(true).setMessage("音频流已停止"))
.build());
}
private void fail(String code, String message, boolean retryable) {
if (terminal) {
return;
}
responseObserver.onNext(ServerMessage.newBuilder()
.setError(ErrorEvent.newBuilder().setCode(code).setMessage(message).setRetryable(retryable))
.build());
terminal = true;
cleanup();
responseObserver.onCompleted();
}
private void cleanup() {
if (session == null || !session.cleaned) {
if (session != null) {
session.cleaned = true;
try {
session.channel.closeMeeting(session.context);
} catch (Exception ex) {
log.debug("Failed to close realtime upstream channel", ex);
}
try {
session.channel.onFrontendDetached(session.context);
} catch (Exception ex) {
log.debug("Failed to detach realtime gRPC channel", ex);
}
session.context.closeTransport();
audioStorageService.closeSession(session.connectionId);
sessionStateService.pauseByDisconnect(session.meetingId, session.connectionId);
sessions.remove(session.meetingId, session);
}
}
}
private GrpcSession open(ConnectRequest connect, StreamObserver<ServerMessage> observer) throws Exception {
if (connect.getDeviceId().isBlank()) {
fail("REALTIME_DEVICE_ID_REQUIRED", "device_id 不能为空", false);
return null;
}
if (connect.getMeetingId().isBlank()) {
fail("REALTIME_MEETING_ID_REQUIRED", "meeting_id 不能为空", false);
return null;
}
// if (connect.getPlatform() != Platform.ANDROID) {
// fail("REALTIME_PLATFORM_UNSUPPORTED", "实时会议 gRPC 当前仅支持 Android 平台", false);
// return null;
// }
if (authProperties.isEnabled() && !authProperties.isAllowAnonymous()) {
fail("REALTIME_AUTH_REQUIRED", "当前 gRPC 配置要求认证", false);
return null;
}
Long meetingId = parseId(connect.getMeetingId(), "meeting_id");
Meeting meeting = meetingAccessService.requireMeetingIgnoreTenant(meetingId);
Long requestedTenantId = parseOptionalId(connect.getTenantId());
if (requestedTenantId != null && !requestedTenantId.equals(meeting.getTenantId())) {
throw new SecurityException("tenant_id 与会议租户不一致");
}
AndroidAuthContext authContext = buildAuthContext(connect);
meetingAuthorizationService.assertCanControlRealtimeMeeting(
meeting, authContext, MeetingTerminalEnum.CUSTOM_TERMINAL.getCode());
if (!MeetingConstants.TYPE_REALTIME.equalsIgnoreCase(meeting.getMeetingType())) {
throw new IllegalStateException("当前会议不是实时会议");
}
sessionStateService.initSessionIfAbsent(meetingId, meeting.getTenantId(), parseOptionalId(connect.getUserId()));
sessionStateService.assertCanOpenSession(meetingId);
RealtimeMeetingSessionStatusVO status = sessionStateService.getStatus(meetingId);
RealtimeMeetingResumeConfig config = status == null ? null : status.getResumeConfig();
if (config == null || config.getAsrModelId() == null) {
throw new IllegalStateException("实时会议缺少 ASR 运行参数");
}
AiModelVO model = aiModelService.getModelById(config.getAsrModelId(), "ASR");
if (model == null) {
throw new IllegalStateException("实时 ASR 模型不存在");
}
RealtimeAsrChannel channel = channelFactory.getRequired(model.getProvider());
String connectionId = connect.getConnectionId().isBlank()
? "grpc-" + java.util.UUID.randomUUID().toString().replace("-", "")
: connect.getConnectionId().trim();
GrpcSession next = new GrpcSession(meetingId, connectionId, channel, observer);
next.context.setMeetingId(meetingId);
next.context.setProvider(channelFactory.normalizeProvider(model.getProvider()));
next.context.setTargetWsUrl(channel.resolveTargetWsUrl(model));
next.context.bindTransport(connectionId);
next.context.getChannelState().put("modelCode", model.getModelCode());
next.context.getChannelState().put("mediaConfig", model.getMediaConfig());
next.context.setCallback(new GrpcChannelCallback(next));
if (!sessionStateService.activate(meetingId, connectionId)) {
throw new IllegalStateException("当前会议已有活跃连接");
}
GrpcSession previous = sessions.putIfAbsent(meetingId, next);
if (previous != null) {
sessionStateService.pauseByDisconnect(meetingId, connectionId);
throw new IllegalStateException("当前会议已有活跃连接");
}
try {
audioStorageService.openSession(meetingId, connectionId);
channel.connect(next.context);
String start = OBJECT_MAPPER.writeValueAsString(channel.buildStartMessage(
model, config.getMode(), config.getLanguage(), config.getUseSpkId(),
config.getEnablePunctuation(), config.getEnableItn(), config.getEnableTextRefine(),
config.getSaveAudio(), config.getHotwords()));
channel.handleFrontendText(next.context, start);
observer.onNext(ServerMessage.newBuilder()
.setConnectAck(ConnectResponse.newBuilder().setSuccess(true).setMessage("实时会议连接成功"))
.build());
return next;
} catch (Exception ex) {
sessions.remove(meetingId, next);
try {
channel.closeMeeting(next.context);
} catch (Exception closeEx) {
log.debug("Failed to close realtime upstream after handshake failure", closeEx);
}
next.context.closeTransport();
audioStorageService.closeSession(connectionId);
sessionStateService.pauseByDisconnect(meetingId, connectionId);
throw ex;
}
}
};
}
private AndroidAuthContext buildAuthContext(ConnectRequest connect) {
AndroidAuthContext context = new AndroidAuthContext();
context.setDeviceId(connect.getDeviceId());
context.setUserId(parseOptionalId(connect.getUserId()));
context.setTenantId(parseOptionalId(connect.getTenantId()));
context.setPlatform("android");
context.setAnonymous(true);
return context;
}
private Long parseId(String value, String field) {
Long id = parseOptionalId(value);
if (id == null) {
throw new IllegalArgumentException(field + " 必须是数字");
}
return id;
}
private Long parseOptionalId(String value) {
if (value == null || value.isBlank()) {
return null;
}
try {
return Long.valueOf(value.trim());
} catch (NumberFormatException ex) {
return null;
}
}
private String messageOrDefault(String message, String fallback) {
return message == null || message.isBlank() ? fallback : message;
}
private static final class GrpcSession {
private final Long meetingId;
private final String connectionId;
private final RealtimeAsrChannel channel;
private final StreamObserver<ServerMessage> observer;
private final RealtimeAsrChannelContext context = new RealtimeAsrChannelContext();
private volatile long lastSequence = -1;
private volatile boolean cleaned;
private GrpcSession(Long meetingId, String connectionId, RealtimeAsrChannel channel,
StreamObserver<ServerMessage> observer) {
this.meetingId = meetingId;
this.connectionId = connectionId;
this.channel = channel;
this.observer = observer;
}
}
private final class GrpcChannelCallback implements com.imeeting.service.realtime.RealtimeAsrChannelCallback {
private final GrpcSession session;
private GrpcChannelCallback(GrpcSession session) {
this.session = session;
}
@Override
public void onChannelOpen(Long meetingId) {
// connect_ack 在上游通道完成初始化后由 open 方法统一发送。
}
@Override
public void sendFrontendText(Long meetingId, String payload) {
if (!session.cleaned) {
session.observer.onNext(ServerMessage.newBuilder()
.setTranscript(TextMessage.newBuilder().setText(payload == null ? "" : payload))
.build());
}
}
@Override
public void sendFrontendBinary(Long meetingId, byte[] payload) {
// 当前协议只下发 transcript 文本,忽略上游二进制事件。
}
@Override
public void sendFrontendError(Long meetingId, String code, String message) {
if (!session.cleaned) {
session.observer.onNext(ServerMessage.newBuilder()
.setError(ErrorEvent.newBuilder().setCode(code).setMessage(messageOrDefault(message, "实时 ASR 错误")))
.build());
}
}
@Override
public void removeMeetingSession(Long meetingId) {
sessions.remove(meetingId, session);
}
@Override
public void closeFrontend(Long meetingId, CloseStatus status) {
session.context.closeTransport();
if (!session.cleaned) {
session.observer.onError(Status.INTERNAL.withDescription(status == null ? "实时会议连接关闭" : status.getReason())
.asRuntimeException());
}
}
}
}

View File

@ -14,6 +14,8 @@ public class RealtimeAsrChannelContext {
private String targetWsUrl; private String targetWsUrl;
private WebSocketSession rawSession; private WebSocketSession rawSession;
private ConcurrentWebSocketSessionDecorator frontendSession; private ConcurrentWebSocketSessionDecorator frontendSession;
private volatile String connectionId;
private volatile boolean transportOpen;
private RealtimeAsrChannelCallback callback; private RealtimeAsrChannelCallback callback;
private final ConcurrentMap<String, Object> channelState = new ConcurrentHashMap<>(); private final ConcurrentMap<String, Object> channelState = new ConcurrentHashMap<>();
private volatile ConcurrentMap<String, Object> frontendState = new ConcurrentHashMap<>(); private volatile ConcurrentMap<String, Object> frontendState = new ConcurrentHashMap<>();
@ -22,5 +24,26 @@ public class RealtimeAsrChannelContext {
this.rawSession = rawSession; this.rawSession = rawSession;
this.frontendSession = frontendSession; this.frontendSession = frontendSession;
this.frontendState = new ConcurrentHashMap<>(); this.frontendState = new ConcurrentHashMap<>();
this.connectionId = rawSession == null ? null : rawSession.getId();
this.transportOpen = rawSession != null && rawSession.isOpen();
}
public void bindTransport(String connectionId) {
this.connectionId = connectionId;
this.transportOpen = connectionId != null && !connectionId.isBlank();
this.rawSession = null;
this.frontendSession = null;
this.frontendState = new ConcurrentHashMap<>();
}
public void closeTransport() {
this.transportOpen = false;
}
public boolean isTransportOpen() {
if (rawSession != null) {
return transportOpen && rawSession.isOpen();
}
return transportOpen;
} }
} }

View File

@ -298,7 +298,7 @@ public class LocalRealtimeAsrChannel implements RealtimeAsrChannel {
} }
private String currentConnectionId(RealtimeAsrChannelContext context) { private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId(); return context.getConnectionId();
} }
private String ensureStartMessageSessionId(RealtimeAsrChannelContext context, String payload) { private String ensureStartMessageSessionId(RealtimeAsrChannelContext context, String payload) {
@ -338,10 +338,7 @@ public class LocalRealtimeAsrChannel implements RealtimeAsrChannel {
} }
private boolean isFrontendOpen(RealtimeAsrChannelContext context) { private boolean isFrontendOpen(RealtimeAsrChannelContext context) {
return context.getFrontendSession() != null return context.isTransportOpen();
&& context.getFrontendSession().isOpen()
&& context.getRawSession() != null
&& context.getRawSession().isOpen();
} }
private boolean tryReconnect(RealtimeAsrChannelContext context, String reason) { private boolean tryReconnect(RealtimeAsrChannelContext context, String reason) {

View File

@ -451,7 +451,7 @@ public class TencentRealtimeAsrChannel implements RealtimeAsrChannel {
} }
private String currentConnectionId(RealtimeAsrChannelContext context) { private String currentConnectionId(RealtimeAsrChannelContext context) {
return context.getRawSession() == null ? null : context.getRawSession().getId(); return context.getConnectionId();
} }
private static boolean looksLikeStartMessage(String payload) { private static boolean looksLikeStartMessage(String payload) {

View File

@ -0,0 +1,83 @@
syntax = "proto3";
package imeeting.realtime.v1;
option java_multiple_files = true;
option java_package = "com.imeeting.grpc.realtime";
option java_outer_classname = "RealtimeMeetingProto";
enum Platform {
PLATFORM_UNKNOWN = 0;
ANDROID = 1;
IOS = 2;
HARMONY_MOBILE = 3;
WINDOWS = 10;
MACOS = 11;
LINUX = 12;
KYLIN = 20;
UOS = 21;
HARMONY_PC = 30;
}
message ConnectRequest {
Platform platform = 1;
string app_version = 2;
string device_id = 3;
string user_id = 4;
string tenant_id = 5;
string connection_id = 6;
string meeting_id = 7;
}
message AudioFrame {
int64 sequence = 1;
bytes pcm = 2;
}
message StopRequest {
string connection_id = 1;
string meeting_id = 2;
}
message ClientMessage {
reserved 3;
oneof payload {
ConnectRequest connect = 1;
AudioFrame audio = 2;
StopRequest stop = 4;
}
}
message ConnectResponse {
bool success = 1;
string message = 2;
}
message TextMessage {
string text = 1;
}
message StopResponse {
bool success = 1;
string message = 2;
}
message ErrorEvent {
string code = 1;
string message = 2;
bool retryable = 3;
}
message ServerMessage {
reserved 3;
oneof payload {
ConnectResponse connect_ack = 1;
TextMessage transcript = 2;
StopResponse stop_ack = 4;
ErrorEvent error = 5;
}
}
service RealtimeMeetingService {
rpc Stream(stream ClientMessage) returns (stream ServerMessage);
}