using System; using System.Collections.Generic; using System.Linq; using Vinno.FIS.TRTCClient.Common.Enum; using Vinno.FIS.TRTCClient.Common.Models; using Vinno.IUS.Common.Log; using Vinno.vCloud.Common.FIS.Helper; using Vinno.vCloud.FIS.CrossPlatform.Common; using Vinno.vCloud.FIS.CrossPlatform.Common.Consultation; using Vinno.vCloud.FIS.CrossPlatform.Common.Consultation.Interface; using Vinno.vCloud.FIS.CrossPlatform.Common.Enum; using Vinno.vCloud.FIS.CrossPlatform.Common.LiveVideo; using Vinno.vCloud.Protocol.Infrastructures; using WingInterfaceLibrary.Interface; using WingInterfaceLibrary.LiveConsultation; using WingInterfaceLibrary.Request; namespace Vinno.vCloud.Common.FIS.Consultation { internal class ConsultationLiveVideoProviderV2 : IDisposable { private readonly object _lock = new object(); private readonly object _startConsultationLocker = new object();//Felix之所以加这个lock,会有Start没跑完就Stop的情况。 private readonly List _videoProviders; private IRtcRoom _rtcRoom; private bool _isConsulting; private bool _isOnlyRtmpPushing; private LiveProtocol _liveProtocol; private string _accountId; private ConsultationInfo _consultationInfo; private int _imagePreviewErrorLogCount; private ILiveConsultationService _liveConsultationService; private IOrganizationService _organizationService; public event EventHandler OnTRTCRoomEnterError; /// /// 本地的摄像头数据 /// public event EventHandler ConsultationLocalVideoFrameArrived; /// /// 其余所有用户的摄像头数据 /// public event EventHandler ConsultationRemoteVideoFrameArrived; /// /// 超声机摄像头画面 /// public event EventHandler DeviceCameraFrameArrived; public event EventHandler RemoteUserLeaveRoomArrived; public event EventHandler CloseConsultation; public event EventHandler TryToReconnect; public ConsultationLiveVideoProviderV2(ILiveConsultationService liveConsultationService, IOrganizationService organizationService) { _liveConsultationService = liveConsultationService; _organizationService = organizationService; _videoProviders = new List(); ScanCameraCacheHelper.Instance.ScanCameraVideoFrameDataArrived += OnTerminalCameraFrameArrived; } public void ChangeConsultationCodeForConsultationConnectionKeeper(string consultationCode) { if (string.IsNullOrEmpty(consultationCode)) { return; } var existVideoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == _accountId); if (existVideoProvider != null) { existVideoProvider.ChangeConsultationCode(consultationCode); } } /// /// 开始会诊 /// /// public void StartConsultationVideo(ConsultationInfo consultationInfo, string accountId, string accountName, string accountToken, string videoDeviceId, string micDeviceId, string speakerDeviceId, bool isOnlyRtmpPushing) { if (_isConsulting) { return; } lock (_startConsultationLocker) { _isConsulting = true; _isOnlyRtmpPushing = isOnlyRtmpPushing; _liveProtocol = consultationInfo.LiveProtocol; _accountId = accountId; _consultationInfo = consultationInfo; var terminalInfo = consultationInfo.TerminalInfo; var consultationMembers = consultationInfo.ConsultationMemberInfos; if (consultationInfo.LiveTalkingMode == Protocol.Messages.Client.LiveTalking.LiveTalkingMode.Speech) { videoDeviceId = string.Empty; } if (_liveProtocol == LiveProtocol.RTC) { if (consultationInfo.State == LiveStates.OK) { foreach (var member in consultationMembers) { if (member.Id == _accountId) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == member.Id); if (videoProvider == null) { videoProvider = new RtcVideoProviderV2(member); Logger.WriteLineInfo($"Start To Create RTC Video Provider, Name:{member.Name},Id:{member.Id}"); _videoProviders.Add(videoProvider); } else { Logger.WriteLineInfo($"RTC Video Provider Already exist, Name:{member.Name},Id:{member.Id}"); videoProvider.ConsultationMemberInfo = member; } var userSign = member.UserSign; var integerRoomId = consultationInfo.IntegerRoomId; var appId = consultationInfo.AppId; var mode = consultationInfo.LiveTalkingMode; var roomInfo = new TRTCRoomInfo { AppId = (uint)appId, UserSig = userSign, UserId = accountId, RoomId = (uint)integerRoomId, VideoDeviceId = videoDeviceId, MicDeviceId = micDeviceId, SpeakerDeviceId = speakerDeviceId, TerminalIsPushing = _consultationInfo.TerminalInfo?.State == LiveStates.OK, TerminalId = _consultationInfo.TerminalInfo?.Id, CameraId = _consultationInfo.TerminalInfo?.CameraId, IsMultiChannel = _consultationInfo.TerminalInfo == null ? false : _consultationInfo.TerminalInfo.IsMultiChannels, TerminalRoomId = (uint)_consultationInfo.TerminalInfo?.TerminalIntegerRoomId, IsLiveMode = false, IsOldServerMode = false, Category = EnumLiveChannelCategory.Main, }; if (LiveVideoStatusChecker.Instance.IsConsultationInLiveVideo) { LiveVideoStatusChecker.Instance.TRTCRoomInfo = roomInfo; LiveVideoStatusChecker.Instance.RemoteVideoFrameDataEvent += OnRtcRemoteVideoFrameArrived; LiveVideoStatusChecker.Instance.IsConsultationLivingInCurrentMachine = true; } else { _rtcRoom = CrossPlatformHelper.Instance.RtcRoomCreator.CreateRtcRoom(); _rtcRoom.LocalVideoFrameArrived += OnRtcLocalVideoFrameArrived; _rtcRoom.RemoteVideoFrameArrived += OnRtcRemoteVideoFrameArrived; _rtcRoom.OnTRTCRoomEnterError += OnTRTCRoomEnterErrorHappened; _rtcRoom.RemoteUserLeaveRoomArrived += OnRemoteUserLeaveRoomArrived; _rtcRoom.TryToReconnect += OnTryToReconnect; _rtcRoom.Enter(roomInfo); Logger.WriteLineInfo($"{accountName} enterred room {integerRoomId}"); } if (member.State == LiveStates.OK) { var tokenRequest = new TokenRequest { Token = accountToken, }; int interval = 10; var serverSettingResult = JsonRpcHelper.GetServerSetting(_organizationService, tokenRequest); if (serverSettingResult == null) { Logger.WriteLineError($"GetServerSetting Error:result is null"); } else { interval = serverSettingResult.LiveConsultationRateSeconds; } videoProvider.StartConsultationConnectionKeeper(accountToken, consultationInfo.ConsultationId, interval, _liveConsultationService); videoProvider.Offlined += OnOfflined; } } else { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == member.Id); if (videoProvider == null) { Logger.WriteLineInfo($"Start To Create RTC Video Provider, Name:{member.Name},Id:{member.Id}"); _videoProviders.Add(new RtcVideoProviderV2(member)); } else { Logger.WriteLineInfo($"RTC Video Provider Already exist, Name:{member.Name},Id:{member.Id}"); videoProvider.ConsultationMemberInfo = member; } } } } } else if (_liveProtocol == LiveProtocol.Rtmp) { foreach (var member in consultationMembers) { if (member.Id == _accountId) { try { var pusherUrl = member.PushUrl; VideoProviderV2 videoProvider; if (!string.IsNullOrEmpty(pusherUrl)) { Logger.WriteLineInfo($"Start push with rtmp address {pusherUrl}"); videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == member.Id); if (videoProvider == null) { Logger.WriteLineInfo($"Start To Create Rtmp Video Pusher, Name:{member.Name},Id:{member.Id}"); if (LiveVideoStatusChecker.Instance.IsConsultationInLiveVideo) { LiveVideoStatusChecker.Instance.IsConsultationLivingInCurrentMachine = true; videoProvider = new RtmpVideoProviderV2(member); } else { videoProvider = new RtmpVideoPusherV2(member, videoDeviceId, micDeviceId, _isOnlyRtmpPushing); if (!_isOnlyRtmpPushing) { videoProvider.VideoFrameArrived += OnLocalVideoFrameArrived; } } _videoProviders.Add(videoProvider); } else { Logger.WriteLineInfo($"Rtmp video Pusher already exists,Name:{member.Name},Id:{member.Id}"); videoProvider.ConsultationMemberInfo = member; } if (member.State == LiveStates.OK) { var tokenRequest = new TokenRequest { Token = accountToken, }; int interval = 10; var serverSettingResult = JsonRpcHelper.GetServerSetting(_organizationService, tokenRequest); if (serverSettingResult == null) { Logger.WriteLineError($"GetServerSetting Error:result is null"); } else { interval = serverSettingResult.LiveConsultationRateSeconds; } videoProvider.StartConsultationConnectionKeeper(accountToken, consultationInfo.ConsultationId, interval, _liveConsultationService); videoProvider.Offlined += OnOfflined; } } } catch (Exception ex) { Logger.WriteLineError($"Exception happened while play rtmp{ex}"); } } else if (!_isOnlyRtmpPushing) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == member.Id); if (videoProvider == null) { Logger.WriteLineInfo($"Start To Create Rtmp Player,Name:{member.Name},Id:{member.Id}"); CreateRtmpPlayer(member); } else { Logger.WriteLineInfo($"Rtmp Player already exists,Name:{member.Name},Id:{member.Id}"); videoProvider.ConsultationMemberInfo = member; } } } } } } private void OnTerminalCameraFrameArrived(object sender, CPVideoFrameData e) { if (_isConsulting && _consultationInfo != null) { DeviceCameraFrameArrived?.Invoke(this, e); } } private void OnRtcLocalVideoFrameArrived(object sender, CPVideoFrameData e) { var consultationMembers = _consultationInfo.ConsultationMemberInfos; var consultationMember = consultationMembers.FirstOrDefault(x => x.Id == _accountId); if (consultationMember != null) { OnLocalVideoFrameArrived(this, new ConsultationVideoFrameData(e, consultationMember)); } } private void OnLocalVideoFrameArrived(object sender, ConsultationVideoFrameData e) { ConsultationLocalVideoFrameArrived?.Invoke(this, e); } private void OnRtcRemoteVideoFrameArrived(object sender, CPRemoteVideoFrameData e) { try { if (_consultationInfo != null) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == e.UserId); if (videoProvider != null) { OnVideoFrameArrived(videoProvider, e.Data); } _imagePreviewErrorLogCount = 0; } } catch (Exception ex) { if (_imagePreviewErrorLogCount++ < 1) { Logger.WriteLineError($"OnRtcRemoteVideoFrameArrived ex:{ex}"); } } } private void OnVideoFrameArrived(VideoProviderV2 videoProvider, CPVideoFrameData e) { OnRemoteVideoFrameArrived(this, new ConsultationVideoFrameData(e, videoProvider.ConsultationMemberInfo)); } private void OnTRTCRoomEnterErrorHappened(object sender, EnumTRTCRoomError e) { OnTRTCRoomEnterError?.Invoke(sender, e); } private void OnTryToReconnect(object sender, EventArgs e) { TryToReconnect?.Invoke(this, EventArgs.Empty); } private void OnOfflined(object sender, EventArgs e) { CloseConsultation?.Invoke(sender, e); } private void CreateRtmpPlayer(ConsultationMemberInfo consultationMember) { var rtmpVideoProvider = new RtmpVideoPlayerV2(consultationMember); rtmpVideoProvider.VideoFrameArrived += OnRemoteVideoFrameArrived; _videoProviders.Add(rtmpVideoProvider); } private void OnRemoteVideoFrameArrived(object sender, ConsultationVideoFrameData e) { ConsultationRemoteVideoFrameArrived?.Invoke(this, e); } private void OnRemoteUserLeaveRoomArrived(object sender, RemoteUserLeaveRoomArgs e) { RemoteUserLeaveRoomArrived?.Invoke(this, e); } /// /// 会诊人员发生变化 /// public void ConsultationMemberChange(ConsultationMemberChangeDTO meetingChangeMember) { if (_liveProtocol == LiveProtocol.RTC) { var memberInfo = meetingChangeMember.MemberInfo; if (memberInfo.OperationType == ClientMessageOperationType.Add) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == memberInfo.Id); if (videoProvider == null) { _videoProviders.Add(new RtcVideoProviderV2(memberInfo)); } else { videoProvider.ConsultationMemberInfo = memberInfo; } } else if (memberInfo.OperationType == ClientMessageOperationType.Delete) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == memberInfo.Id); if (videoProvider != null) { videoProvider.Offlined -= OnOfflined; videoProvider.Dispose(); _videoProviders.Remove(videoProvider); } } } else if (_liveProtocol == LiveProtocol.Rtmp && !_isOnlyRtmpPushing) { var memberInfo = meetingChangeMember.MemberInfo; if (memberInfo.OperationType == ClientMessageOperationType.Add) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == memberInfo.Id); if (videoProvider == null) { CreateRtmpPlayer(memberInfo); } else { videoProvider.ConsultationMemberInfo = memberInfo; } } else if (memberInfo.OperationType == ClientMessageOperationType.Delete) { var videoProvider = _videoProviders.FirstOrDefault(f => f.ConsultationMemberInfo.Id == memberInfo.Id); if (videoProvider != null) { videoProvider.Dispose(); _videoProviders.Remove(videoProvider); } } } } public void RemoveVideoProvider(string userId) { lock (_startConsultationLocker) { if (_videoProviders.Count > 0) { var video = _videoProviders.FirstOrDefault(v => v.ConsultationMemberInfo.Id == userId); if (video != null) { video.Dispose(); _videoProviders.Remove(video); } } } } /// /// 挂断 /// public void Hangup() { if (!_isConsulting) { return; } lock (_startConsultationLocker) { try { if (_isConsulting) { if (_videoProviders.Count > 0) { for (int i = _videoProviders.Count - 1; i >= 0; i--) { var videoProvider = _videoProviders[i]; UnregisterReference(videoProvider); videoProvider.Dispose(); } _videoProviders.Clear(); LiveVideoStatusChecker.Instance.RemoteVideoFrameDataEvent -= OnRtcRemoteVideoFrameArrived; LiveVideoStatusChecker.Instance.IsConsultationLivingInCurrentMachine = false; if (_liveProtocol == LiveProtocol.RTC) { if (_rtcRoom != null) { _rtcRoom.LocalVideoFrameArrived -= OnRtcLocalVideoFrameArrived; _rtcRoom.RemoteVideoFrameArrived -= OnRtcRemoteVideoFrameArrived; _rtcRoom.TryToReconnect -= OnTryToReconnect; _rtcRoom.RemoteUserLeaveRoomArrived -= OnRemoteUserLeaveRoomArrived; _rtcRoom.Exit(); } } } } } catch (Exception ex) { Logger.WriteLineInfo($"Hangup failed ex: {ex}"); } finally { _isConsulting = false; _isOnlyRtmpPushing = false; } } } private void UnregisterReference(VideoProviderV2 videoProvider) { if (videoProvider.ConsultationMemberInfo.Id == _accountId) { videoProvider.VideoFrameArrived -= OnLocalVideoFrameArrived; videoProvider.Offlined -= OnOfflined; } else { videoProvider.VideoFrameArrived -= OnRemoteVideoFrameArrived; videoProvider.Offlined -= OnOfflined; } } public void SwitchCamera() { lock (_lock) { if (_isConsulting && _consultationInfo != null) { if (_liveProtocol == LiveProtocol.RTC) { _rtcRoom?.SwitchCamera(); } else { var rtmpPusher = (RtmpVideoPusherV2)_videoProviders.FirstOrDefault(x => x is RtmpVideoPusherV2); if (rtmpPusher != null) { rtmpPusher.SwitchCamera(); } } } } } public void Mute(bool isMute) { lock (_lock) { if (_isConsulting && _consultationInfo != null) { if (_liveProtocol == LiveProtocol.RTC) { _rtcRoom?.Mute(isMute); } else { var rtmpPusher = (RtmpVideoPusherV2)_videoProviders.FirstOrDefault(x => x is RtmpVideoPusherV2); if (rtmpPusher != null) { rtmpPusher.SetMute(isMute); } } } } } public void Dispose() { ScanCameraCacheHelper.Instance.ScanCameraVideoFrameDataArrived -= OnTerminalCameraFrameArrived; Hangup(); } internal void UpdateTerminalInfo(TerminalInfo terminalInfo) { if (_consultationInfo != null && _isConsulting) { _consultationInfo.TerminalInfo = terminalInfo; } if (_liveProtocol == LiveProtocol.RTC) { _rtcRoom?.UpdateCameraId(terminalInfo.CameraId); } } } }