using AI.Common; using AI.Common.Log; using AI.Common.Tools; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Timers; namespace AutoEFInferenceCalcLib { /// /// AutoEF 的计算 /// public class AutoEFCalculation { #region field private int _numCpu = 0; private int _layerCount = 20; private int _edControlLayersCount = 12; private int _esControlLayersCount = 10; private static InferenceCore _inferenceCore = new InferenceCore(); private List _inferNets = new List(); private bool _initialized = false; private ConcurrentDictionary _calculations = new ConcurrentDictionary(); //超过一定时间阈值没有再塞图,则删除该计算,以便减少内存占用和避免内存溢出的风险 private double _noNewImagePushedTimeThreshold = 1200; private bool _closing = false; private ConcurrentQueue _imageInfoQueue = new ConcurrentQueue(); //将处理线程唤醒的一个时间,设置初始值为true,表示有信号, //有信号的时候,_threadAwakeEvent.WaitOne不会阻塞,没有信号的时候则会阻塞 //调用_threadAwakeEvent.Set,将对象设置为有信号状态 //调用_threadAwakeEvent.Reset,将对象重置为无信号状态 private readonly ManualResetEvent _threadAwakeEvent = new ManualResetEvent(true); private readonly object _processThreadLocker = new object(); private List _threads = new List(); #endregion #region properties public int NumCpu { get => _numCpu; } public int LayerCount { get => _layerCount; } public int EDControlLayersCount { get => _edControlLayersCount; } public int ESControlLayersCount { get => _esControlLayersCount; } #endregion #region cpp dll中的数据结构 /// /// cpp中点的结构 /// [StructLayout(LayoutKind.Sequential)] internal struct StructPointCpp { public int x; public int y; } /// /// cpp中的轮廓外接框结构 /// [StructLayout(LayoutKind.Sequential)] internal struct StructRect { public int left; public int top; public int width; public int height; } [StructLayout(LayoutKind.Sequential)] internal struct StructEndPoints { public StructPointCpp leftPoint; public StructPointCpp rightPoint; } /// /// 单帧图像计算的左心室相关 /// [StructLayout(LayoutKind.Sequential)] internal struct StructCppLVVolume { public double volume; public int innerContLen; public StructPointCpp apexPoint; public StructEndPoints endPoints; } #endregion #region import dll [DllImport(@"AutoEFPostProcess.dll", CallingConvention = CallingConvention.Cdecl)] [return: MarshalAs(UnmanagedType.I1)] internal static extern bool LVVolumeCalculate(int oriWidth, int oriHeight, float cmPerPixel, int layerCount, int edControlLayersCount, int esControlLayerCount, int[] controlLayers, IntPtr pDetectedObject, int contoureLen, IntPtr pRect, IntPtr slicePoints, IntPtr innerContour, IntPtr controlPoints, ref StructCppLVVolume pLvVolume); #endregion #region event /// /// 通知订阅者,推理过程中发现了错误 /// public event EventHandler NotifyError; /// /// 通知订阅者,推理过程中产生了log要记录 /// public event EventHandler NotifyLog; #endregion #region public func /// /// 构造函数 /// /// /// /// /// /// /// public AutoEFCalculation(int numCpu = 1, string netDir = "", int layerCount = 20, int edControlLayersCount = 12, int esControlLayerCount = 10) { try { if (numCpu <= 0) { throw new ArgumentOutOfRangeException("numCpu", "numCpu should be greater than 0."); } _numCpu = Math.Max(1, numCpu); _layerCount = layerCount; _edControlLayersCount = EDControlLayersCount; _esControlLayersCount = ESControlLayersCount; //加载网络 var inferNet = new InferNetOnnxAutoEFCalculation(); if (!inferNet.ModelLoaded) { byte[] trainedNetwork = InferenceNetworkUtils.ReadNetworkDataFromFile(netDir, inferNet.NetworkName, inferNet.HashCode); //设置inferenceCore的参数 inferNet.LoadNetwork(_inferenceCore, EnumDeviceType.CPU, trainedNetwork); } _inferNets.Add(inferNet); for (int i = 0; i < _numCpu; i++) { var netClone = _inferNets[0].Clone(EnumCloneMethod.Clone); _inferNets.Add(netClone); } _initialized = true; } catch (Exception excep) { #if ENABLE_AUTOEF_DEBUG LogHelper.EnableLogWrite = true; LogHelper.ErrorLog("AutoEF.AutoEFCalculation.Error:", excep.Message); #endif throw new Exception("Failed at Loading AutoEFCalculation:" + excep); } } /// /// 开始一次新的计算 /// 在整个计算过程中的图像,cmPerPixel不应该发生变化 /// /// /// /// /// /// public string StartCalculation(double totalTime, float cmPerPixel, double intervalTime = 0) { try { if (!_initialized) { throw new Exception("Uninitialized before call StartCalculation!"); } if (!(totalTime > 0)) { throw new ArgumentOutOfRangeException("The value of totalTime must be greater than 0!"); } if (!(cmPerPixel > 0)) { throw new ArgumentOutOfRangeException("The value of cmPerPixel must be greater than 0!"); } if (intervalTime > 500) { throw new ArgumentOutOfRangeException("The value of intervalTime must be less than 0.5s!"); } //判断之前的计算是否已经过了一定的时间,过了就清理掉,防止占用内存 if (_calculations.Count != 0) { var keys = _calculations.Keys.ToList(); foreach (var key in keys) { TimeSpan timeDiff = DateTime.Now - _calculations[key].LastPushedImageSucceedTime; if (timeDiff.TotalSeconds > _noNewImagePushedTimeThreshold) { _calculations.TryRemove(key, out _); } } } //生成随机码,作为calculationId string calculationId = Guid.NewGuid().ToString(); //创建一个calculation并添加到_calculations里 CardiacCycleRecognition calculation = new CardiacCycleRecognition(calculationId, totalTime, cmPerPixel, intervalTime); //注:创建时立即订阅事件,以便从calculation中能将事件传出 calculation.NotifyLog += OnLogWrite; calculation.NotifyError += OnErrorOccur; _calculations.TryAdd(calculationId, calculation); //如果预设的时间间隔大于0.2s时,由于实际预测帧数较少, 导致预测的曲线相较于实际的相差较大,记一下WarnLog,提醒使用者 if (intervalTime >= 200) { #if ENABLE_AUTOEF_DEBUG LogHelper.EnableLogWrite = true; LogHelper.WarnLog("AutoEF.StartCalculation.Warn:", "calculationId:" + calculationId, "intervalTime,should be less than or equal to 0.2s."); #endif } return calculationId; } catch (Exception excep) { #if ENABLE_AUTOEF_DEBUG LogHelper.EnableLogWrite = true; LogHelper.ErrorLog("Failed at loading StartCalculation:" + excep.Message); #endif throw new Exception("Failed at loading StartCalculation:" + excep); } } /// /// 塞入一幅图,计算完成后返回结果,实时计算使用 /// /// /// /// /// /// /// public CardiacCurveInfos PushOneImageSync(string calculationId, RawImage rawImage, double realFrameTime) { //判断输入图像的类型,并统一转成BGR if (rawImage.ColorType == EnumColorType.GrayF32 || rawImage.ColorType == EnumColorType.Gray16) { throw new ArgumentOutOfRangeException("The ColorType of GrayF32 or Gray16 are not supported for PushOneImageSync by now!"); } RawImage imgRGB = rawImage.Clone(EnumColorType.Bgr); //判断此次calculation是否开启 if (!_calculations.ContainsKey(calculationId)) { throw new Exception("calculation:" + calculationId + "shuould be started first."); } //判断当前计算类型有没有改变 //初始化时type类型是None,第一次推图后才会变成真实代表的类型, 所以要排除为None的情况 var type = _calculations[calculationId].Type; if (type != CalcType.None && type != CalcType.OnLine) { throw new Exception("calculation:" + calculationId + "cannot call PushOneImageSync and PushOneImageAsync at the same time"); } try { //塞图到队列里计算 PushOneImage(calculationId, imgRGB, realFrameTime, CalcType.OnLine); //判断结果是否计算完 if (_calculations[calculationId].StartTime == realFrameTime) { //等待该图的计算结果 while (!_calculations[calculationId].ProcessFinished(realFrameTime)) { Thread.Sleep(5); } } else { //等待该图的计算结果 while (!_calculations[calculationId].ProcessFinished(realFrameTime)) { Thread.Sleep(1); } } return _calculations[calculationId].GetResult(); } catch (Exception excep) { NotifyError?.Invoke(this, new ErrorEventArgs(excep)); return _calculations[calculationId].GetResult(); } } /// /// 塞入一副图像,不需要等待计算结果,冻结时使用 /// /// /// /// public void PushOneImageAsync(string caculationId, RawImage rawImage, double realFrameTime) { try { if (rawImage.ColorType == EnumColorType.GrayF32 || rawImage.ColorType == EnumColorType.Gray16) { throw new Exception("The colorType of GrayF16 and GrayF32 are not supported for PushOneImageAsync by now."); } RawImage imgBGR = rawImage.Clone(EnumColorType.Bgr); //先判断此次calculation是否开启 if (!_calculations.ContainsKey(caculationId)) { throw new Exception("calculation_" + caculationId + "should be started first."); } //判断type是否改变 //初始化时type设为None,第一次推图后才会变成真实所代表的类型,所以要排除type类型为None的情况 var type = _calculations[caculationId].Type; if (type != CalcType.None && type != CalcType.OffLine) { throw new Exception("calculation_" + caculationId + "cannot call PushOneImageAsync and PushOneImageSync at the same time."); } //塞图到队列中计算 PushOneImage(caculationId, imgBGR, realFrameTime, CalcType.OffLine); } catch (Exception excep) { NotifyError?.Invoke(this, new ErrorEventArgs(excep)); #if ENABLE_AUTOEF_DEBUG LogHelper.EnableLogWrite = true; LogHelper.ErrorLog("AutoEF.PushOneImageAsync.Error:", excep.Message); #endif } } /// /// 获取结果(单次要处理的视频全部使用PushOneImageAsync塞图) /// /// /// public CardiacCurveInfos GetResult(string calculationId) { try { //等待最后一帧结果计算完毕 var frameTime = _calculations[calculationId].LastPushedFrameTime; while (!_calculations[calculationId].ProcessFinished(frameTime)) { Thread.Sleep(1); } //获取计算结果 return _calculations[calculationId].GetResult(); } catch (Exception excep) { NotifyError?.Invoke(this, new ErrorEventArgs(excep)); #if ENABLE_AUTOEF_DEBUG LogHelper.EnableLogWrite = true; LogHelper.ErrorLog("GetResult.Error:", excep.Message); #endif return _calculations[calculationId].GetResult(); } } /// /// 销毁 /// public void Dispose() { DoDispose(); GC.SuppressFinalize(this); } /// /// 析构函数 /// ~AutoEFCalculation() { DoDispose(); } #endregion #region private private void PushOneImage(string calculationId, RawImage rawImage, double realFrameTime, CalcType calcType) { //推图,如果成功则塞入队列开始计算,如果没有成功,则不做计算直接返回(返回的时候需要给当前时间赋一个空的检测结果), //注意:_closing的时候不能继续塞图了, 要把所有未处理的数据都清空,如果这时候输入另外的数据,则会导致又重新开始计算了 if (!_closing && _calculations[calculationId].PushOneImage(realFrameTime, calcType)) { //塞入的图像的信息 ImageInfo imageInfo = new ImageInfo { CalculationId = _calculations[calculationId].Id, Time = realFrameTime, RawImage = rawImage, CmPerPixel = _calculations[calculationId].CmPerPixel, }; _imageInfoQueue.Enqueue(imageInfo); //设置_threadAwakeEvent状态为有信号,代表有新的输入,线程可以继续跑了 _threadAwakeEvent.Set(); int threadCount = Math.Min(_numCpu, _imageInfoQueue.Count); for (int i = 0; i < threadCount; i++) { lock (_processThreadLocker) { if (_threads.Count < i + 1) { var thread = new Thread(DoImageProcessing); thread.IsBackground = true; thread.Name = "AutoEF_ImgProcessing_" + i.ToString(); _threads.Add(thread); thread.Start(i); } else { var thread = _threads[i]; if (thread == null || !thread.IsAlive) { thread = new Thread(DoImageProcessing); thread.IsBackground = true; thread.Name = "AutoEF_ImgProcessing_" + i.ToString(); thread.Start(i); _threads[i] = thread; } } } } } else { _calculations[calculationId].UpdateImageResult(realFrameTime, new LVVolumeCalcResult(), ResultType.None); } } /// /// 单帧图像推理 /// /// private void DoImageProcessing(object threadIndex) { if (threadIndex == null) { return; } while (!_closing) { if (_imageInfoQueue.Count > 0) { if (_imageInfoQueue.TryDequeue(out var sigleImageInfo)) { try { var frameTime = sigleImageInfo.Time; var rawImage = sigleImageInfo.RawImage; var cmPerPixel = sigleImageInfo.CmPerPixel; var calculationId = sigleImageInfo.CalculationId; //1.满足预设的时间间隔才实际推理,否则基于时间序列给出结果 //当前帧时间与上一次计算的时间间隔小于预设的时间间隔 if (frameTime - _calculations[calculationId].LastRealCalcFrameTimeStamp < _calculations[calculationId].IntervalTime && frameTime != _calculations[calculationId].StartTime) { _calculations[calculationId].UpdateImageResult(frameTime, new LVVolumeCalcResult(), ResultType.FrameSequenceBasedPredict); continue; } //2.当本次计算最新Push的图像的时间戳与将要计算的时间戳大于一定的时间阈值(即计算滞后于Push一定的时间) //则不计算,否则计算的结果将越来越滞后,导致计算失败 //一般输入帧率是20-30帧每秒,所以暂取0.15s,即如果滞后3帧,则不计算 if (frameTime - _calculations[calculationId].LastPushedFrameTime > Math.Max(_calculations[calculationId].IntervalTime, 150)) { _calculations[calculationId].UpdateImageResult(frameTime, new LVVolumeCalcResult(), ResultType.FrameSequenceBasedPredict); continue; } //3.正常情况下 _calculations[calculationId].LastRealCalcFrameTimeStamp = frameTime; LVVolumeCalcResult resultPerImage = ProcessOneImage(rawImage, cmPerPixel, (int)threadIndex); _calculations[calculationId].UpdateImageResult(frameTime, resultPerImage, ResultType.SingleFrameBasedPredict); } catch (Exception excep) { NotifyError.Invoke(this, new ErrorEventArgs(excep)); } } } else { //将_threadAwakeEvent设为无信号,表示现在没有新的数据了,但又不想让线程结束(避免频繁创建和销毁线程带来的开销) //所以让Waitone一直等卡在这里,直到又新的数据输入,waitOne才继续 _threadAwakeEvent.Reset(); _threadAwakeEvent.WaitOne(); } } } /// /// 推理单帧图像结果 /// /// /// /// /// private LVVolumeCalcResult ProcessOneImage(RawImage rawImage, float cmPerPixel, int threadIndex = 0) { //对一张图像进行推理 InferenceNetworkInputImage inferInput = new InferenceNetworkInputImage(rawImage, new Rect(0, 0, rawImage.Width, rawImage.Height)); IDetectedObject[] segResult = _inferNets[threadIndex].Process(inferInput); LVVolumeCalcResult resultPerImage = new LVVolumeCalcResult(); if (segResult.Length != 1) { return LVVolumeCalcResult.Empty; } resultPerImage = LVVolumeProcess(rawImage, cmPerPixel, segResult); return resultPerImage; } /// /// 左心室容积推理 /// /// /// /// /// private LVVolumeCalcResult LVVolumeProcess(RawImage rawImage, float cmPerPixel, IDetectedObject[] segResult) { int apexPointSize = 1; int endPointSize = 2; int slicePointSize = _layerCount * 2; ; int controlNum = _edControlLayersCount > _esControlLayersCount ? _edControlLayersCount : _esControlLayersCount; //转换分割结果 int pointCount = segResult[0].Contours[0].Length; StructPointCpp[] structPoints = new StructPointCpp[pointCount]; for (int i = 0; i < pointCount; i++) { structPoints[i].x = segResult[0].Contours[0][i].X; structPoints[i].y = segResult[0].Contours[0][i].Y; } StructRect cppRect = new StructRect(); cppRect.left = segResult[0].BoundingBox.Left; cppRect.top = segResult[0].BoundingBox.Top; cppRect.width = segResult[0].BoundingBox.Width; cppRect.height = segResult[0].BoundingBox.Height; //初始化需要返回的结果 StructCppLVVolume structCppLVVolume = new StructCppLVVolume(); structCppLVVolume.volume = 0; structCppLVVolume.innerContLen = 0; structCppLVVolume.apexPoint = new StructPointCpp(); structCppLVVolume.endPoints = new StructEndPoints(); StructPointCpp[] slicePoints = new StructPointCpp[slicePointSize]; StructPointCpp[] innerContour = new StructPointCpp[rawImage.Width * rawImage.Height]; StructPointCpp[] controlPoints = new StructPointCpp[controlNum * 2]; int[] controlCount = new int[1]; controlCount[0] = 0; //获取轮廓指针 GCHandle hDetectedObjectArray = GCHandle.Alloc(structPoints, GCHandleType.Pinned); IntPtr pDetectedObject = hDetectedObjectArray.AddrOfPinnedObject(); //获取外接矩形的指针 GCHandle hObjectRect = GCHandle.Alloc(cppRect, GCHandleType.Pinned); IntPtr pObjectRect = hObjectRect.AddrOfPinnedObject(); GCHandle hSlicePoint = GCHandle.Alloc(slicePoints, GCHandleType.Pinned); IntPtr pSlicePoints = hSlicePoint.AddrOfPinnedObject(); GCHandle hInnerPoints = GCHandle.Alloc(innerContour, GCHandleType.Pinned); IntPtr pInnerPoints = hInnerPoints.AddrOfPinnedObject(); GCHandle hControlPoints = GCHandle.Alloc(controlPoints, GCHandleType.Pinned); IntPtr pConrolPoints = hControlPoints.AddrOfPinnedObject(); bool ret = false; ret = LVVolumeCalculate(rawImage.Width, rawImage.Height, cmPerPixel, _layerCount, _edControlLayersCount, _esControlLayersCount, controlCount, pDetectedObject, pointCount, pObjectRect, pSlicePoints, pInnerPoints, pConrolPoints, ref structCppLVVolume); //释放 if (hDetectedObjectArray.IsAllocated) { hDetectedObjectArray.Free(); } if (hObjectRect.IsAllocated) { hObjectRect.Free(); } if (hSlicePoint.IsAllocated) { hSlicePoint.Free(); } if (hInnerPoints.IsAllocated) { hInnerPoints.Free(); } if (hControlPoints.IsAllocated) { hControlPoints.Free(); } //当返回false或计算的内轮廓的长度为0时,返回空值 if (!ret) { NotifyLog?.Invoke(this, new LogEventArgs(EnumLogType.WarnLog, "Failed at Loading LVVolumeCalculate.")); return LVVolumeCalcResult.Empty; } if (structCppLVVolume.innerContLen == 0 || controlCount[0] == 0) { return LVVolumeCalcResult.Empty; } //转换cpp计算的结果 Point2D[] endPoints = new Point2D[endPointSize]; for (int i = 0; i < endPointSize; i++) { endPoints[0] = new Point2D(structCppLVVolume.endPoints.leftPoint.x, structCppLVVolume.endPoints.leftPoint.y); endPoints[1] = new Point2D(structCppLVVolume.endPoints.rightPoint.x, structCppLVVolume.endPoints.rightPoint.y); } Point2D[] slicePointsTran = new Point2D[slicePointSize]; for (int i = 0; i < slicePointSize; i++) { slicePointsTran[i] = new Point2D(slicePoints[i].x, slicePoints[i].y); } Point2D[] innerContourPoint = new Point2D[structCppLVVolume.innerContLen]; for (int i = 0; i < structCppLVVolume.innerContLen; i++) { innerContourPoint[i] = new Point2D(innerContour[i].x, innerContour[i].y); } Point2D[] controlPoint = new Point2D[controlCount[0]]; for (int i = 0; i < controlCount[0]; i++) { controlPoint[i] = new Point2D(controlPoints[i].x, controlPoints[i].y); } LVVolumeMeasureMarks lVVolumeMeasure = new LVVolumeMeasureMarks(new Point2D(structCppLVVolume.apexPoint.x, structCppLVVolume.apexPoint.y), endPoints, slicePointsTran, controlPoint); LVVolumeCalcResult lVVolumeCalcResult = new LVVolumeCalcResult(innerContourPoint, structCppLVVolume.volume, segResult[0].Confidence, lVVolumeMeasure); return lVVolumeCalcResult; } /// /// 主动销毁 /// private void DoDispose() { _closing = true; //因为_threadAwakeEvent一直在waitone,所以需要先Set一下,使其不阻塞 _threadAwakeEvent.Set(); //等待所有线程结束 while (true) { bool runing = false; lock (_processThreadLocker) { for (int i = 0; i < _threads.Count; i++) { var thread = _threads[i]; if (thread != null && thread.IsAlive) { runing = true; break; } } } if (runing) { Thread.Sleep(10); } else { break; } } //所有的calculation停止 foreach (var calculation in _calculations.Values) { calculation.NotifyLog -= OnLogWrite; calculation.NotifyError -= OnErrorOccur; calculation.Dispose(); } //所有待处理数据清空 while (_imageInfoQueue.Count > 0) { _imageInfoQueue.TryDequeue(out var _); } //逐个释放inferNet for (int i = 0; i < _inferNets.Count; i++) { _inferNets[i]?.Dispose(); _inferNets[i] = null; } _initialized = false; _closing = false; } /// /// 有log要记 /// /// /// private void OnLogWrite(object sender, LogEventArgs e) { NotifyLog?.Invoke(this, e); } /// /// 推理过程中发生了错误 /// /// /// private void OnErrorOccur(object sender, ErrorEventArgs e) { NotifyError?.Invoke(this, e); } #endregion } }