using AI.Common;
using AI.Common.Log;
using AI.Common.Tools;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Timers;
namespace AutoEFInferenceCalcLib
{
///
/// AutoEF 的计算
///
public class AutoEFCalculation
{
#region field
private int _numCpu = 0;
private int _layerCount = 20;
private int _edControlLayersCount = 12;
private int _esControlLayersCount = 10;
private static InferenceCore _inferenceCore = new InferenceCore();
private List _inferNets = new List();
private bool _initialized = false;
private ConcurrentDictionary _calculations = new ConcurrentDictionary();
//超过一定时间阈值没有再塞图,则删除该计算,以便减少内存占用和避免内存溢出的风险
private double _noNewImagePushedTimeThreshold = 1200;
private bool _closing = false;
private ConcurrentQueue _imageInfoQueue = new ConcurrentQueue();
//将处理线程唤醒的一个时间,设置初始值为true,表示有信号,
//有信号的时候,_threadAwakeEvent.WaitOne不会阻塞,没有信号的时候则会阻塞
//调用_threadAwakeEvent.Set,将对象设置为有信号状态
//调用_threadAwakeEvent.Reset,将对象重置为无信号状态
private readonly ManualResetEvent _threadAwakeEvent = new ManualResetEvent(true);
private readonly object _processThreadLocker = new object();
private List _threads = new List();
#endregion
#region properties
public int NumCpu { get => _numCpu; }
public int LayerCount { get => _layerCount; }
public int EDControlLayersCount { get => _edControlLayersCount; }
public int ESControlLayersCount { get => _esControlLayersCount; }
#endregion
#region cpp dll中的数据结构
///
/// cpp中点的结构
///
[StructLayout(LayoutKind.Sequential)]
internal struct StructPointCpp
{
public int x;
public int y;
}
///
/// cpp中的轮廓外接框结构
///
[StructLayout(LayoutKind.Sequential)]
internal struct StructRect
{
public int left;
public int top;
public int width;
public int height;
}
[StructLayout(LayoutKind.Sequential)]
internal struct StructEndPoints
{
public StructPointCpp leftPoint;
public StructPointCpp rightPoint;
}
///
/// 单帧图像计算的左心室相关
///
[StructLayout(LayoutKind.Sequential)]
internal struct StructCppLVVolume
{
public double volume;
public int innerContLen;
public StructPointCpp apexPoint;
public StructEndPoints endPoints;
}
#endregion
#region import dll
[DllImport(@"AutoEFPostProcess.dll", CallingConvention = CallingConvention.Cdecl)]
[return: MarshalAs(UnmanagedType.I1)]
internal static extern bool LVVolumeCalculate(int oriWidth, int oriHeight, float cmPerPixel, int layerCount,
int edControlLayersCount, int esControlLayerCount, int[] controlLayers, IntPtr pDetectedObject,
int contoureLen, IntPtr pRect, IntPtr slicePoints, IntPtr innerContour, IntPtr controlPoints, ref StructCppLVVolume pLvVolume);
#endregion
#region event
///
/// 通知订阅者,推理过程中发现了错误
///
public event EventHandler NotifyError;
///
/// 通知订阅者,推理过程中产生了log要记录
///
public event EventHandler NotifyLog;
#endregion
#region public func
///
/// 构造函数
///
///
///
///
///
///
///
public AutoEFCalculation(int numCpu = 1, string netDir = "", int layerCount = 20, int edControlLayersCount = 12,
int esControlLayerCount = 10)
{
try
{
if (numCpu <= 0)
{
throw new ArgumentOutOfRangeException("numCpu", "numCpu should be greater than 0.");
}
_numCpu = Math.Max(1, numCpu);
_layerCount = layerCount;
_edControlLayersCount = EDControlLayersCount;
_esControlLayersCount = ESControlLayersCount;
//加载网络
var inferNet = new InferNetOnnxAutoEFCalculation();
if (!inferNet.ModelLoaded)
{
byte[] trainedNetwork = InferenceNetworkUtils.ReadNetworkDataFromFile(netDir,
inferNet.NetworkName, inferNet.HashCode);
//设置inferenceCore的参数
inferNet.LoadNetwork(_inferenceCore, EnumDeviceType.CPU, trainedNetwork);
}
_inferNets.Add(inferNet);
for (int i = 0; i < _numCpu; i++)
{
var netClone = _inferNets[0].Clone(EnumCloneMethod.Clone);
_inferNets.Add(netClone);
}
_initialized = true;
}
catch (Exception excep)
{
#if ENABLE_AUTOEF_DEBUG
LogHelper.EnableLogWrite = true;
LogHelper.ErrorLog("AutoEF.AutoEFCalculation.Error:", excep.Message);
#endif
throw new Exception("Failed at Loading AutoEFCalculation:" + excep);
}
}
///
/// 开始一次新的计算
/// 在整个计算过程中的图像,cmPerPixel不应该发生变化
///
///
///
///
///
///
public string StartCalculation(double totalTime, float cmPerPixel, double intervalTime = 0)
{
try
{
if (!_initialized)
{
throw new Exception("Uninitialized before call StartCalculation!");
}
if (!(totalTime > 0))
{
throw new ArgumentOutOfRangeException("The value of totalTime must be greater than 0!");
}
if (!(cmPerPixel > 0))
{
throw new ArgumentOutOfRangeException("The value of cmPerPixel must be greater than 0!");
}
if (intervalTime > 500)
{
throw new ArgumentOutOfRangeException("The value of intervalTime must be less than 0.5s!");
}
//判断之前的计算是否已经过了一定的时间,过了就清理掉,防止占用内存
if (_calculations.Count != 0)
{
var keys = _calculations.Keys.ToList();
foreach (var key in keys)
{
TimeSpan timeDiff = DateTime.Now - _calculations[key].LastPushedImageSucceedTime;
if (timeDiff.TotalSeconds > _noNewImagePushedTimeThreshold)
{
_calculations.TryRemove(key, out _);
}
}
}
//生成随机码,作为calculationId
string calculationId = Guid.NewGuid().ToString();
//创建一个calculation并添加到_calculations里
CardiacCycleRecognition calculation = new CardiacCycleRecognition(calculationId, totalTime, cmPerPixel, intervalTime);
//注:创建时立即订阅事件,以便从calculation中能将事件传出
calculation.NotifyLog += OnLogWrite;
calculation.NotifyError += OnErrorOccur;
_calculations.TryAdd(calculationId, calculation);
//如果预设的时间间隔大于0.2s时,由于实际预测帧数较少, 导致预测的曲线相较于实际的相差较大,记一下WarnLog,提醒使用者
if (intervalTime >= 200)
{
#if ENABLE_AUTOEF_DEBUG
LogHelper.EnableLogWrite = true;
LogHelper.WarnLog("AutoEF.StartCalculation.Warn:", "calculationId:" + calculationId,
"intervalTime,should be less than or equal to 0.2s.");
#endif
}
return calculationId;
}
catch (Exception excep)
{
#if ENABLE_AUTOEF_DEBUG
LogHelper.EnableLogWrite = true;
LogHelper.ErrorLog("Failed at loading StartCalculation:" + excep.Message);
#endif
throw new Exception("Failed at loading StartCalculation:" + excep);
}
}
///
/// 塞入一幅图,计算完成后返回结果,实时计算使用
///
///
///
///
///
///
///
public CardiacCurveInfos PushOneImageSync(string calculationId, RawImage rawImage, double realFrameTime)
{
//判断输入图像的类型,并统一转成BGR
if (rawImage.ColorType == EnumColorType.GrayF32 || rawImage.ColorType == EnumColorType.Gray16)
{
throw new ArgumentOutOfRangeException("The ColorType of GrayF32 or Gray16 are not supported for PushOneImageSync by now!");
}
RawImage imgRGB = rawImage.Clone(EnumColorType.Bgr);
//判断此次calculation是否开启
if (!_calculations.ContainsKey(calculationId))
{
throw new Exception("calculation:" + calculationId + "shuould be started first.");
}
//判断当前计算类型有没有改变
//初始化时type类型是None,第一次推图后才会变成真实代表的类型, 所以要排除为None的情况
var type = _calculations[calculationId].Type;
if (type != CalcType.None && type != CalcType.OnLine)
{
throw new Exception("calculation:" + calculationId + "cannot call PushOneImageSync and PushOneImageAsync at the same time");
}
try
{
//塞图到队列里计算
PushOneImage(calculationId, imgRGB, realFrameTime, CalcType.OnLine);
//判断结果是否计算完
if (_calculations[calculationId].StartTime == realFrameTime)
{
//等待该图的计算结果
while (!_calculations[calculationId].ProcessFinished(realFrameTime))
{
Thread.Sleep(5);
}
}
else
{
//等待该图的计算结果
while (!_calculations[calculationId].ProcessFinished(realFrameTime))
{
Thread.Sleep(1);
}
}
return _calculations[calculationId].GetResult();
}
catch (Exception excep)
{
NotifyError?.Invoke(this, new ErrorEventArgs(excep));
return _calculations[calculationId].GetResult();
}
}
///
/// 塞入一副图像,不需要等待计算结果,冻结时使用
///
///
///
///
public void PushOneImageAsync(string caculationId, RawImage rawImage, double realFrameTime)
{
try
{
if (rawImage.ColorType == EnumColorType.GrayF32 || rawImage.ColorType == EnumColorType.Gray16)
{
throw new Exception("The colorType of GrayF16 and GrayF32 are not supported for PushOneImageAsync by now.");
}
RawImage imgBGR = rawImage.Clone(EnumColorType.Bgr);
//先判断此次calculation是否开启
if (!_calculations.ContainsKey(caculationId))
{
throw new Exception("calculation_" + caculationId + "should be started first.");
}
//判断type是否改变
//初始化时type设为None,第一次推图后才会变成真实所代表的类型,所以要排除type类型为None的情况
var type = _calculations[caculationId].Type;
if (type != CalcType.None && type != CalcType.OffLine)
{
throw new Exception("calculation_" + caculationId + "cannot call PushOneImageAsync and PushOneImageSync at the same time.");
}
//塞图到队列中计算
PushOneImage(caculationId, imgBGR, realFrameTime, CalcType.OffLine);
}
catch (Exception excep)
{
NotifyError?.Invoke(this, new ErrorEventArgs(excep));
#if ENABLE_AUTOEF_DEBUG
LogHelper.EnableLogWrite = true;
LogHelper.ErrorLog("AutoEF.PushOneImageAsync.Error:", excep.Message);
#endif
}
}
///
/// 获取结果(单次要处理的视频全部使用PushOneImageAsync塞图)
///
///
///
public CardiacCurveInfos GetResult(string calculationId)
{
try
{
//等待最后一帧结果计算完毕
var frameTime = _calculations[calculationId].LastPushedFrameTime;
while (!_calculations[calculationId].ProcessFinished(frameTime))
{
Thread.Sleep(1);
}
//获取计算结果
return _calculations[calculationId].GetResult();
}
catch (Exception excep)
{
NotifyError?.Invoke(this, new ErrorEventArgs(excep));
#if ENABLE_AUTOEF_DEBUG
LogHelper.EnableLogWrite = true;
LogHelper.ErrorLog("GetResult.Error:", excep.Message);
#endif
return _calculations[calculationId].GetResult();
}
}
///
/// 销毁
///
public void Dispose()
{
DoDispose();
GC.SuppressFinalize(this);
}
///
/// 析构函数
///
~AutoEFCalculation()
{
DoDispose();
}
#endregion
#region private
private void PushOneImage(string calculationId, RawImage rawImage, double realFrameTime, CalcType calcType)
{
//推图,如果成功则塞入队列开始计算,如果没有成功,则不做计算直接返回(返回的时候需要给当前时间赋一个空的检测结果),
//注意:_closing的时候不能继续塞图了, 要把所有未处理的数据都清空,如果这时候输入另外的数据,则会导致又重新开始计算了
if (!_closing && _calculations[calculationId].PushOneImage(realFrameTime, calcType))
{
//塞入的图像的信息
ImageInfo imageInfo = new ImageInfo
{
CalculationId = _calculations[calculationId].Id,
Time = realFrameTime,
RawImage = rawImage,
CmPerPixel = _calculations[calculationId].CmPerPixel,
};
_imageInfoQueue.Enqueue(imageInfo);
//设置_threadAwakeEvent状态为有信号,代表有新的输入,线程可以继续跑了
_threadAwakeEvent.Set();
int threadCount = Math.Min(_numCpu, _imageInfoQueue.Count);
for (int i = 0; i < threadCount; i++)
{
lock (_processThreadLocker)
{
if (_threads.Count < i + 1)
{
var thread = new Thread(DoImageProcessing);
thread.IsBackground = true;
thread.Name = "AutoEF_ImgProcessing_" + i.ToString();
_threads.Add(thread);
thread.Start(i);
}
else
{
var thread = _threads[i];
if (thread == null || !thread.IsAlive)
{
thread = new Thread(DoImageProcessing);
thread.IsBackground = true;
thread.Name = "AutoEF_ImgProcessing_" + i.ToString();
thread.Start(i);
_threads[i] = thread;
}
}
}
}
}
else
{
_calculations[calculationId].UpdateImageResult(realFrameTime, new LVVolumeCalcResult(), ResultType.None);
}
}
///
/// 单帧图像推理
///
///
private void DoImageProcessing(object threadIndex)
{
if (threadIndex == null)
{
return;
}
while (!_closing)
{
if (_imageInfoQueue.Count > 0)
{
if (_imageInfoQueue.TryDequeue(out var sigleImageInfo))
{
try
{
var frameTime = sigleImageInfo.Time;
var rawImage = sigleImageInfo.RawImage;
var cmPerPixel = sigleImageInfo.CmPerPixel;
var calculationId = sigleImageInfo.CalculationId;
//1.满足预设的时间间隔才实际推理,否则基于时间序列给出结果
//当前帧时间与上一次计算的时间间隔小于预设的时间间隔
if (frameTime - _calculations[calculationId].LastRealCalcFrameTimeStamp < _calculations[calculationId].IntervalTime &&
frameTime != _calculations[calculationId].StartTime)
{
_calculations[calculationId].UpdateImageResult(frameTime, new LVVolumeCalcResult(), ResultType.FrameSequenceBasedPredict);
continue;
}
//2.当本次计算最新Push的图像的时间戳与将要计算的时间戳大于一定的时间阈值(即计算滞后于Push一定的时间)
//则不计算,否则计算的结果将越来越滞后,导致计算失败
//一般输入帧率是20-30帧每秒,所以暂取0.15s,即如果滞后3帧,则不计算
if (frameTime - _calculations[calculationId].LastPushedFrameTime > Math.Max(_calculations[calculationId].IntervalTime, 150))
{
_calculations[calculationId].UpdateImageResult(frameTime, new LVVolumeCalcResult(), ResultType.FrameSequenceBasedPredict);
continue;
}
//3.正常情况下
_calculations[calculationId].LastRealCalcFrameTimeStamp = frameTime;
LVVolumeCalcResult resultPerImage = ProcessOneImage(rawImage, cmPerPixel, (int)threadIndex);
_calculations[calculationId].UpdateImageResult(frameTime, resultPerImage, ResultType.SingleFrameBasedPredict);
}
catch (Exception excep)
{
NotifyError.Invoke(this, new ErrorEventArgs(excep));
}
}
}
else
{
//将_threadAwakeEvent设为无信号,表示现在没有新的数据了,但又不想让线程结束(避免频繁创建和销毁线程带来的开销)
//所以让Waitone一直等卡在这里,直到又新的数据输入,waitOne才继续
_threadAwakeEvent.Reset();
_threadAwakeEvent.WaitOne();
}
}
}
///
/// 推理单帧图像结果
///
///
///
///
///
private LVVolumeCalcResult ProcessOneImage(RawImage rawImage, float cmPerPixel, int threadIndex = 0)
{
//对一张图像进行推理
InferenceNetworkInputImage inferInput = new InferenceNetworkInputImage(rawImage, new Rect(0, 0, rawImage.Width, rawImage.Height));
IDetectedObject[] segResult = _inferNets[threadIndex].Process(inferInput);
LVVolumeCalcResult resultPerImage = new LVVolumeCalcResult();
if (segResult.Length != 1)
{
return LVVolumeCalcResult.Empty;
}
resultPerImage = LVVolumeProcess(rawImage, cmPerPixel, segResult);
return resultPerImage;
}
///
/// 左心室容积推理
///
///
///
///
///
private LVVolumeCalcResult LVVolumeProcess(RawImage rawImage, float cmPerPixel, IDetectedObject[] segResult)
{
int apexPointSize = 1;
int endPointSize = 2;
int slicePointSize = _layerCount * 2; ;
int controlNum = _edControlLayersCount > _esControlLayersCount ? _edControlLayersCount : _esControlLayersCount;
//转换分割结果
int pointCount = segResult[0].Contours[0].Length;
StructPointCpp[] structPoints = new StructPointCpp[pointCount];
for (int i = 0; i < pointCount; i++)
{
structPoints[i].x = segResult[0].Contours[0][i].X;
structPoints[i].y = segResult[0].Contours[0][i].Y;
}
StructRect cppRect = new StructRect();
cppRect.left = segResult[0].BoundingBox.Left;
cppRect.top = segResult[0].BoundingBox.Top;
cppRect.width = segResult[0].BoundingBox.Width;
cppRect.height = segResult[0].BoundingBox.Height;
//初始化需要返回的结果
StructCppLVVolume structCppLVVolume = new StructCppLVVolume();
structCppLVVolume.volume = 0;
structCppLVVolume.innerContLen = 0;
structCppLVVolume.apexPoint = new StructPointCpp();
structCppLVVolume.endPoints = new StructEndPoints();
StructPointCpp[] slicePoints = new StructPointCpp[slicePointSize];
StructPointCpp[] innerContour = new StructPointCpp[rawImage.Width * rawImage.Height];
StructPointCpp[] controlPoints = new StructPointCpp[controlNum * 2];
int[] controlCount = new int[1];
controlCount[0] = 0;
//获取轮廓指针
GCHandle hDetectedObjectArray = GCHandle.Alloc(structPoints, GCHandleType.Pinned);
IntPtr pDetectedObject = hDetectedObjectArray.AddrOfPinnedObject();
//获取外接矩形的指针
GCHandle hObjectRect = GCHandle.Alloc(cppRect, GCHandleType.Pinned);
IntPtr pObjectRect = hObjectRect.AddrOfPinnedObject();
GCHandle hSlicePoint = GCHandle.Alloc(slicePoints, GCHandleType.Pinned);
IntPtr pSlicePoints = hSlicePoint.AddrOfPinnedObject();
GCHandle hInnerPoints = GCHandle.Alloc(innerContour, GCHandleType.Pinned);
IntPtr pInnerPoints = hInnerPoints.AddrOfPinnedObject();
GCHandle hControlPoints = GCHandle.Alloc(controlPoints, GCHandleType.Pinned);
IntPtr pConrolPoints = hControlPoints.AddrOfPinnedObject();
bool ret = false;
ret = LVVolumeCalculate(rawImage.Width, rawImage.Height, cmPerPixel, _layerCount, _edControlLayersCount,
_esControlLayersCount, controlCount, pDetectedObject, pointCount, pObjectRect,
pSlicePoints, pInnerPoints, pConrolPoints, ref structCppLVVolume);
//释放
if (hDetectedObjectArray.IsAllocated)
{
hDetectedObjectArray.Free();
}
if (hObjectRect.IsAllocated)
{
hObjectRect.Free();
}
if (hSlicePoint.IsAllocated)
{
hSlicePoint.Free();
}
if (hInnerPoints.IsAllocated)
{
hInnerPoints.Free();
}
if (hControlPoints.IsAllocated)
{
hControlPoints.Free();
}
//当返回false或计算的内轮廓的长度为0时,返回空值
if (!ret)
{
NotifyLog?.Invoke(this, new LogEventArgs(EnumLogType.WarnLog, "Failed at Loading LVVolumeCalculate."));
return LVVolumeCalcResult.Empty;
}
if (structCppLVVolume.innerContLen == 0 || controlCount[0] == 0)
{
return LVVolumeCalcResult.Empty;
}
//转换cpp计算的结果
Point2D[] endPoints = new Point2D[endPointSize];
for (int i = 0; i < endPointSize; i++)
{
endPoints[0] = new Point2D(structCppLVVolume.endPoints.leftPoint.x, structCppLVVolume.endPoints.leftPoint.y);
endPoints[1] = new Point2D(structCppLVVolume.endPoints.rightPoint.x, structCppLVVolume.endPoints.rightPoint.y);
}
Point2D[] slicePointsTran = new Point2D[slicePointSize];
for (int i = 0; i < slicePointSize; i++)
{
slicePointsTran[i] = new Point2D(slicePoints[i].x, slicePoints[i].y);
}
Point2D[] innerContourPoint = new Point2D[structCppLVVolume.innerContLen];
for (int i = 0; i < structCppLVVolume.innerContLen; i++)
{
innerContourPoint[i] = new Point2D(innerContour[i].x, innerContour[i].y);
}
Point2D[] controlPoint = new Point2D[controlCount[0]];
for (int i = 0; i < controlCount[0]; i++)
{
controlPoint[i] = new Point2D(controlPoints[i].x, controlPoints[i].y);
}
LVVolumeMeasureMarks lVVolumeMeasure = new LVVolumeMeasureMarks(new Point2D(structCppLVVolume.apexPoint.x,
structCppLVVolume.apexPoint.y), endPoints, slicePointsTran, controlPoint);
LVVolumeCalcResult lVVolumeCalcResult = new LVVolumeCalcResult(innerContourPoint, structCppLVVolume.volume,
segResult[0].Confidence, lVVolumeMeasure);
return lVVolumeCalcResult;
}
///
/// 主动销毁
///
private void DoDispose()
{
_closing = true;
//因为_threadAwakeEvent一直在waitone,所以需要先Set一下,使其不阻塞
_threadAwakeEvent.Set();
//等待所有线程结束
while (true)
{
bool runing = false;
lock (_processThreadLocker)
{
for (int i = 0; i < _threads.Count; i++)
{
var thread = _threads[i];
if (thread != null && thread.IsAlive)
{
runing = true;
break;
}
}
}
if (runing)
{
Thread.Sleep(10);
}
else
{
break;
}
}
//所有的calculation停止
foreach (var calculation in _calculations.Values)
{
calculation.NotifyLog -= OnLogWrite;
calculation.NotifyError -= OnErrorOccur;
calculation.Dispose();
}
//所有待处理数据清空
while (_imageInfoQueue.Count > 0)
{
_imageInfoQueue.TryDequeue(out var _);
}
//逐个释放inferNet
for (int i = 0; i < _inferNets.Count; i++)
{
_inferNets[i]?.Dispose();
_inferNets[i] = null;
}
_initialized = false;
_closing = false;
}
///
/// 有log要记
///
///
///
private void OnLogWrite(object sender, LogEventArgs e)
{
NotifyLog?.Invoke(this, e);
}
///
/// 推理过程中发生了错误
///
///
///
private void OnErrorOccur(object sender, ErrorEventArgs e)
{
NotifyError?.Invoke(this, e);
}
#endregion
}
}