WingAIDiagnosisService/Carotid/Utilities/GetModelVesselAndPlaque.cs

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using Emgu.CV;
using Emgu.CV.Structure;
using System.Drawing;
using System.Linq;
using Emgu.CV.Util;
using WingAIDiagnosisService.Carotid.Utilities.CntkSeg;
using WingServerCommon.Log;
using WingAIDiagnosisService.Carotid.MathTools;
using AI.DiagSystem;
using AI.Common;
using System.IO;
using AI.Common.Log;
using Rectangle = System.Drawing.Rectangle;
using Point = System.Drawing.Point;
using RotatedRect = Emgu.CV.Structure.RotatedRect;

namespace WingAIDiagnosisService.Carotid.Utilities
{
    public class OneImageArteryContours
    {
        //图像序号
        public int SerialNumber { get; }

        //动脉轮廓
        public List<CntkDetectResult> ArteryContours { get; }

        public OneImageArteryContours(int serialNumber, List<CntkDetectResult> arteryContours)
        {
            SerialNumber = serialNumber;
            ArteryContours = arteryContours;
        }
    }

    public struct OnePlaqueContours
    {
        public int SerialNumber;
        public Rectangle ImageRect;
        public Point[] ArteryContour;
        public Point[] PlaqueContour;
        public double PlaqueArea;
        //狭窄率
        public float Ratio;
        //在原图中的斑块外界矩形
        public RotatedRect RealRect;
        //在原图中的斑块中心
        public Point RealCenterPoint;
        public float Credibility;
    }

    public struct VesselYEdge
    {
        public int AvgTop;
        public int AvgBottom;
    }

    public class ArteryImage
    {
        //图像序号
        public int SerialNumber { get; }

        //动脉轮廓
        public Point[] ArteryContour { get; }

        //动脉图像
        public Image<Gray, byte> Image { get; }

        //图像在原图中roi
        public Rectangle Rect { get; }

        //roi的中心
        public Point RectCenterPoint { get; }

        //把大图像轮廓点，转成roi图像轮廓点坐标
        private Point[] ChangeToRoiCoordinate(Point[] arteryContour, Rectangle rect)
        {
            var len = arteryContour.Length;
            var left = rect.Left;
            var top = rect.Top;
            var newArteryContour = new Point[len];
            for (var i = 0; i < len; i++)
            {
                newArteryContour[i] = new Point(arteryContour[i].X - left, arteryContour[i].Y - top);
            }
            return newArteryContour;
        }

        public ArteryImage(int serialNumber, Point[] arteryContour, Image<Gray, byte> image, Rectangle rect)
        {
            SerialNumber = serialNumber;
            ArteryContour = ChangeToRoiCoordinate(arteryContour, rect);
            Image = image;
            Rect = rect;
            RectCenterPoint = new Point(rect.Left + rect.Width / 2, rect.Top + rect.Height / 2);
        }

    }

    public class GetModelVesselAndPlaque : IDisposable
    {

        #region private variable

        private AIDiagSystem _diagSystem;
        private static GetModelVesselAndPlaque _instance;
        private bool _isInitializationSuccess = false;
        private ModelSize _modelSize = new ModelSize();
        private List<OneImageArteryContours> _allArteryContours = new List<OneImageArteryContours>();
        private List<OnePlaqueContours> _allPlaqueContours = new List<OnePlaqueContours>();
        private static readonly object _locker = new object();
        private RawImage _image = new RawImage(EnumColorType.Gray8);

        #endregion


        #region public funcs

        /// <summary>
        /// 得到一个AAAA的实例
        /// </summary>
        public static GetModelVesselAndPlaque Instance
        {
            get => _instance ?? (_instance = new GetModelVesselAndPlaque());
        }

        /// <summary>
        /// 是否已初始化
        /// </summary>
        /// <returns></returns>
        public bool InitializationStatus()
        {
            return _isInitializationSuccess;
        }

        /// <summary>
        /// 初始化
        /// </summary>
        /// <param name="modelArtery"></param>
        /// <param name="modelPlaque"></param>
        /// <returns></returns>
        public bool Initialization(EnumPerformance performance, string modelFolder)
        {
            _diagSystem = new AIDiagSystem(performance, modelFolder, EnumInferWorkName.CarotidArteryPlaque, isCropped: true);
            _diagSystem.NotifyError += AIdiagSystem_NotifyError;
            _diagSystem.NotifyLog += AIdiagSystem_NotifyLog;
            _isInitializationSuccess = true;
            return true;
        }

        private void AIdiagSystem_NotifyError(object sender, ErrorEventArgs e)
        {
            Logger.WriteLineError("AIdiagSystem_NotifyError:" + e.GetException());
        }

        private void AIdiagSystem_NotifyLog(object sender, LogEventArgs e)
        {
            if (e != null && !string.IsNullOrEmpty(e.Msg))
            {
                switch (e.LogType)
                {
                    case EnumLogType.InfoLog:
                        Logger.WriteLineInfo($"AIdiagSystem_NotifyLog:{e.Msg}");
                        break;
                    case EnumLogType.ErrorLog:
                        Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}");
                        break;
                    case EnumLogType.WarnLog:
                        Logger.WriteLineWarn($"AIdiagSystem_NotifyLog:{e.Msg}");
                        break;
                    case EnumLogType.FatalLog:
                        Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}");
                        break;
                    default:
                        Logger.WriteLineInfo(e.Msg);
                        break;
                }
            }
        }

        /// <summary>
        /// 获得所有斑块的轮廓
        /// </summary>
        /// <returns></returns>
        public List<OnePlaqueContours> GetAllPlaqueContours()
        {
            return _allPlaqueContours;
        }

        /// <summary>
        /// 获得有内膜的血管（颈动脉？）
        /// </summary>
        /// <returns></returns>
        public List<OneImageArteryContours> GetIntimaVessel()
        {
            try
            {
                lock (_locker)
                {
                    var count = _allArteryContours.Count;
                    if (count == 0)
                    {
                        return new List<OneImageArteryContours>();
                    }
                    var arteryContoursList1 = new List<OneImageArteryContours>();
                    var minIndex = 0;
                    var maxIndex = _modelSize.ModelLengthZ - 1;

                    for (var i = 0; i < count; i++)
                    {
                        if (_allArteryContours[i].SerialNumber < minIndex) continue;
                        if (_allArteryContours[i] == null)
                        {
                            continue;
                        }
                        arteryContoursList1.Add(_allArteryContours[i]);
                        if (_allArteryContours[i].SerialNumber > maxIndex) break;
                    }
                    return arteryContoursList1;
                }
            }
            catch (Exception e)
            {
                Logger.WriteLineError($"GetModelVesselAndPlaque GetIntimaVessel have an error{e}");
                return new List<OneImageArteryContours>();
            }
        }

        /// <summary>
        /// Get top and bottom edge of vessel.
        /// </summary>
        /// <returns>Vessel's y edge.</returns>
        public VesselYEdge GetVesselYEdge()
        {
            int count = 0;
            int sumTopY = 0;
            int sumBottomY = 0;
            lock (_locker)
            {
                foreach (var countour in _allArteryContours)
                {
                    if (countour.ArteryContours.Count > 0)
                    {
                        count++;
                        sumTopY += countour.ArteryContours[0].Rect.Top;
                        sumBottomY += countour.ArteryContours[0].Rect.Bottom;
                    }
                }
            }
            int avgTopY = sumTopY / count;
            int avgBottomY = sumBottomY / count;
            return new VesselYEdge { AvgTop = avgTopY, AvgBottom = avgBottomY };
        }

        /// <summary>
        /// 获得Y形的起始位置？
        /// </summary>
        /// <returns></returns>
        public List<OneImageArteryContours> GetYBeginVessel()
        {
            lock (_locker)
            {
                var count = _allArteryContours.Count;
                if (count == 0)
                {
                    return new List<OneImageArteryContours>();
                }
                var arteryContoursList = new List<OneImageArteryContours>();
                var maxIndex = _modelSize.ModelLengthZ / 4;
                for (var i = 0; i < count; ++i)
                {
                    if (_allArteryContours[i].SerialNumber < maxIndex)
                    {
                        arteryContoursList.Add(_allArteryContours[i]);
                    }
                    else
                    {
                        break;
                    }
                }
                return arteryContoursList;
            }
        }

        /// <summary>
        /// 获得Y形的终止位置？
        /// </summary>
        /// <returns></returns>
        public List<OneImageArteryContours> GetYEndVessel()
        {
            lock (_locker)
            {
                var count = _allArteryContours.Count;
                if (count == 0)
                {
                    return new List<OneImageArteryContours>();
                }
                var arteryContoursList = new List<OneImageArteryContours>();
                var minIndex = (_modelSize.ModelLengthZ * 3) / 4;
                for (var i = 0; i < count; ++i)
                {
                    if (_allArteryContours[i].SerialNumber > minIndex)
                    {
                        arteryContoursList.Add(_allArteryContours[i]);
                    }
                }
                return arteryContoursList;
            }
        }

        /// <summary>
        /// 进行检测
        /// </summary>
        /// <param name="modelSource"></param>
        /// <param name="modelSize"></param>
        /// <returns></returns>
        public bool DoDetect(byte[][] modelSource, ModelSize modelSize)
        {
            _modelSize = modelSize;
            try
            {
                var vesselList = new List<OneImageArteryContours>();
                var plaqueList = new List<OnePlaqueContours>();
                // 每隔3帧算一次
                int modelImgCount = _modelSize.ModelLengthZ;
                int interval = 3;
                for (int ni = 0; ni < modelImgCount; ni += interval)
                {
                    // 复制出待测图像
                    _image.CopyFrom(modelSource[ni], _modelSize.ModelLengthX, _modelSize.ModelLengthY, 0);
                    // 输入待测图像进行检测
                    var diagResult = _diagSystem.EvaluateOneImage(_image);
                    if (diagResult == null)
                    {
                        continue;
                    }
                    // 将当前图像上检测到的颈动脉血管找出来
                    List<CntkDetectResult> detectedArtery = new List<CntkDetectResult>();
                    foreach (var organ in diagResult.DiagResultsForEachOrgan)
                    {
                        // 只取每一个颈动脉OrganContours里的第一条轮廓
                        // 注意：如有多个颈动脉，则会有多个organ=EnumOrgans.CarotidArtery的结果，每个organ里还是只有一条轮廓
                        // （目前一个目标有多条轮廓只适用于圆环形心肌这种情况，这时心肌需要用内外两条线才能完整表示一个目标）
                        if (organ.Organ == EnumOrgans.CarotidArtery && organ.OrganContours?.Length == 1)
                        {
                            // 目前AIDiagSystem中未返回organ的置信度，因此这里暂时全设为0.8 (该值不影响后续的流程)
                            // 斑块和颈动脉 好像都是CntkLabelType.LabelOne
                            var oneArtery = GenCntkDetectResult(0.8f, CntkLabelType.LabelOne, organ.OrganBoundBox,
                                organ.OrganContours[0]);
                            // 将当前轮廓添加到detectedArtery里
                            detectedArtery.Add(oneArtery);

                            // 将当前血管上找到的斑块 转成所需的格式 放到plaqueList里
                            foreach (var detectedObject in organ.DetectedObjects)
                            {
                                if (detectedObject.Contours.Length > 0)
                                {
                                    var onePlaque = GenCntkDetectResult(detectedObject.Confidence, CntkLabelType.LabelOne,
                                        detectedObject.BoundingBox, detectedObject.Contours[0]);
                                    OnePlaqueContours onePlaqueContours = new OnePlaqueContours();
                                    onePlaqueContours.ArteryContour = oneArtery.Contour;
                                    onePlaqueContours.ImageRect = GetExtendRect(oneArtery.Rect, new Size(_image.Width, _image.Height));
                                    onePlaqueContours.PlaqueContour = onePlaque.Contour;
                                    onePlaqueContours.Credibility = onePlaque.Credibility;
                                    onePlaqueContours.SerialNumber = ni;
                                    onePlaqueContours.PlaqueArea = GetArea(onePlaque.Contour);
                                    plaqueList.Add(onePlaqueContours);
                                }
                            }
                        }
                    }
                    // 将有动脉的那些帧的序号和对应的动脉轮廓，存到vesselList里
                    if (detectedArtery.Count > 0)
                    {
                        vesselList.Add(new OneImageArteryContours(ni, detectedArtery));
                    }
                }
                // 去除从异常位置的颈动脉血管上找到的斑块
                var selectedPlaqueList = RemoveAbnormalImage(vesselList, plaqueList);

                // 更新_allArteryContours 和 _allPlaqueContours
                lock (_locker)
                {
                    _allArteryContours = vesselList;
                    if (_allArteryContours.Count == 0)
                    {
                        Logger.WriteLineWarn($"GetModelVesselAndPlaque DetectArtery  get zero artery contours.");
                    }
                    _allPlaqueContours = selectedPlaqueList;
                }
                return true;
            }
            catch (Exception e)
            {
                Logger.WriteLineError($"GetModelVesselAndPlaque DoDetect  have an error{e}");
                return false;
            }
        }

        /// <summary>
        /// 销毁
        /// </summary>
        public void Dispose()
        {
            _diagSystem.Dispose();
            _diagSystem = null;
            _instance = null;
        }

        #endregion


        #region private funcs

        private CntkDetectResult GenCntkDetectResult(float confidence, CntkLabelType label, Rect boundingBox, Point2D[] contour)
        {
            CntkDetectResult result = new CntkDetectResult();
            result.Credibility = confidence;
            result.LabelType = label;
            result.Rect = new Rectangle(boundingBox.Left, boundingBox.Top,
                boundingBox.Width, boundingBox.Height);
            int contourLen = contour.Length;
            Point[] contourDst = new Point[contourLen];
            for (int ki = 0; ki < contourLen; ki++)
            {
                contourDst[ki].X = contour[ki].X;
                contourDst[ki].Y = contour[ki].Y;
            }
            result.Contour = contourDst;
            // 轮廓面积
            result.ContourArea = (float)CvInvoke.ContourArea(new VectorOfPoint(contourDst));
            return result;
        }

        private Rectangle GetExtendRect(Rectangle rect1, Size imageSize)
        {
            var change = 20;
            var left = Math.Max(0, rect1.Left - change);
            var top = Math.Max(0, rect1.Top - change);
            var endW = Math.Min(left + rect1.Width + 2 * change, imageSize.Width - 1);
            var endH = Math.Min(top + rect1.Height + 2 * change, imageSize.Height - 1);
            return new Rectangle(left, top, endW - left, endH - top);
        }

        private double GetArea(Point[] points)
        {
            var vector = new VectorOfPoint(points);
            return CvInvoke.ContourArea(vector);
        }

        private List<OnePlaqueContours> RemoveAbnormalImage(List<OneImageArteryContours> vesselList, List<OnePlaqueContours> plaqueList)
        {
            if (plaqueList.Count <= 0)
            {
                return plaqueList;
            }
            // 数量太少的不好拟合直线，直接返回
            var count = vesselList.Count;
            if (count < 10)
            {
                return plaqueList;
            }
            // 将所有血管的中心点取出来
            List<Point> points = new List<Point>();
            for (var i = 0; i < count; ++i)
            {
                // 一幅图上有多个血管，则取多个血管外边框的最左最右最上最下点来计算中心
                int left = 0, top = 0, right = 0, bottom = 0;
                var arterys = vesselList[i].ArteryContours;
                for (int ni = 0; ni < arterys.Count; ni++)
                {
                    if (ni == 0)
                    {
                        left = arterys[ni].Rect.Left;
                        top = arterys[ni].Rect.Top;
                        right = arterys[ni].Rect.Right;
                        bottom = arterys[ni].Rect.Bottom;
                    }
                    else
                    {
                        left = Math.Min(left, arterys[ni].Rect.Left);
                        top = Math.Min(top, arterys[ni].Rect.Top);
                        right = Math.Max(right, arterys[ni].Rect.Right);
                        bottom = Math.Max(bottom, arterys[ni].Rect.Bottom);
                    }
                }
                var x = (left + right) / 2;
                var y = (top + bottom) / 2;
                points.Add(new Point(x, y));
            }

            // 将这些点拟合成一条线
            var line = MathTools2D.CalculateFittedLine(points);
            var pointDiff = new LineFittingError[count];

            //计算每个点的差值
            double sumDiff = 0;
            for (var i = 0; i < count; ++i)
            {
                //var newX = line[0] * points[i].X + line[2];
                double diff = MathTools2D.GetDistaceBetweenPointAndLine(line, points[i]);
                sumDiff += diff;
                pointDiff[i] = new LineFittingError()
                {
                    SerialNumber = i,
                    OriginalPoint = points[i],
                    DiffValue = diff
                };
            }
            //计算差值的，平均值和标准差
            var averageDiff = sumDiff / count;
            double sumStd = 0;
            for (var i = 0; i < count; ++i)
            {
                var diff = pointDiff[i].DiffValue;
                sumStd += (diff - averageDiff) * (diff - averageDiff);
            }
            sumStd = Math.Sqrt(sumStd / count);

            var minDiff = averageDiff - 3 * sumStd;
            var maxDiff = averageDiff + 3 * sumStd;
            //删除差值大于平均值加减3倍的标准差的点
            List<int> removeFlag = new List<int>();
            for (var i = 0; i < count; ++i)
            {
                var diff = pointDiff[i].DiffValue;
                if (diff < minDiff || diff > maxDiff)
                {
                    int serialNumber = vesselList[i].SerialNumber;
                    if (!removeFlag.Contains(serialNumber))
                    {
                        removeFlag.Add(serialNumber);
                    }
                    pointDiff[i].DiffValue = -100000;
                }
            }
            //去掉5%的数据
            var count1 = count / 20;
            var diffList = pointDiff.ToList().OrderByDescending(o => o.DiffValue);
            var diffList1 = diffList.Skip(count1).ToList();
            var maxDiffValue = diffList1[0].DiffValue;
            for (var i = 0; i < count; ++i)
            {
                if (pointDiff[i].DiffValue > maxDiffValue)
                {
                    int serialNumber = vesselList[i].SerialNumber;
                    if (!removeFlag.Contains(serialNumber))
                    {
                        removeFlag.Add(serialNumber);
                    }
                }
            }
            int plaqueCount = plaqueList.Count;
            for (var i = plaqueCount - 1; i >= 0; i--)
            {
                var serialNumber = plaqueList[i].SerialNumber;
                if (removeFlag.Contains(serialNumber))
                {
                    plaqueList.RemoveAt(i);
                }
            }
            return plaqueList;
        }


        #endregion

    }
}