Algorithm-Repo/SourceCode/CVAlgorithmUtils/YOLOInstanceSegPostProcessUtils/YOLOInstanceSegDemo/MainWindow.xaml.cs

using System;
using System.Windows;
using System.Windows.Threading;
using System.Windows.Media.Imaging;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Threading.Tasks;
using System.Drawing;
using System.Drawing.Imaging;
using Microsoft.Win32;
using System.Threading;
using System.Diagnostics;
using Accord.Video.FFMPEG;
using System.IO;
using System.Runtime.InteropServices;
using Accord.Video.DirectShow;
using Accord.Video;
using System.Windows.Media;
using PixelFormat = System.Drawing.Imaging.PixelFormat;
using Color = System.Drawing.Color;
using AI.Common.Implements;
using AI.Common.Interface;
using YOLOInstanceSegProcessLib;
using Rect = AI.Common.Interface.Rect;

namespace YOLOInstanceSegDemo
{
    /// <summary>
    /// MainWindow.xaml 的交互逻辑
    /// </summary>
    public partial class MainWindow : Window
    {
        private IInferenceNetwork _inferNet = null;

        private BitmapImage _origImg = null;
        private BitmapImage _dstImg = null;
        private int _doneImageNum = 0;
        private volatile bool _isCropped = false;
        private Stopwatch _stopWatch = new Stopwatch();
        private LesionSegImg _LesionDetectSeg;
        int coreCount = Environment.ProcessorCount * 2;
        int numCPU;
        private volatile bool _enableAI = true;
        private volatile bool _continuouslyDetecting = false;

        List<DetectedObject> inferResult = new List<DetectedObject>();
        int oriWidth = 0;
        int oriHeight = 0;
        private readonly object _videoReaderLocker = new object();
        private VideoFileReader _videoReader;
        private volatile int _displayCount = 0;
        private ManualResetEvent _playEvent = new ManualResetEvent(false);
        private VideoCaptureDevice _videoCapture;

        /// <summary>
        /// 用于RawImage和Bitmap之间进行转换的帮助类
        /// </summary>
        public static class RawImageShowUtils
        {
            /// <summary>
            /// 将System.Drawimg.Bitmap转为RawImage
            /// </summary>
            /// <param name="image"></param>
            /// <param name="keepChannel"></param>
            /// <returns></returns>
            public static RawImage BitmapToRawImage(Bitmap image, bool keepColorType = false)
            {
                int width = image.Width;
                int height = image.Height;
                PixelFormat pixelFormat = image.PixelFormat;
                if (pixelFormat != PixelFormat.Format24bppRgb && pixelFormat != PixelFormat.Format32bppArgb
                    && pixelFormat != PixelFormat.Format8bppIndexed && pixelFormat != PixelFormat.Format16bppGrayScale)
                {
                    throw new Exception("Unexpected image pixel format:" + pixelFormat);
                }
                EnumColorType origColorType = EnumColorType.Gray8;
                int origBytesPerPixel = 1;
                if (pixelFormat == PixelFormat.Format8bppIndexed)
                {
                    origColorType = EnumColorType.Gray8;
                    origBytesPerPixel = 1;
                }
                else if (pixelFormat == PixelFormat.Format16bppGrayScale)
                {
                    origColorType = EnumColorType.Gray16;
                    origBytesPerPixel = 2;
                }
                else if (pixelFormat == PixelFormat.Format24bppRgb)
                {
                    // 注意Bitmap里的24bppRgb其实通道顺序是BGR
                    origColorType = EnumColorType.Bgr;
                    origBytesPerPixel = 3;
                }
                else
                {
                    // 注意Bitmap里的32Argb其实通道顺序是Bgra
                    origColorType = EnumColorType.Bgra;
                    origBytesPerPixel = 4;
                }
                EnumColorType dstColorType;
                int dstBytesPerPixel;
                if (keepColorType)
                {
                    dstColorType = origColorType;
                    dstBytesPerPixel = origBytesPerPixel;
                }
                else
                {
                    // 如果没有指定要keepColorType的话，默认都转成BGR
                    dstColorType = EnumColorType.Bgr;
                    dstBytesPerPixel = 3;
                }
                var bmData = image.LockBits(new System.Drawing.Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, pixelFormat);
                byte[] dstDataArray = new byte[width * height * dstBytesPerPixel];
                unsafe
                {
                    int stride = bmData.Stride;
                    int dstStride = width * dstBytesPerPixel;
                    IntPtr ptr = bmData.Scan0;
                    IntPtr ptrH, ptrW;
                    // 原图和目标图的ColorType相同
                    if (origColorType == dstColorType)
                    {
                        for (int nh = 0; nh < height; nh++)
                        {
                            ptrH = IntPtr.Add(ptr, nh * stride);
                            Marshal.Copy(ptrH, dstDataArray, nh * dstStride, dstStride);
                        }
                    }
                    // 原图和目标图的ColorType不同
                    else
                    {
                        // 如果原图是8位灰度图
                        if (origColorType == EnumColorType.Gray8)
                        {
                            if (dstColorType == EnumColorType.Gray16)
                            {
                                byte gray;
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        gray = Marshal.ReadByte(ptrW);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        dstDataArray[dstOffset] = gray;
                                        // 高位为0，因此不用赋值
                                    }
                                }
                            }
                            else
                            {
                                byte gray;
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        gray = Marshal.ReadByte(ptrW);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        dstDataArray[dstOffset] = gray;
                                        dstDataArray[dstOffset + 1] = gray;
                                        dstDataArray[dstOffset + 2] = gray;
                                    }
                                }
                                if (dstColorType == EnumColorType.Bgra)
                                {
                                    for (int nh = 0; nh < height; nh++)
                                    {
                                        for (int nw = 0; nw < width; nw++)
                                        {
                                            dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                            dstDataArray[dstOffset + 3] = 255;
                                        }
                                    }
                                }
                            }
                        }
                        // 如果原图是BGR彩色图
                        if (origColorType == EnumColorType.Bgr)
                        {
                            if (dstColorType == EnumColorType.Gray8 || dstColorType == EnumColorType.Gray16)
                            {
                                byte red, green, blue, gray;
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        blue = Marshal.ReadByte(ptrW);
                                        ptrW = IntPtr.Add(ptrW, 1);
                                        green = Marshal.ReadByte(ptrW);
                                        ptrW = IntPtr.Add(ptrW, 1);
                                        red = Marshal.ReadByte(ptrW);
                                        gray = RawImage.PixelRGBToGray(red, green, blue);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        dstDataArray[dstOffset] = gray;
                                    }
                                }
                            }
                            if (dstColorType == EnumColorType.Bgra)
                            {
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        Marshal.Copy(ptrW, dstDataArray, dstOffset, 3);
                                        dstDataArray[dstOffset + 3] = 255;
                                    }
                                }
                            }
                        }
                        // 如果原图是BGRA彩色图
                        if (origColorType == EnumColorType.Bgra)
                        {
                            if (dstColorType == EnumColorType.Gray8 || dstColorType == EnumColorType.Gray16)
                            {
                                byte red, green, blue, gray;
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        blue = Marshal.ReadByte(ptrW);
                                        ptrW = IntPtr.Add(ptrW, 1);
                                        green = Marshal.ReadByte(ptrW);
                                        ptrW = IntPtr.Add(ptrW, 1);
                                        red = Marshal.ReadByte(ptrW);
                                        gray = RawImage.PixelRGBToGray(red, green, blue);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        dstDataArray[dstOffset] = gray;
                                    }
                                }
                            }
                            if (dstColorType == EnumColorType.Bgr)
                            {
                                int dstOffset;
                                for (int nh = 0; nh < height; nh++)
                                {
                                    ptrH = IntPtr.Add(ptr, nh * stride);
                                    for (int nw = 0; nw < width; nw++)
                                    {
                                        ptrW = IntPtr.Add(ptrH, nw * origBytesPerPixel);
                                        dstOffset = nh * dstStride + nw * dstBytesPerPixel;
                                        Marshal.Copy(ptrW, dstDataArray, dstOffset, 3);
                                    }
                                }
                            }
                        }
                    }
                }
                image.UnlockBits(bmData);
                return new RawImage(dstDataArray, width, height, dstColorType);
            }

            /// <summary>
            /// 将RawImage转为System.Drawing.Bitmap
            /// </summary>
            /// <param name="image"></param>
            /// <returns></returns>
            public static Bitmap RawImageToBitmap(RawImage image)
            {
                int width = image.Width;
                int height = image.Height;
                EnumColorType colorType = image.ColorType;
                PixelFormat pixelFormat;
                EnumColorType dstColorType;
                IRawImage clonedImage;
                switch (colorType)
                {
                    case EnumColorType.Gray8:
                        clonedImage = image;
                        pixelFormat = PixelFormat.Format8bppIndexed;
                        dstColorType = EnumColorType.Gray8;
                        break;
                    case EnumColorType.Gray16:
                        clonedImage = image;
                        pixelFormat = PixelFormat.Format16bppGrayScale;
                        dstColorType = EnumColorType.Gray16;
                        break;
                    case EnumColorType.GrayF32:
                        clonedImage = image.Clone(EnumColorType.Gray8);
                        pixelFormat = PixelFormat.Format8bppIndexed;
                        dstColorType = EnumColorType.Gray8;
                        break;
                    case EnumColorType.Bgr:
                        clonedImage = image;
                        pixelFormat = PixelFormat.Format24bppRgb;
                        dstColorType = EnumColorType.Bgr;
                        break;
                    case EnumColorType.Rgb:
                        clonedImage = image.Clone(EnumColorType.Bgr);
                        pixelFormat = PixelFormat.Format24bppRgb;
                        dstColorType = EnumColorType.Bgr;
                        break;
                    case EnumColorType.Bgra:
                        clonedImage = image;
                        pixelFormat = PixelFormat.Format32bppArgb;
                        dstColorType = EnumColorType.Bgra;
                        break;
                    case EnumColorType.Rgba:
                        clonedImage = image.Clone(EnumColorType.Bgra);
                        pixelFormat = PixelFormat.Format32bppArgb;
                        dstColorType = EnumColorType.Bgra;
                        break;
                    default:
                        throw new ArgumentOutOfRangeException("Unexpected image.colorType.");
                }
                Bitmap dstBmp = new Bitmap(width, height, pixelFormat);
                var bmData = dstBmp.LockBits(new System.Drawing.Rectangle(0, 0, width, height), ImageLockMode.ReadWrite, pixelFormat);
                unsafe
                {
                    int stride = bmData.Stride;
                    IntPtr ptr = bmData.Scan0;
                    byte[] dataBuffer = clonedImage.DataBuffer;
                    int bytePerPixel = clonedImage.BytesPerPixel;
                    if (stride == width * bytePerPixel)
                    {
                        Marshal.Copy(dataBuffer, 0, ptr, width * height * bytePerPixel);
                    }
                    else
                    {
                        IntPtr ptrDst;
                        for (int ni = 0; ni < height; ni++)
                        {
                            ptrDst = IntPtr.Add(ptr, ni * stride);
                            Marshal.Copy(dataBuffer, ni * width * bytePerPixel, ptrDst, width * bytePerPixel);
                        }
                    }
                }
                dstBmp.UnlockBits(bmData);
                // 灰度图像需要写调色板
                if (dstColorType == EnumColorType.Gray8)
                {
                    var palette = dstBmp.Palette;
                    for (int ni = 0; ni < 256; ni++)
                    {
                        palette.Entries[ni] = Color.FromArgb(ni, ni, ni);
                    }
                    dstBmp.Palette = palette;
                }
                return dstBmp;
            }

            /// <summary>
            /// 将Bitmap转换为BitmapImage
            /// </summary>
            /// <param name="img"></param>
            /// <returns></returns>
            public static BitmapImage BitmapToBitmapImage(Bitmap img)
            {
                BitmapImage bmpimg = new BitmapImage();
                using (System.IO.MemoryStream ms = new System.IO.MemoryStream())
                {
                    img.Save(ms, ImageFormat.Png);
                    bmpimg.BeginInit();
                    bmpimg.StreamSource = ms;
                    bmpimg.CacheOption = BitmapCacheOption.OnLoad;
                    bmpimg.EndInit();
                    bmpimg.Freeze();
                    ms.Dispose();
                }
                return bmpimg;
            }

        }

        public MainWindow()
        {
            InitializeComponent();
            try
            {
                numCPU = Convert.ToInt32(TextBoxCPU.Text);
                if (numCPU <= 0 || numCPU > coreCount)
                    throw new Exception("CPU数目设置错误");

            }
            catch (Exception excep)
            {
                MessageBox.Show("CPU数目设置错误，只能为正整数且小于本机CPU线程总数！");
                return;
            }

            try
            {
                string netDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Networks");
                _LesionDetectSeg = new LesionSegImg(numCPU, netDir);
            }

            catch (Exception excep)
            {
                MessageBox.Show("doMyocardialSegment失败！");
                return;
            }
        }


        private void OnBtnTestClicked(object sender, RoutedEventArgs e)
        {
            StopPlay();
            try
            {
                OpenFileDialog openFileDialog = new OpenFileDialog
                {
                    Filter = "图片文件|*.png;*.bmp;*.jpg;*.jpeg;*.avi;*.mp4",
                    Multiselect = false,
                    Title = "选择一幅或多幅图",
                };
                if (openFileDialog.ShowDialog() == true)
                {
                    var fileName = openFileDialog.FileName;
                    var fileExtension = Path.GetExtension(fileName);
                    if (fileExtension == ".png" || fileExtension == ".jpg" || fileExtension == ".bmp" || fileExtension == ".jpeg")
                    {
                        var bitmap = new Bitmap(fileName);

                        if (_enableAI)
                        {
                            DoImgTest(bitmap);
                        }
                        MyImageCanvas.UpdateImage(bitmap);
                        bitmap.Dispose();
                    }
                    else
                    {
                        long frameCount;
                        lock (_videoReaderLocker)
                        {
                            _videoReader = new VideoFileReader();
                            _videoReader.Open(fileName);
                            frameCount = _videoReader.FrameCount;
                        }
                        _displayCount = 0;
                        _playEvent.Reset();
                        Task.Run(() =>
                        {
                            _continuouslyDetecting = true;
                            bool breakFlag = false;
                            while (!_playEvent.WaitOne(1) && !breakFlag)
                            {
                                for (var ni = 0; ni < frameCount; ni++)
                                {
                                    if (_playEvent.WaitOne(1))
                                    {
                                        breakFlag = true;
                                        break;
                                    }
                                    Bitmap bitmap;
                                    lock (_videoReaderLocker)
                                    {
                                        bitmap = _videoReader.ReadVideoFrame(ni);
                                    }
                                    _displayCount++;
                                    //Console.WriteLine(_displayCount); 
                                    // Thread.Sleep(1000);
                                    if (_enableAI)
                                    {
                                        DoImgTest(bitmap);
                                    }
                                    // 更新显示图像
                                    MyImageCanvas.UpdateImage(bitmap);
                                    bitmap.Dispose();

                                    if (_playEvent.WaitOne(0))
                                    {
                                        breakFlag = true;
                                        break;
                                    }
                                }
                            }
                            _continuouslyDetecting = false;
                        });


                        VideoFileReader videoReader = new VideoFileReader();
                        videoReader.Open(fileName);
                        int totalCount = (int)videoReader.FrameCount;
                        videoReader.Close();
                        for (int ni = 0; ni < totalCount; ni++)
                        {
                            videoReader.Open(fileName);

                            videoReader.Close();
                        }
                        videoReader.Dispose();
                    }


                }
            }
            catch (Exception excep)
            {
                MessageBox.Show("加载测试图片时出错：" + excep);
            }
        }

        private void DoImgTest(Bitmap bmp)
        {
            try
            {
                RawImage rawImg = RawImageShowUtils.BitmapToRawImage(bmp);

                _stopWatch.Restart();

                inferResult = _LesionDetectSeg.ProcessLesionSegImg(rawImg, _isCropped);

                // inferResult = new List<DetectedObject>();
                long inferTime = _stopWatch.ElapsedMilliseconds;
                _stopWatch.Stop();
                Dispatcher.BeginInvoke(DispatcherPriority.Normal, new Action(() =>
                {
                    ForwardTime.Text = inferTime.ToString();
                }));
                using (var g = Graphics.FromImage(bmp))
                {
                    System.Drawing.Brush brush = System.Drawing.Brushes.Red;

                    if (inferResult.Count > 0)
                    {
                        int len = inferResult.Count;
                        for (int nj = 0; nj < len; nj++)
                        {
                            string labels = inferResult[nj].Label.Name;

                            Rect box = (Rect)inferResult[nj].BoundingBox;
                           
                            g.DrawString(labels + ":" + inferResult[nj].Score.ToString("0.0" + "%"), new Font("Arial", 10),
                                brush, box.Left + 10, box.Top + 10);

                            System.Drawing.Rectangle rect2 = new System.Drawing.Rectangle(box.Left,
                                box.Top, box.Width, box.Height);
                            g.DrawRectangle(new System.Drawing.Pen(brush, 2), rect2);

                            IContour contour = inferResult[nj].Contour;

                            if (contour == null)
                            {
                                continue;
                            }
                            int count = contour.Contours.Length;

                            for (int ni = 0; ni < count; ni++)
                            {                               
                                var contourPoints = contour.Contours[ni];
                                int pointCount = contourPoints.Points.Length;

                                System.Drawing.Point[] contourPointsDraw = new System.Drawing.Point[pointCount];  // Use an array of Point

                                for (int j = 0; j < pointCount; j++)
                                {
                                    var point = new System.Drawing.Point(contourPoints.Points[j].X, contourPoints.Points[j].Y);
                                    contourPointsDraw[j] = point;
                                }

                                // Create a GraphicsPath object for drawing the contour
                                System.Drawing.Drawing2D.GraphicsPath contourPath = new System.Drawing.Drawing2D.GraphicsPath();
                                contourPath.AddPolygon(contourPointsDraw);

                                // Draw the contour using the GraphicsPath
                                g.DrawPath(new System.Drawing.Pen(brush, 2), contourPath);
                            }



                        }

                    }
                }
                _doneImageNum += 1;
            }

            catch (Exception excep)
            {
                MessageBox.Show("测试图像时出错：" + excep);
            }
        }


        private void OnUncheckedEnableAI(object sender, RoutedEventArgs e)
        {
            _enableAI = false;
            MyImageCanvas.SetShowParams(_enableAI);
        }

        private void OnCheckedEnableAI(object sender, RoutedEventArgs e)
        {
            _enableAI = true;
            MyImageCanvas.SetShowParams(_enableAI);
            if (!_continuouslyDetecting && MyImageCanvas.Image != null)
            {
                DoImgTest(MyImageCanvas.Image);
            }
        }

        private void StopPlay()
        {
            _playEvent.Set();
            lock (_videoReaderLocker)
            {
                if (_videoReader != null)
                {
                    _videoReader.Close();
                    _videoReader.Dispose();
                    _videoReader = null;
                }
            }
            if (_videoCapture != null)
            {
                _videoCapture.NewFrame -= OnVideoCaptureNewFrame;
                _videoCapture.SignalToStop();
                _videoCapture = null;
            }
            _continuouslyDetecting = false;
        }

        private void OnVideoCaptureNewFrame(object sender, NewFrameEventArgs e)
        {
            var bitmap = e.Frame;
            // 更新显示图像
            MyImageCanvas.UpdateImage(bitmap);
            _displayCount++;
            if (_enableAI)
            {
                DoImgTest(bitmap);
            }
            MyImageCanvas.UpdateImage(bitmap);
            bitmap.Dispose();
        }

        private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
        {
            _inferNet?.Dispose();
        }

        private void OnCheckedIsCropped(object sender, RoutedEventArgs e)
        {
            _isCropped = true;
        }

        private void OnUncheckedIsCropped(object sender, RoutedEventArgs e)
        {
            _isCropped = false;
        }
    }
}