using WingServerCommon.Service; using WingInterfaceLibrary.Interface; using WingInterfaceLibrary.Request; using WingInterfaceLibrary.Enum; using WingServerCommon.Config; using WingServerCommon.Config.Parameters; using WingAIDiagnosisService.Carotid.Utilities; using Vinno.vCloud.Common.Vid2; using WingAIDiagnosisService.Manage; using WingAIDiagnosisService.Carotid; using Newtonsoft.Json; using AI.DiagSystem; using WingServerCommon.Log; using System.IO; using System; using System.Threading.Tasks; using System.Collections.Generic; using System.Linq; using AI.Common; using System.Threading; using AI.Common.Log; using SkiaSharp; using System.Net.Http; using System.Net; using WingInterfaceLibrary.Request.Storage; using WingInterfaceLibrary.Result.AIDiagnosis; using WingInterfaceLibrary.Request.AIDiagnosis; using WingInterfaceLibrary.DTO.Record; using WingInterfaceLibrary.DTO.Comment; using Emgu.CV; using Emgu.CV.CvEnum; using Emgu.CV.Structure; using System.Net.Http.Headers; using WingAIDiagnosisService.Common; using WingInterfaceLibrary.Interface.DBInterface; using WingInterfaceLibrary.Request.DBRequest; using WingInterfaceLibrary.DTO.DiagnosisResult; using WingInterfaceLibrary.Request.RemedicalAISelected; using ContourModifyUtils; namespace WingAIDiagnosisService.Service { /// /// AI诊断服务 /// public partial class AIDiagnosisService : JsonRpcService, IAIDiagnosisService { private EnumPerformance _workerLevel = EnumPerformance.Medium; private int _batchImageSize = 16; private int _sleepTime = 400; private readonly string _tempFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "DiagnosisTemp"); private AIDiagSystem _diagSystem; private AILesionCompareUtils _lesionCompareUtil; private RecognizeCarotidType _recognizeCarotidType = new RecognizeCarotidType(); //识别左右颈是否初始化成功 private static bool _recognizeCarotidTypeCanBeUse = false; static HttpClient _httpClient = new HttpClient(); private IAuthenticationService _authenticationService; private IStorageService _storageService; private readonly string _carotidName = "CarotidPlaqueDetect"; private int _contourInterval = 10; private IDiagnosisResultDBService _diagnosisResultDBService; /// /// Init service /// public override void Load(JsonRpcClientPool jsonRpcClientPool) { base.Load(jsonRpcClientPool); _authenticationService = GetProxy(); _storageService = GetProxy(); _workerLevel = (EnumPerformance)ConfigurationManager.GetParammeter("AI", "WorkerLevel").Value; _batchImageSize = ConfigurationManager.GetParammeter("AI", "BatchImageSize").Value; _contourInterval = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value; _httpClient.Timeout = TimeSpan.FromMinutes(15); _diagnosisResultDBService = GetProxy(); InitAISystem(); //初始化识别左右颈 _recognizeCarotidTypeCanBeUse = _recognizeCarotidType.Initialization(); var keyPointDistanceThreshold = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value; var dragStartPointCatchDistanceThreshold = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value; ContourModifyHelper.KeyPointDistanceThreshold = keyPointDistanceThreshold; ContourModifyHelper.DragStartPointCatchDistanceThreshold = dragStartPointCatchDistanceThreshold; } private void InitAISystem() { var modeFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AIDiagnosis\\Networks"); _diagSystem = new AIDiagSystem(_workerLevel, modeFilePath, EnumInferWorkName.Default, false, true, true); if (_diagSystem != null) { _diagSystem.NotifyError += AIdiagSystem_NotifyError; _diagSystem.NotifyLog += AIdiagSystem_NotifyLog; } _lesionCompareUtil = new AILesionCompareUtils(); } /// /// 查询杏聆荟支持的AI模块 /// /// 查询杏聆荟支持的AI模块请求实体 /// /// false public async Task> FindDiagnosisModulesAsync(TokenRequest request) { try { var moduleNames = AIDiagSystem.GetValidModuleNamesForVclound() ?? new List(); var resultData = moduleNames.Select(x => x.ToString()).ToList(); if (!resultData.Contains(_carotidName)) { resultData.Remove("CarotidArtery"); resultData.Add(_carotidName); } return await Task.FromResult(resultData.Distinct().ToList()); } catch (Exception) { return new List(); } } /// /// 图像诊断 /// /// 图像诊断请求实体 /// 图像诊断结果 /// false public async Task DiagnosisImageAsync(DiagnosisImageRequest request) { try { var relationCode = request.RelationCode; var fileUrl = request.FileToken; var localFile = ""; if (!string.IsNullOrWhiteSpace(request.DiskPath)) { localFile = request.DiskPath; } else { localFile = await DownloadAsync(fileUrl); } if (File.Exists(localFile)) { if (_diagSystem == null) { InitAISystem(); } using (var imageData = new VinnoImageData(localFile, OperationMode.Open)) { if (DiagnosisHelper.IsCarotid(imageData)) { if (imageData.ImageCount > DiagnosisHelper.CarotidMinImageCounts) { //颈动脉 var result = await CarotidDiagnosis(imageData); var resultData = new DiagnosisImageResult() { DiagnosisOrgans = new List() { DiagnosisOrganEnum.CarotidArtery }, CarotidResult = result }; return resultData; } } else { //乳腺、肝脏 var results = NormalDiagnosis(imageData); //AI实体有很多参数仅支持get,为了方便后续反序列化,db入库前先做个转换 var diagnosisPerImages = results.Select(x => new AIDiagnosisPerImageModel(x.Key, x.Value, 1)).ToList(); await AddDiagnosisResultInfosAsync(relationCode, fileUrl, diagnosisPerImages); //轮廓线分隔处理 diagnosisPerImages = results.Select(x => new AIDiagnosisPerImageModel(x.Key, x.Value, _contourInterval)).ToList(); var diagnosisResult = Newtonsoft.Json.JsonConvert.DeserializeObject>(Newtonsoft.Json.JsonConvert.SerializeObject(diagnosisPerImages)); var resultData = new DiagnosisImageResult() { DiagnosisConclusion = (DiagnosisConclusionEnum)GetDiagnosisConclusion(diagnosisPerImages), DiagnosisResult = diagnosisResult, DiagnosisOrgans = GetDiagnosisOrgans(diagnosisResult), }; return resultData; } } } } catch (Exception ex) { Logger.WriteLineWarn($"DiagnosisImageAsync err {ex}"); } return new DiagnosisImageResult(); } /// /// 生成AI报告 /// /// 生成AI报告请求实体 /// /// false public async Task DiagnosisReportAsync(DiagnosisReportRequest request) { try { if (request.Organ == DiagnosisOrganEnum.CarotidArtery) { var result = await GetCarotidAIMeasureResult(request); return result; } else { var manager = await CreateDiagnosisManagerAsync(request); var reportPerImages = manager.GetReportResults(); var diagnosisConclusion = GetDiagnosisConclusion(reportPerImages); var diagnosisResult = new List(); foreach (var item in reportPerImages) { var aiFileToken = await UploadFileAsync(item.AILocalImagePath, Path.GetFileName(item.AILocalImagePath)); var perImageResult = Newtonsoft.Json.JsonConvert.DeserializeObject(Newtonsoft.Json.JsonConvert.SerializeObject(item)); perImageResult.RemedicalCode = item.RemedicalCode; perImageResult.DataType = (RemedicalFileDataTypeEnum)item.DataType; perImageResult.Pixel = item.Pixel; perImageResult.AIFileToken = aiFileToken; diagnosisResult.Add(perImageResult); } return new DiagnosisReportResult { DiagnosisConclusion = (DiagnosisConclusionEnum)diagnosisConclusion, DiagnosisResult = diagnosisResult, }; } } catch (Exception ex) { Logger.WriteLineWarn($"AIService DiagnosisReport err, {ex}"); } return new DiagnosisReportResult(); } /// /// 查询AI相关枚举集合 /// /// 查询AI相关枚举集合请求实体 /// AI相关枚举集合 /// /// public async Task GetDiagnosisEnumItemsAsync(GetDiagnosisEnumItemsRequest request) { var diagnosisItems = new List(); //病灶 diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "Breast", new List { "BIRads1" })); diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "Liver", new List { "BIRads1" }, prefix: "Liver")); diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "Thyroid", new List { "TIRADS0" })); //病灶特性描述 diagnosisItems.Add(GetEnumItem(typeof(EnumDesShapeValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesOrientationValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesEchoPatternValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesLesionBoundaryValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesMarginValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumCalcificationsValue), "Calcification")); diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverShapeValue), prefix: "LiveShape")); diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverBoundaryValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverEchoTextureValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidEchoPatternValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidShapeValue))); diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidMarginValue), prefix: "Thyroid")); diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidEchogenicFociValue))); //局灶和弥漫区分 diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "BreastLocalLesion", includeFields: new List { "Lipomyoma", "BIRads2", "BIRads3" })); diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "BreastDiffuseLesion", includeFields: new List { "BIRads4A", "BIRads4B", "BIRads4C", "BIRads5" })); diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "LiverLocalLesion", includeFields: new List { "Hyperechoic", "HHE", "CYST", "PossibleCancer" }, prefix: "Liver")); diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "LiverDiffuseLesion", includeFields: new List { "FattyLiver", "DiffuseLesions", "Cirrhosis", "PCLD" }, prefix: "Liver")); diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "ThyroidLocalLesion", includeFields: new List { "TIRADS2", "TIRADS3", "TIRADS4a", "TIRADS4b", "TIRADS4c", "TIRADS5" })); diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "ThyroidDiffuseLesion", includeFields: new List { "DiffuseDisease" })); var resultData = new GetDiagnosisEnumItemsResult { Source = diagnosisItems }; return await Task.FromResult(resultData); } private EnumItemDTO GetEnumItem(Type enumType, string keyCode = "", List excludeFields = null, List includeFields = null, string prefix = "") { keyCode = !string.IsNullOrWhiteSpace(keyCode) ? keyCode : enumType.Name.ToString().Replace("EnumDes", "").Replace("Enum", "").Replace("Value", ""); var enumNames = Enum.GetNames(enumType).ToList(); if (excludeFields != null && excludeFields.Any()) { enumNames = enumNames.Except(excludeFields).ToList(); } if (includeFields != null && includeFields.Any()) { enumNames = enumNames.Intersect(includeFields).ToList(); } var children = new List(); foreach (var val in enumNames) { var id = (int)enumType.GetField(val).GetValue(val); children.Add(new EnumFieldDTO { Id = id, Value = $"{prefix}{val}", }); } return new EnumItemDTO { Code = keyCode, Children = children, }; } /// /// 查询病灶轮廓的关键点 /// /// 查询病灶轮廓的关键点请求实体 /// 关键点集合 /// /// public async Task> GetKeyPointsOfContourAsync(GetKeyPointsOfContourRequest request) { var contourPoints = request.Contours.Select(c => new Point2D { X = c.X, Y = c.Y }).ToArray(); var ls = request.LesionSize; var horizontalP1 = new Point2D(ls.HorizontalPoint1.X, ls.HorizontalPoint1.Y); var horizontalP2 = new Point2D(ls.HorizontalPoint2.X, ls.HorizontalPoint2.Y); var verticalP1 = new Point2D(ls.VerticalPoint1.X, ls.VerticalPoint1.Y); var verticalP2 = new Point2D(ls.VerticalPoint2.X, ls.VerticalPoint2.Y); var lesionSize = new LesionSize(horizontalP1, horizontalP2, verticalP1, verticalP2); // lesionSize.HorizontalLengthInPixel = ls.HorizontalLengthInPixel; // lesionSize.VerticalLengthInPixel = ls.VerticalLengthInPixel; var aiResult = ContourModifyHelper.KeyPointsOfContour(contourPoints, lesionSize); var resultData = KeyPointToDto(aiResult); return await Task.FromResult(resultData); } /// /// 移动光标,查询受影响关键点 /// /// 移动光标,查询受影响关键点请求实体 /// 受影响关键点下标 /// /// public async Task> AffectedKeyPointsByDragActionAsync(AffectedKeyPointsByDragActionRequest request) { var origKeyPoints = request.KeyPoints.Select(c => new KeyPointInfo { Type = (EnumKeyPointType)c.Type, IndexInContour = c.IndexInContour, Point = new Point2D(c.Point.X, c.Point.Y) }).ToArray(); var mousePoint = new Point2D(request.MousePoint.X, request.MousePoint.Y); var resultData = ContourModifyHelper.AffectedKeyPointsByDragAction(origKeyPoints, mousePoint).ToList(); return await Task.FromResult(resultData); } /// /// 拖动光标,查询所有轮廓点和关键点 /// /// 拖动光标,查询所有轮廓点和关键点请求实体 /// 所有轮廓点和关键点 /// /// public async Task ContourAndKeyPointsAfterDragAsync(ContourAndKeyPointsAfterDragRequest request) { var origContourPoints = request.Contours.Select(c => new Point2D { X = c.X, Y = c.Y }).ToArray(); var origKeyPoints = request.KeyPoints.Select(c => new KeyPointInfo { Type = (EnumKeyPointType)c.Type, IndexInContour = c.IndexInContour, Point = new Point2D(c.Point.X, c.Point.Y) }).ToArray(); var dragStartPoint = new Point2D(request.StartPoint.X, request.StartPoint.Y); var dragEndPoint = new Point2D(request.EndPoint.X, request.EndPoint.Y); ContourModifyHelper.ContourAndKeyPointsAfterDrag(origContourPoints, origKeyPoints, dragStartPoint, dragEndPoint , out Point2D[] dstContourPoints, out KeyPointInfo[] dstKeyPoints, out int[] affectedKeyPointIndexes); var dstContours = PointToDto(dstContourPoints); var dstKeys = KeyPointToDto(dstKeyPoints); var resultData = new ContourAndKeyPointsAfterDragResult { DstContours = dstContours, DstKeyPoints = dstKeys, AffectedKeyPointIndexes = affectedKeyPointIndexes.ToList(), }; return await Task.FromResult(resultData); } /// /// 画轮廓模式,查询鼠标位置到轮廓线的最短距离 /// /// 画轮廓模式,查询鼠标位置到轮廓线的最短距离请求实体 /// 鼠标离轮廓线的距离,高亮点的下标 /// /// public async Task MinimumDistanceToContourPointsAsync(MinimumDistanceToContourPointsRequest request) { var origContourPoints = request.ContourPoints.Select(c => new Point2D { X = c.X, Y = c.Y }).ToArray(); var mousePoint = new Point2D(request.MousePoint.X, request.MousePoint.Y); var distance = ContourModifyHelper.MinimumDistanceToContourPoints(origContourPoints, mousePoint, out int closestPointIndex); var resultData = new MinimumDistanceToContourPointsResult { DistanceCaught = distance, ClosestPointIndex = closestPointIndex, }; return await Task.FromResult(resultData); } /// /// 画轮廓模式,合并新旧轮廓 /// /// 画轮廓模式,查询鼠标位置到轮廓线的最短距离请求实体 /// 合并后的完整轮廓线,合并后的横纵径尺寸结果 /// /// public async Task ContourMergeAsync(ContourMergeRequest request) { var origContourPoints = request.ContourPoints.Select(c => new Point2D { X = c.X, Y = c.Y }).ToArray(); var ls = request.LesionSize; var horizontalP1 = new Point2D(ls.HorizontalPoint1.X, ls.HorizontalPoint1.Y); var horizontalP2 = new Point2D(ls.HorizontalPoint2.X, ls.HorizontalPoint2.Y); var verticalP1 = new Point2D(ls.VerticalPoint1.X, ls.VerticalPoint1.Y); var verticalP2 = new Point2D(ls.VerticalPoint2.X, ls.VerticalPoint2.Y); var lesionSize = new LesionSize(horizontalP1, horizontalP2, verticalP1, verticalP2); // lesionSize.HorizontalLengthInPixel = ls.HorizontalLengthInPixel; // lesionSize.VerticalLengthInPixel = ls.VerticalLengthInPixel; var drawingNewContourPoints = request.DrawingNewContourPoints.Select(c => new Point2D { X = c.X, Y = c.Y }).ToArray(); var aiResult = ContourModifyHelper.ContourMerge(origContourPoints, lesionSize, drawingNewContourPoints, out Point2D[] dstContourPoints, out LesionSize dstLesionSize); var dstContours = PointToDto(dstContourPoints); var resultData = new ContourMergeResult { DstContours = dstContours, DstLesionSize = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisLesionSize { HorizontalPoint1 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.HorizontalPoint1.X, Y = dstLesionSize.HorizontalPoint1.Y }, HorizontalPoint2 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.HorizontalPoint2.X, Y = dstLesionSize.HorizontalPoint2.Y }, VerticalPoint1 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.VerticalPoint1.X, Y = dstLesionSize.VerticalPoint1.Y }, VerticalPoint2 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.VerticalPoint2.X, Y = dstLesionSize.VerticalPoint2.Y }, HorizontalLengthInPixel = dstLesionSize.HorizontalLengthInPixel, VerticalLengthInPixel = dstLesionSize.VerticalLengthInPixel, }, }; return await Task.FromResult(resultData); } /// /// 获取颈动脉ai报告数据 /// /// 生成AI报告请求实体 /// private async Task GetCarotidAIMeasureResult(DiagnosisReportRequest request) { var result = new DiagnosisReportResult() { DiagnosisConclusion = DiagnosisConclusionEnum.NoObviousLesion }; var orderedExamDatas = request.RemedicalList.OrderByDescending(m => m.CarotidResult?.CarotidScanType).Where(m => m.CarotidResult?.MeasureImageFiles != null && m.CarotidResult.MeasureImageFiles.Count > 0); if (orderedExamDatas?.Count() > 0) { DiagnosisRemicalDTO leftData = null; //left var leftExamDatas = orderedExamDatas.Where(m => m.CarotidResult.CarotidScanType == CarotidScanTypeEnum.CarotidLeft); if (leftExamDatas.Any()) { foreach (var leftItem in leftExamDatas) { if (string.IsNullOrEmpty(leftItem.CarotidResult.MeasureResult)) { result.DiagnosisConclusion = DiagnosisConclusionEnum.Unrecognized; continue; } //leftItem.CarotidResult.MeasureResult = "{\"IntimaResult\":{\"IsSuccess\":true,\"AntIntima\":{\"IntimaThick\":0.0,\"IsSuccess\":false},\"PostIntima\":{\"IntimaThick\":0.75,\"IsSuccess\":true}},\"IsYImageSuccess\":false,\"PlaqueResult\":{\"PlaquePostion\":0,\"PlaqueCountType\":2,\"PlaqueType\":0,\"PlaqueWidth\":1.75,\"PlaqueHeight\":3.76,\"Stenosis\":10.37,\"IsSuccess\":true}}"; var measureResult = JsonConvert.DeserializeObject(leftItem.CarotidResult.MeasureResult) ?? new CarotidAIMeasureResult(); if (measureResult.PlaqueResult?.PlaqueCountType != Carotid.Utilities.DetectPlaque.PlaqueCountType.NoPlaque) { leftData = leftItem; result.DiagnosisConclusion = DiagnosisConclusionEnum.Other; break; } } if (leftData == null) { leftData = leftExamDatas.First(); } } if (leftData != null) { result.CarotidResult.Add(leftData); } //right float intimaThickStandard = 1.0f; DiagnosisRemicalDTO rightData = null; var rightExamDatas = orderedExamDatas.Where(m => m.CarotidResult.CarotidScanType == CarotidScanTypeEnum.CarotidRight); if (rightExamDatas.Any()) { foreach (var rightItem in rightExamDatas) { if (string.IsNullOrEmpty(rightItem.CarotidResult.MeasureResult)) { result.DiagnosisConclusion = DiagnosisConclusionEnum.Unrecognized; continue; } //rightItem.CarotidResult.MeasureResult = "{\"IntimaResult\":{\"IsSuccess\":true,\"AntIntima\":{\"IntimaThick\":0.0,\"IsSuccess\":false},\"PostIntima\":{\"IntimaThick\":0.75,\"IsSuccess\":true}},\"IsYImageSuccess\":false,\"PlaqueResult\":{\"PlaquePostion\":0,\"PlaqueCountType\":2,\"PlaqueType\":0,\"PlaqueWidth\":1.75,\"PlaqueHeight\":3.76,\"Stenosis\":10.37,\"IsSuccess\":true}}"; var measureResult = JsonConvert.DeserializeObject(rightItem.CarotidResult.MeasureResult) ?? new CarotidAIMeasureResult(); if (measureResult.IntimaResult?.PostIntima?.IntimaThick > intimaThickStandard) { rightData = rightItem; result.DiagnosisConclusion = DiagnosisConclusionEnum.Other; break; } } if (rightData == null) { rightData = rightExamDatas.First(); } } if (rightData != null) { result.CarotidResult.Add(rightData); } Logger.WriteLineInfo($"AIDiagnosisService package carotidAIMeasureResult finished, CarotidLeftRemedicalCode:{leftData?.RemedicalCode}, CarotidLeft:{leftData?.CarotidResult?.MeasureResult}, CarotidRightRemedicalCode:{rightData?.RemedicalCode}, CarotidRight:" + rightData?.CarotidResult?.MeasureResult); } return result; } private async Task CreateDiagnosisManagerAsync(DiagnosisReportRequest request) { var recordResults = new List(); foreach (var remedical in request.RemedicalList) { foreach (var perImage in remedical.DiagnosisResult) { var localFile = await DownloadAsync(remedical.FileToken); if (File.Exists((localFile))) { var diagnosisPerImage = new DiagnosisPerImageModel(); diagnosisPerImage.RemedicalCode = remedical.RemedicalCode; diagnosisPerImage.DataType = (WingAIDiagnosisService.Manage.DataType)remedical.DataType; diagnosisPerImage.LocalVidPath = localFile; diagnosisPerImage.Index = perImage.Index; diagnosisPerImage.PriorityScore = perImage.PriorityScore; var diagnosisResults = Newtonsoft.Json.JsonConvert.DeserializeObject>(Newtonsoft.Json.JsonConvert.SerializeObject(perImage.DiagResultsForEachOrgan)); diagnosisPerImage.DiagResultsForEachOrgan = diagnosisResults; recordResults.Add(diagnosisPerImage); } } } switch (request.Organ) { case DiagnosisOrganEnum.Breast: return new BreastDiagnosis(recordResults, request.Organ); case DiagnosisOrganEnum.Liver: return new LiverDiagnosis(recordResults, request.Organ); case DiagnosisOrganEnum.Thyroid://甲状腺 return new ThyroidDiagnosis(recordResults, request.Organ); default: throw new Exception($"not support organ type:{request.Organ.ToString()}"); } } private async Task CarotidDiagnosis(VinnoImageData imageData) { var resampleInputData = new ResampleInputData(); resampleInputData.SurfaceFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.SurfaceFileSuffix); resampleInputData.MdlFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.ModelFileSuffix); resampleInputData.MdlZipFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.ZipFileSuffix); resampleInputData.BaseAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix); resampleInputData.PlaqueAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix); resampleInputData.YShapeAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix); var resampleResult = TryResample(imageData, resampleInputData); if (resampleResult.ResampleErrorCode == ResampleErrorCode.Success) { var carotidAIImageTokens = new List(); if (resampleResult.CarotidAIMeasureResult.IsYImageSuccess) { var fileUrl = await UploadFileAsync(resampleInputData.YShapeAIImagePath, Path.GetFileName(resampleInputData.YShapeAIImagePath)); if (!string.IsNullOrWhiteSpace(fileUrl)) { carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.YShape }); } File.Delete(resampleInputData.YShapeAIImagePath); } if (resampleResult.CarotidAIMeasureResult.IntimaResult.IsSuccess) { var fileUrl = await UploadFileAsync(resampleInputData.BaseAIImagePath, Path.GetFileName(resampleInputData.BaseAIImagePath)); if (!string.IsNullOrWhiteSpace(fileUrl)) { carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.Base }); } File.Delete(resampleInputData.BaseAIImagePath); } if (resampleResult.CarotidAIMeasureResult.PlaqueResult.IsSuccess && resampleResult.CarotidAIMeasureResult.PlaqueResult.PlaqueCountType != WingAIDiagnosisService.Carotid.Utilities.DetectPlaque.PlaqueCountType.NoPlaque) { var fileUrl = await UploadFileAsync(resampleInputData.PlaqueAIImagePath, Path.GetFileName(resampleInputData.PlaqueAIImagePath)); if (!string.IsNullOrWhiteSpace(fileUrl)) { carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.Plaque }); } File.Delete(resampleInputData.PlaqueAIImagePath); } var surfaceImageList = new List(); long surfaceFileSize = 0; var cdnSurfaceFile = ""; if (!string.IsNullOrWhiteSpace(resampleInputData.SurfaceFilePath) && File.Exists(resampleInputData.SurfaceFilePath)) { surfaceImageList = await SurfaceFile(resampleInputData.SurfaceFilePath); var surfaceFileTuple = await UploadFileTupleAsync(resampleInputData.SurfaceFilePath, Path.GetFileName(resampleInputData.SurfaceFilePath)); surfaceFileSize = surfaceFileTuple.Item2; File.Delete(resampleInputData.SurfaceFilePath); resampleInputData.SurfaceFilePath = surfaceFileTuple.Item1; cdnSurfaceFile = surfaceFileTuple.Item1.ToUrlToken();//ToCDN } long mdlFileFileSize = 0; var cdnMdlFile = ""; if (!string.IsNullOrWhiteSpace(resampleInputData.MdlFilePath) && File.Exists(resampleInputData.MdlFilePath)) { var mdlFileTuple = await UploadFileTupleAsync(resampleInputData.MdlFilePath, Path.GetFileName(resampleInputData.MdlFilePath)); mdlFileFileSize = mdlFileTuple.Item2; File.Delete(resampleInputData.MdlFilePath); resampleInputData.MdlFilePath = mdlFileTuple.Item1; cdnMdlFile = mdlFileTuple.Item1.ToUrlToken();//ToCDN } var result = new CarotidResultDTO { CarotidScanType = (CarotidScanTypeEnum)resampleInputData.ScanType, CarotidScanDirection = (CarotidScanDirectionEnum)resampleInputData.CarotidScanDirection, SurfaceFile = resampleInputData.SurfaceFilePath, SurfaceFileSize = surfaceFileSize, CDNSurfaceFile = cdnSurfaceFile, MdlFileSize = mdlFileFileSize, MdlFile = resampleInputData.MdlFilePath, CDNMdlFile = cdnMdlFile, MeasureImageFiles = carotidAIImageTokens.Select(x => new MeasureImageFileDTO { ImageType = (CarotidAIImageTypeEnum)x.AIImageType, ImageFile = x.AIImageToken }).ToList(), MeasureResult = JsonConvert.SerializeObject(resampleResult.CarotidAIMeasureResult), SurfaceImageList = surfaceImageList, }; return result; } return null; } private ResampleResult TryResample(VinnoImageData vinnoImageData, ResampleInputData resampleInputData) { float scanDistance = 7; ResampleResult resampleResult = new ResampleResult(ResampleErrorCode.Fail); var vinnoExtendedData = VinnoCarotidExtendedData.FromBytes(vinnoImageData.ExtendedData); CarotidScanType scanType = CarotidScanType.CarotidLeft; CarotidScanDirection direction = CarotidScanDirection.TopToBottom; if (vinnoExtendedData != null) { // Should be 6~9cm normally. //scanDistance = vinnoExtendedData.ScanDistance; scanType = vinnoExtendedData.CarotidType == CarotidType.Left ? CarotidScanType.CarotidLeft : CarotidScanType.CarotidRight; direction = vinnoExtendedData.CarotidDirection == CarotidDirection.BottomToTop ? CarotidScanDirection.BottomToTop : CarotidScanDirection.TopToBottom; } else { //This is a walkaround : vid's first frame is black image or depth coordinate is not in the right place. var middleCount = vinnoImageData.ImageCount / 2; var vinnoImage = vinnoImageData.GetImage(middleCount); var imageBuffer = vinnoImage.ImageData; if (_recognizeCarotidTypeCanBeUse) { //转成彩色图像 using var img = new Mat(); CvInvoke.Imdecode(imageBuffer, ImreadModes.Color, img); using var image = img.ToImage(); //识别左右颈信息 scanType = _recognizeCarotidType.RecognizeType(image); } } resampleInputData.ScanDistance = scanDistance; resampleInputData.ScanType = scanType; resampleInputData.CarotidScanDirection = direction; resampleResult = ResampleModel.Instance.Resample(vinnoImageData, resampleInputData, _workerLevel); return resampleResult; } /// /// /// /// private async Task> SurfaceFile(string filePath) { var fileUrls = new List(); using (var stream = new FileStream(filePath, FileMode.Open)) { var reader = new Carotid3DStreamReader(stream); var width = reader.ReadInt(); var height = reader.ReadInt(); var depth = reader.ReadInt(); var physicalData = VinnoCarotid3DPhysicalData.FromBytes(reader.ReadBytes()); var imageCount = reader.ReadInt(); var fileName = $"{Guid.NewGuid():N}"; for (var i = 0; i < imageCount; i++) { var readBytes = reader.ReadBytes(); if (readBytes.Any()) { using var img = new Mat(); CvInvoke.Imdecode(readBytes, ImreadModes.Color, img); var buf = new Emgu.CV.Util.VectorOfByte(); using var image = img.ToImage(); CvInvoke.Imencode(".jpg", image, buf, new KeyValuePair(ImwriteFlags.JpegQuality, 80)); var jpgByte = buf.ToArray(); var localPath = Path.Combine(_tempFolder, $"{fileName}_{i + 1}.jpg"); File.WriteAllBytes(localPath, jpgByte); var fileUrl = await UploadFileAsync(localPath, Path.GetFileName(localPath)); fileUrls.Add(fileUrl); File.Delete(localPath); } } } return fileUrls; } private SKBitmap CreateBitmap(VinnoImage vinnoImage) { try { return SKBitmap.Decode(vinnoImage.ImageData); } catch (Exception ex) { Logger.WriteLineError($"Create skbitmap by VinnoImage error:{ex}"); } return null; } private EnumColorType MapTo(SKColorType sKColor) { switch (sKColor) { case SKColorType.Rgba8888: return EnumColorType.Rgba; case SKColorType.Rgb888x: return EnumColorType.Rgba; case SKColorType.Bgra8888: return EnumColorType.Bgra; case SKColorType.Gray8: return EnumColorType.Gray8; } throw new Exception($"AIService not support color type:{sKColor}"); } private void AIdiagSystem_NotifyError(object sender, ErrorEventArgs e) { Logger.WriteLineError("AIdiagSystem_NotifyError:" + e.GetException()); } private void AIdiagSystem_NotifyLog(object sender, LogEventArgs e) { if (e != null && !string.IsNullOrEmpty(e.Msg)) { switch (e.LogType) { case EnumLogType.InfoLog: Logger.WriteLineInfo($"AIdiagSystem_NotifyLog:{e.Msg}"); break; case EnumLogType.ErrorLog: Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}"); break; case EnumLogType.WarnLog: Logger.WriteLineWarn($"AIdiagSystem_NotifyLog:{e.Msg}"); break; case EnumLogType.FatalLog: Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}"); break; default: Logger.WriteLineInfo(e.Msg); break; } } } private DiagnosisConclusion GetDiagnosisConclusion(List results) where T : AIDiagnosisPerImageModel { var diagnosisConclusions = new List(); foreach (var imageResult in results) { foreach (var diagnosisResult in imageResult.DiagResultsForEachOrgan) { var benignLabels = new List(); var malignantLabels = new List(); if (diagnosisResult.Organ == DiagnosisOrganEnum.Breast) { benignLabels = new List { 1, 2, 3 }; malignantLabels = new List { 4, 5, 6, 7 }; } else if (diagnosisResult.Organ == DiagnosisOrganEnum.Liver) { benignLabels = new List { 1, 2, 3, 5, 6, 7, 8 }; malignantLabels = new List { 4 }; } else if (diagnosisResult.Organ == DiagnosisOrganEnum.Thyroid) { benignLabels = new List { 1, 2, 7 }; malignantLabels = new List { 3, 4, 5, 6 }; } var labels = diagnosisResult.DetectedObjects.Select(x => x.Label); if (labels.Contains(0)) { diagnosisConclusions.Add(DiagnosisConclusion.NoObviousLesion); } if (labels.Intersect(benignLabels).Any()) { diagnosisConclusions.Add(DiagnosisConclusion.Benign); } if (labels.Intersect(malignantLabels).Any()) { diagnosisConclusions.Add(DiagnosisConclusion.Malignant); } } } var containsBenign = diagnosisConclusions.Contains(DiagnosisConclusion.Benign); var containsMalignant = diagnosisConclusions.Contains(DiagnosisConclusion.Malignant); var containsNoObviousLesion = diagnosisConclusions.Contains(DiagnosisConclusion.NoObviousLesion); if (containsBenign && containsMalignant) { return DiagnosisConclusion.BenignAndMalignant; } else if (containsBenign) { return DiagnosisConclusion.Benign; } else if (containsMalignant) { return DiagnosisConclusion.Malignant; } else if (containsNoObviousLesion) { return DiagnosisConclusion.NoObviousLesion; } else { return DiagnosisConclusion.Unrecognized; } } private List GetDiagnosisOrgans(List results) { var diagnosisOrgans = new List(); foreach (var imageResult in results) { foreach (var diagnosisResult in imageResult.DiagResultsForEachOrgan) { var organName = Enum.GetName(typeof(DiagnosisOrganEnum), (int)diagnosisResult.Organ); if (!string.IsNullOrWhiteSpace(organName) && diagnosisResult.Organ != DiagnosisOrganEnum.Null && diagnosisResult.DetectedObjects?.Any() == true) { diagnosisOrgans.Add((DiagnosisOrganEnum)diagnosisResult.Organ); } } } return diagnosisOrgans.Distinct().ToList(); } private Dictionary NormalDiagnosis(VinnoImageData imageData) { var images = new List(); var results = new Dictionary(); try { var totalCount = imageData.ImageCount; for (var i = 0; i < totalCount; i++) { var image = imageData.GetImage(i); var bitmap = CreateBitmap(image); if (bitmap != null) { images.Add(new RawImage(bitmap.Bytes, bitmap.Width, bitmap.Height, MapTo(bitmap.ColorType))); } } if (images.Count > 0) { var excuteCount = images.Count / _batchImageSize; var remainderCount = images.Count % _batchImageSize; var diagId = _diagSystem.StartEvalutationOfMultipleImageBatches(images.Count); for (var j = 0; j < excuteCount; j++) { var excuteImages = images.GetRange(j * _batchImageSize, _batchImageSize); _diagSystem.PushOneBatchOfImagesAsync(diagId, excuteImages); Thread.Sleep(_sleepTime); } if (remainderCount > 0) { var excuteImages = images.GetRange(excuteCount * _batchImageSize, remainderCount); _diagSystem.PushOneBatchOfImagesAsync(diagId, excuteImages); } results = _diagSystem.GetEvaluationsOfPushedMultipleImageBatches(diagId); } } catch (Exception ex) { Logger.WriteLineWarn($"AIService NormalDiagnosis err, {ex}"); } finally { foreach (var image in images) { image?.Dispose(); } } return results; } /// /// 保存AI诊断结果 /// /// /// /// /// private async Task AddDiagnosisResultInfosAsync(string relationCode, string fileUrl, List diagnosisResultInfos) { try { var resultInfos = new List(); for (var i = 0; i < diagnosisResultInfos.Count; i++) { resultInfos.Add(new DiagnosisResultDTO { Index = i, DiagnosisResult = Newtonsoft.Json.JsonConvert.SerializeObject(diagnosisResultInfos[i]), }); } var addRequest = new AddDiagnosisResultInfosDBRequest { RelationCode = relationCode, FileUrl = fileUrl, DiagnosisResultInfos = resultInfos, }; _diagnosisResultDBService.AddDiagnosisResultInfosAsync(addRequest); return true; } catch (Exception ex) { Logger.WriteLineWarn($"AIDiagnosisService AddDiagnosisResultInfosAsync err, ex:{ex}"); return false; } } /// 下载文件 /// /// private async Task DownloadAsync(string fileUrl) { try { if (string.IsNullOrEmpty(fileUrl)) { return string.Empty; } if (!Directory.Exists(_tempFolder)) { Directory.CreateDirectory(_tempFolder); } var fileName = Path.GetFileName(fileUrl); var tempFile = Path.Combine(_tempFolder, fileName); if (File.Exists(tempFile)) { return tempFile; } long fileSize = 0; using (var request = new HttpRequestMessage()) { request.RequestUri = new Uri(fileUrl); request.Method = HttpMethod.Get; var response = await _httpClient.SendAsync(request); if (response != null && response.StatusCode == HttpStatusCode.OK) { var contentLength = response.Content.Headers.ContentLength; fileSize = contentLength == null ? 0 : contentLength.Value; } } if (fileSize <= 0) { throw new NotSupportedException($"fileSize is {fileSize}"); } byte[] bytes = await _httpClient.GetByteArrayAsync(fileUrl); File.WriteAllBytes(tempFile, bytes); return tempFile; } catch (Exception ex) { Logger.WriteLineWarn($"DiagnosisService download file err, url: {fileUrl}, {ex}"); } finally { //Logger.WriteLineInfo($"download file:{fileUrl}"); } return string.Empty; } /// /// 上传文件 /// /// /// /// private async Task UploadFileAsync(string filePath, string fileName) { var fileToken = ""; using (var fileStream = new FileStream(filePath, FileMode.Open)) { var size = fileStream.Length; byte[] buffer = new byte[fileStream.Length]; fileStream.Read(buffer, 0, buffer.Length); fileToken = await DoUploadFile(fileName, buffer); } return fileToken; } /// /// 上传文件 /// /// /// /// private async Task> UploadFileTupleAsync(string filePath, string fileName) { var fileToken = ""; long size = 0; using (var fileStream = new FileStream(filePath, FileMode.Open)) { size = fileStream.Length; byte[] buffer = new byte[fileStream.Length]; fileStream.Read(buffer, 0, buffer.Length); fileToken = await DoUploadFile(fileName, buffer); } return new Tuple(fileToken, size); } /// /// 上传文件 /// /// /// /// async Task DoUploadFile(string fileName, byte[] fileData) { var requestHeads = new Dictionary(); var defaultToken = await _authenticationService.GetServerDefaultTokenAsync(); var authorizationRet = await _storageService.GetAuthorizationAsync( new FileServiceRequest { Token = defaultToken, FileName = fileName, }); requestHeads.Add("Authorization", authorizationRet.Authorization); var fileUrl = authorizationRet.StorageUrl; Logger.WriteLineInfo($"DoUploadFile fileUrl:{fileUrl}"); using (var request = new HttpRequestMessage()) { var fileExtension = Path.GetExtension(fileName); var mimeType = FileHelper.GetMimeType(fileExtension); var contentType = MediaTypeHeaderValue.Parse(mimeType); using (UploadContent content = new UploadContent(fileData, contentType)) { request.RequestUri = new Uri(fileUrl); request.Method = HttpMethod.Put; request.Content = content; foreach (var head in requestHeads) { request.Headers.TryAddWithoutValidation(head.Key, head.Value); } var result = await ExecuteRequest(request); if (!result) { throw new Exception("Upload file failed!"); } } } return fileUrl; } private UploadFileTypeEnum GetUploadFileType(string fileName) { if (fileName.EndsWith(".vid")) { return UploadFileTypeEnum.VID; } else if (fileName.EndsWith(".jpg")) { return UploadFileTypeEnum.JPG; } else { return UploadFileTypeEnum.MP4; } } /// /// 执行请求 /// /// /// /// public async Task ExecuteRequest(HttpRequestMessage httpRequestMessage) { try { var response = await _httpClient.SendAsync(httpRequestMessage); if (response != null && response.StatusCode == HttpStatusCode.OK) { return true; } return false; } catch (Exception ex) { throw ex; } } private List PointToDto(Point2D[] points) { var dstContours = new List(); foreach (var p in points) { dstContours.Add(new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = p.X, Y = p.Y }); } return dstContours; } private List KeyPointToDto(KeyPointInfo[] points) { var dstKeys = new List(); foreach (var point in points) { dstKeys.Add(new DiagnosisKeyPointDTO { Type = (DiagnosisKeyPointType)point.Type, IndexInContour = point.IndexInContour, Point = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = point.Point.X, Y = point.Point.Y, }, }); } return dstKeys; } } }