using WingServerCommon.Service;
using WingInterfaceLibrary.Interface;
using WingInterfaceLibrary.Request;
using WingInterfaceLibrary.Enum;
using WingServerCommon.Config;
using WingServerCommon.Config.Parameters;
using WingAIDiagnosisService.Carotid.Utilities;
using Vinno.vCloud.Common.Vid2;
using WingAIDiagnosisService.Manage;
using WingAIDiagnosisService.Carotid;
using Newtonsoft.Json;
using AI.DiagSystem;
using WingServerCommon.Log;
using System.IO;
using System;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Linq;
using AI.Common;
using System.Threading;
using AI.Common.Log;
using SkiaSharp;
using System.Net.Http;
using System.Net;
using WingInterfaceLibrary.Request.Storage;
using WingInterfaceLibrary.Result.AIDiagnosis;
using WingInterfaceLibrary.Request.AIDiagnosis;
using WingInterfaceLibrary.DTO.Record;
using WingInterfaceLibrary.DTO.Comment;
using Emgu.CV;
using Emgu.CV.CvEnum;
using Emgu.CV.Structure;
using System.Net.Http.Headers;
using WingAIDiagnosisService.Common;
using WingInterfaceLibrary.Interface.DBInterface;
using WingInterfaceLibrary.Request.DBRequest;
using WingInterfaceLibrary.DTO.DiagnosisResult;
using WingInterfaceLibrary.Request.RemedicalAISelected;
using ContourModifyUtils;
namespace WingAIDiagnosisService.Service
{
///
/// AI诊断服务
///
public partial class AIDiagnosisService : JsonRpcService, IAIDiagnosisService
{
private EnumPerformance _workerLevel = EnumPerformance.Medium;
private int _batchImageSize = 16;
private int _sleepTime = 400;
private readonly string _tempFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "DiagnosisTemp");
private AIDiagSystem _diagSystem;
private AILesionCompareUtils _lesionCompareUtil;
private RecognizeCarotidType _recognizeCarotidType = new RecognizeCarotidType();
//识别左右颈是否初始化成功
private static bool _recognizeCarotidTypeCanBeUse = false;
static HttpClient _httpClient = new HttpClient();
private IAuthenticationService _authenticationService;
private IStorageService _storageService;
private readonly string _carotidName = "CarotidPlaqueDetect";
private int _contourInterval = 10;
private IDiagnosisResultDBService _diagnosisResultDBService;
///
/// Init service
///
public override void Load(JsonRpcClientPool jsonRpcClientPool)
{
base.Load(jsonRpcClientPool);
_authenticationService = GetProxy();
_storageService = GetProxy();
_workerLevel = (EnumPerformance)ConfigurationManager.GetParammeter("AI", "WorkerLevel").Value;
_batchImageSize = ConfigurationManager.GetParammeter("AI", "BatchImageSize").Value;
_contourInterval = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value;
_httpClient.Timeout = TimeSpan.FromMinutes(15);
_diagnosisResultDBService = GetProxy();
InitAISystem();
//初始化识别左右颈
_recognizeCarotidTypeCanBeUse = _recognizeCarotidType.Initialization();
var keyPointDistanceThreshold = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value;
var dragStartPointCatchDistanceThreshold = ConfigurationManager.GetParammeter("AI", "ContourInterval").Value;
ContourModifyHelper.KeyPointDistanceThreshold = keyPointDistanceThreshold;
ContourModifyHelper.DragStartPointCatchDistanceThreshold = dragStartPointCatchDistanceThreshold;
}
private void InitAISystem()
{
var modeFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AIDiagnosis\\Networks");
_diagSystem = new AIDiagSystem(_workerLevel, modeFilePath, EnumInferWorkName.Default, false, true, true);
if (_diagSystem != null)
{
_diagSystem.NotifyError += AIdiagSystem_NotifyError;
_diagSystem.NotifyLog += AIdiagSystem_NotifyLog;
}
_lesionCompareUtil = new AILesionCompareUtils();
}
///
/// 查询杏聆荟支持的AI模块
///
/// 查询杏聆荟支持的AI模块请求实体
///
/// false
public async Task> FindDiagnosisModulesAsync(TokenRequest request)
{
try
{
var moduleNames = AIDiagSystem.GetValidModuleNamesForVclound() ?? new List();
var resultData = moduleNames.Select(x => x.ToString()).ToList();
if (!resultData.Contains(_carotidName))
{
resultData.Remove("CarotidArtery");
resultData.Add(_carotidName);
}
return await Task.FromResult(resultData.Distinct().ToList());
}
catch (Exception)
{
return new List();
}
}
///
/// 图像诊断
///
/// 图像诊断请求实体
/// 图像诊断结果
/// false
public async Task DiagnosisImageAsync(DiagnosisImageRequest request)
{
try
{
var relationCode = request.RelationCode;
var fileUrl = request.FileToken;
var localFile = "";
if (!string.IsNullOrWhiteSpace(request.DiskPath))
{
localFile = request.DiskPath;
}
else
{
localFile = await DownloadAsync(fileUrl);
}
if (File.Exists(localFile))
{
if (_diagSystem == null)
{
InitAISystem();
}
using (var imageData = new VinnoImageData(localFile, OperationMode.Open))
{
if (DiagnosisHelper.IsCarotid(imageData))
{
if (imageData.ImageCount > DiagnosisHelper.CarotidMinImageCounts)
{
//颈动脉
var result = await CarotidDiagnosis(imageData);
var resultData = new DiagnosisImageResult()
{
DiagnosisOrgans = new List() { DiagnosisOrganEnum.CarotidArtery },
CarotidResult = result
};
return resultData;
}
}
else
{
//乳腺、肝脏
var results = NormalDiagnosis(imageData);
//AI实体有很多参数仅支持get,为了方便后续反序列化,db入库前先做个转换
var diagnosisPerImages = results.Select(x => new AIDiagnosisPerImageModel(x.Key, x.Value, 1)).ToList();
await AddDiagnosisResultInfosAsync(relationCode, fileUrl, diagnosisPerImages);
//轮廓线分隔处理
diagnosisPerImages = results.Select(x => new AIDiagnosisPerImageModel(x.Key, x.Value, _contourInterval)).ToList();
var diagnosisResult = Newtonsoft.Json.JsonConvert.DeserializeObject>(Newtonsoft.Json.JsonConvert.SerializeObject(diagnosisPerImages));
var resultData = new DiagnosisImageResult()
{
DiagnosisConclusion = (DiagnosisConclusionEnum)GetDiagnosisConclusion(diagnosisPerImages),
DiagnosisResult = diagnosisResult,
DiagnosisOrgans = GetDiagnosisOrgans(diagnosisResult),
};
return resultData;
}
}
}
}
catch (Exception ex)
{
Logger.WriteLineWarn($"DiagnosisImageAsync err {ex}");
}
return new DiagnosisImageResult();
}
///
/// 生成AI报告
///
/// 生成AI报告请求实体
///
/// false
public async Task DiagnosisReportAsync(DiagnosisReportRequest request)
{
try
{
if (request.Organ == DiagnosisOrganEnum.CarotidArtery)
{
var result = await GetCarotidAIMeasureResult(request);
return result;
}
else
{
var manager = await CreateDiagnosisManagerAsync(request);
var reportPerImages = manager.GetReportResults();
var diagnosisConclusion = GetDiagnosisConclusion(reportPerImages);
var diagnosisResult = new List();
foreach (var item in reportPerImages)
{
var aiFileToken = await UploadFileAsync(item.AILocalImagePath, Path.GetFileName(item.AILocalImagePath));
var perImageResult = Newtonsoft.Json.JsonConvert.DeserializeObject(Newtonsoft.Json.JsonConvert.SerializeObject(item));
perImageResult.RemedicalCode = item.RemedicalCode;
perImageResult.DataType = (RemedicalFileDataTypeEnum)item.DataType;
perImageResult.Pixel = item.Pixel;
perImageResult.AIFileToken = aiFileToken;
diagnosisResult.Add(perImageResult);
}
return new DiagnosisReportResult
{
DiagnosisConclusion = (DiagnosisConclusionEnum)diagnosisConclusion,
DiagnosisResult = diagnosisResult,
};
}
}
catch (Exception ex)
{
Logger.WriteLineWarn($"AIService DiagnosisReport err, {ex}");
}
return new DiagnosisReportResult();
}
///
/// 查询AI相关枚举集合
///
/// 查询AI相关枚举集合请求实体
/// AI相关枚举集合
///
///
public async Task GetDiagnosisEnumItemsAsync(GetDiagnosisEnumItemsRequest request)
{
var diagnosisItems = new List();
//病灶
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "Breast", new List { "BIRads1" }));
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "Liver", new List { "BIRads1" }, prefix: "Liver"));
diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "Thyroid", new List { "TIRADS0" }));
//病灶特性描述
diagnosisItems.Add(GetEnumItem(typeof(EnumDesShapeValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesOrientationValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesEchoPatternValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesLesionBoundaryValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesMarginValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumCalcificationsValue), "Calcification"));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverShapeValue), prefix: "LiveShape"));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverBoundaryValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesLiverEchoTextureValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidEchoPatternValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidShapeValue)));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidMarginValue), prefix: "Thyroid"));
diagnosisItems.Add(GetEnumItem(typeof(EnumDesThyroidEchogenicFociValue)));
//局灶和弥漫区分
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "BreastLocalLesion", includeFields: new List { "Lipomyoma", "BIRads2", "BIRads3" }));
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisBreastLabelEnum), "BreastDiffuseLesion", includeFields: new List { "BIRads4A", "BIRads4B", "BIRads4C", "BIRads5" }));
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "LiverLocalLesion", includeFields: new List { "Hyperechoic", "HHE", "CYST", "PossibleCancer" }, prefix: "Liver"));
diagnosisItems.Add(GetEnumItem(typeof(DiagnosisLiverLabelEnum), "LiverDiffuseLesion", includeFields: new List { "FattyLiver", "DiffuseLesions", "Cirrhosis", "PCLD" }, prefix: "Liver"));
diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "ThyroidLocalLesion", includeFields: new List { "TIRADS2", "TIRADS3", "TIRADS4a", "TIRADS4b", "TIRADS4c", "TIRADS5" }));
diagnosisItems.Add(GetEnumItem(typeof(AIThyroidLabelEnum), "ThyroidDiffuseLesion", includeFields: new List { "DiffuseDisease" }));
var resultData = new GetDiagnosisEnumItemsResult { Source = diagnosisItems };
return await Task.FromResult(resultData);
}
private EnumItemDTO GetEnumItem(Type enumType, string keyCode = "", List excludeFields = null, List includeFields = null, string prefix = "")
{
keyCode = !string.IsNullOrWhiteSpace(keyCode) ? keyCode : enumType.Name.ToString().Replace("EnumDes", "").Replace("Enum", "").Replace("Value", "");
var enumNames = Enum.GetNames(enumType).ToList();
if (excludeFields != null && excludeFields.Any())
{
enumNames = enumNames.Except(excludeFields).ToList();
}
if (includeFields != null && includeFields.Any())
{
enumNames = enumNames.Intersect(includeFields).ToList();
}
var children = new List();
foreach (var val in enumNames)
{
var id = (int)enumType.GetField(val).GetValue(val);
children.Add(new EnumFieldDTO
{
Id = id,
Value = $"{prefix}{val}",
});
}
return new EnumItemDTO
{
Code = keyCode,
Children = children,
};
}
///
/// 查询病灶轮廓的关键点
///
/// 查询病灶轮廓的关键点请求实体
/// 关键点集合
///
///
public async Task> GetKeyPointsOfContourAsync(GetKeyPointsOfContourRequest request)
{
var contourPoints = request.Contours.Select(c => new Point2D
{
X = c.X,
Y = c.Y
}).ToArray();
var ls = request.LesionSize;
var horizontalP1 = new Point2D(ls.HorizontalPoint1.X, ls.HorizontalPoint1.Y);
var horizontalP2 = new Point2D(ls.HorizontalPoint2.X, ls.HorizontalPoint2.Y);
var verticalP1 = new Point2D(ls.VerticalPoint1.X, ls.VerticalPoint1.Y);
var verticalP2 = new Point2D(ls.VerticalPoint2.X, ls.VerticalPoint2.Y);
var lesionSize = new LesionSize(horizontalP1, horizontalP2, verticalP1, verticalP2);
// lesionSize.HorizontalLengthInPixel = ls.HorizontalLengthInPixel;
// lesionSize.VerticalLengthInPixel = ls.VerticalLengthInPixel;
var aiResult = ContourModifyHelper.KeyPointsOfContour(contourPoints, lesionSize);
var resultData = KeyPointToDto(aiResult);
return await Task.FromResult(resultData);
}
///
/// 移动光标,查询受影响关键点
///
/// 移动光标,查询受影响关键点请求实体
/// 受影响关键点下标
///
///
public async Task> AffectedKeyPointsByDragActionAsync(AffectedKeyPointsByDragActionRequest request)
{
var origKeyPoints = request.KeyPoints.Select(c => new KeyPointInfo
{
Type = (EnumKeyPointType)c.Type,
IndexInContour = c.IndexInContour,
Point = new Point2D(c.Point.X, c.Point.Y)
}).ToArray();
var mousePoint = new Point2D(request.MousePoint.X, request.MousePoint.Y);
var resultData = ContourModifyHelper.AffectedKeyPointsByDragAction(origKeyPoints, mousePoint).ToList();
return await Task.FromResult(resultData);
}
///
/// 拖动光标,查询所有轮廓点和关键点
///
/// 拖动光标,查询所有轮廓点和关键点请求实体
/// 所有轮廓点和关键点
///
///
public async Task ContourAndKeyPointsAfterDragAsync(ContourAndKeyPointsAfterDragRequest request)
{
var origContourPoints = request.Contours.Select(c => new Point2D
{
X = c.X,
Y = c.Y
}).ToArray();
var origKeyPoints = request.KeyPoints.Select(c => new KeyPointInfo
{
Type = (EnumKeyPointType)c.Type,
IndexInContour = c.IndexInContour,
Point = new Point2D(c.Point.X, c.Point.Y)
}).ToArray();
var dragStartPoint = new Point2D(request.StartPoint.X, request.StartPoint.Y);
var dragEndPoint = new Point2D(request.EndPoint.X, request.EndPoint.Y);
ContourModifyHelper.ContourAndKeyPointsAfterDrag(origContourPoints, origKeyPoints, dragStartPoint, dragEndPoint
, out Point2D[] dstContourPoints, out KeyPointInfo[] dstKeyPoints, out int[] affectedKeyPointIndexes);
var dstContours = PointToDto(dstContourPoints);
var dstKeys = KeyPointToDto(dstKeyPoints);
var resultData = new ContourAndKeyPointsAfterDragResult
{
DstContours = dstContours,
DstKeyPoints = dstKeys,
AffectedKeyPointIndexes = affectedKeyPointIndexes.ToList(),
};
return await Task.FromResult(resultData);
}
///
/// 画轮廓模式,查询鼠标位置到轮廓线的最短距离
///
/// 画轮廓模式,查询鼠标位置到轮廓线的最短距离请求实体
/// 鼠标离轮廓线的距离,高亮点的下标
///
///
public async Task MinimumDistanceToContourPointsAsync(MinimumDistanceToContourPointsRequest request)
{
var origContourPoints = request.ContourPoints.Select(c => new Point2D
{
X = c.X,
Y = c.Y
}).ToArray();
var mousePoint = new Point2D(request.MousePoint.X, request.MousePoint.Y);
var distance = ContourModifyHelper.MinimumDistanceToContourPoints(origContourPoints, mousePoint, out int closestPointIndex);
var resultData = new MinimumDistanceToContourPointsResult
{
DistanceCaught = distance,
ClosestPointIndex = closestPointIndex,
};
return await Task.FromResult(resultData);
}
///
/// 画轮廓模式,合并新旧轮廓
///
/// 画轮廓模式,查询鼠标位置到轮廓线的最短距离请求实体
/// 合并后的完整轮廓线,合并后的横纵径尺寸结果
///
///
public async Task ContourMergeAsync(ContourMergeRequest request)
{
var origContourPoints = request.ContourPoints.Select(c => new Point2D
{
X = c.X,
Y = c.Y
}).ToArray();
var ls = request.LesionSize;
var horizontalP1 = new Point2D(ls.HorizontalPoint1.X, ls.HorizontalPoint1.Y);
var horizontalP2 = new Point2D(ls.HorizontalPoint2.X, ls.HorizontalPoint2.Y);
var verticalP1 = new Point2D(ls.VerticalPoint1.X, ls.VerticalPoint1.Y);
var verticalP2 = new Point2D(ls.VerticalPoint2.X, ls.VerticalPoint2.Y);
var lesionSize = new LesionSize(horizontalP1, horizontalP2, verticalP1, verticalP2);
// lesionSize.HorizontalLengthInPixel = ls.HorizontalLengthInPixel;
// lesionSize.VerticalLengthInPixel = ls.VerticalLengthInPixel;
var drawingNewContourPoints = request.DrawingNewContourPoints.Select(c => new Point2D
{
X = c.X,
Y = c.Y
}).ToArray();
var aiResult = ContourModifyHelper.ContourMerge(origContourPoints, lesionSize, drawingNewContourPoints, out Point2D[] dstContourPoints, out LesionSize dstLesionSize);
var dstContours = PointToDto(dstContourPoints);
var resultData = new ContourMergeResult
{
DstContours = dstContours,
DstLesionSize = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisLesionSize
{
HorizontalPoint1 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.HorizontalPoint1.X, Y = dstLesionSize.HorizontalPoint1.Y },
HorizontalPoint2 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.HorizontalPoint2.X, Y = dstLesionSize.HorizontalPoint2.Y },
VerticalPoint1 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.VerticalPoint1.X, Y = dstLesionSize.VerticalPoint1.Y },
VerticalPoint2 = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D { X = dstLesionSize.VerticalPoint2.X, Y = dstLesionSize.VerticalPoint2.Y },
HorizontalLengthInPixel = dstLesionSize.HorizontalLengthInPixel,
VerticalLengthInPixel = dstLesionSize.VerticalLengthInPixel,
},
};
return await Task.FromResult(resultData);
}
///
/// 获取颈动脉ai报告数据
///
/// 生成AI报告请求实体
///
private async Task GetCarotidAIMeasureResult(DiagnosisReportRequest request)
{
var result = new DiagnosisReportResult()
{
DiagnosisConclusion = DiagnosisConclusionEnum.NoObviousLesion
};
var orderedExamDatas = request.RemedicalList.OrderByDescending(m => m.CarotidResult?.CarotidScanType).Where(m => m.CarotidResult?.MeasureImageFiles != null && m.CarotidResult.MeasureImageFiles.Count > 0);
if (orderedExamDatas?.Count() > 0)
{
DiagnosisRemicalDTO leftData = null;
//left
var leftExamDatas = orderedExamDatas.Where(m => m.CarotidResult.CarotidScanType == CarotidScanTypeEnum.CarotidLeft);
if (leftExamDatas.Any())
{
foreach (var leftItem in leftExamDatas)
{
if (string.IsNullOrEmpty(leftItem.CarotidResult.MeasureResult))
{
result.DiagnosisConclusion = DiagnosisConclusionEnum.Unrecognized;
continue;
}
//leftItem.CarotidResult.MeasureResult = "{\"IntimaResult\":{\"IsSuccess\":true,\"AntIntima\":{\"IntimaThick\":0.0,\"IsSuccess\":false},\"PostIntima\":{\"IntimaThick\":0.75,\"IsSuccess\":true}},\"IsYImageSuccess\":false,\"PlaqueResult\":{\"PlaquePostion\":0,\"PlaqueCountType\":2,\"PlaqueType\":0,\"PlaqueWidth\":1.75,\"PlaqueHeight\":3.76,\"Stenosis\":10.37,\"IsSuccess\":true}}";
var measureResult = JsonConvert.DeserializeObject(leftItem.CarotidResult.MeasureResult) ?? new CarotidAIMeasureResult();
if (measureResult.PlaqueResult?.PlaqueCountType != Carotid.Utilities.DetectPlaque.PlaqueCountType.NoPlaque)
{
leftData = leftItem;
result.DiagnosisConclusion = DiagnosisConclusionEnum.Other;
break;
}
}
if (leftData == null)
{
leftData = leftExamDatas.First();
}
}
if (leftData != null)
{
result.CarotidResult.Add(leftData);
}
//right
float intimaThickStandard = 1.0f;
DiagnosisRemicalDTO rightData = null;
var rightExamDatas = orderedExamDatas.Where(m => m.CarotidResult.CarotidScanType == CarotidScanTypeEnum.CarotidRight);
if (rightExamDatas.Any())
{
foreach (var rightItem in rightExamDatas)
{
if (string.IsNullOrEmpty(rightItem.CarotidResult.MeasureResult))
{
result.DiagnosisConclusion = DiagnosisConclusionEnum.Unrecognized;
continue;
}
//rightItem.CarotidResult.MeasureResult = "{\"IntimaResult\":{\"IsSuccess\":true,\"AntIntima\":{\"IntimaThick\":0.0,\"IsSuccess\":false},\"PostIntima\":{\"IntimaThick\":0.75,\"IsSuccess\":true}},\"IsYImageSuccess\":false,\"PlaqueResult\":{\"PlaquePostion\":0,\"PlaqueCountType\":2,\"PlaqueType\":0,\"PlaqueWidth\":1.75,\"PlaqueHeight\":3.76,\"Stenosis\":10.37,\"IsSuccess\":true}}";
var measureResult = JsonConvert.DeserializeObject(rightItem.CarotidResult.MeasureResult) ?? new CarotidAIMeasureResult();
if (measureResult.IntimaResult?.PostIntima?.IntimaThick > intimaThickStandard)
{
rightData = rightItem;
result.DiagnosisConclusion = DiagnosisConclusionEnum.Other;
break;
}
}
if (rightData == null)
{
rightData = rightExamDatas.First();
}
}
if (rightData != null)
{
result.CarotidResult.Add(rightData);
}
Logger.WriteLineInfo($"AIDiagnosisService package carotidAIMeasureResult finished, CarotidLeftRemedicalCode:{leftData?.RemedicalCode}, CarotidLeft:{leftData?.CarotidResult?.MeasureResult}, CarotidRightRemedicalCode:{rightData?.RemedicalCode}, CarotidRight:" + rightData?.CarotidResult?.MeasureResult);
}
return result;
}
private async Task CreateDiagnosisManagerAsync(DiagnosisReportRequest request)
{
var recordResults = new List();
foreach (var remedical in request.RemedicalList)
{
foreach (var perImage in remedical.DiagnosisResult)
{
var localFile = await DownloadAsync(remedical.FileToken);
if (File.Exists((localFile)))
{
var diagnosisPerImage = new DiagnosisPerImageModel();
diagnosisPerImage.RemedicalCode = remedical.RemedicalCode;
diagnosisPerImage.DataType = (WingAIDiagnosisService.Manage.DataType)remedical.DataType;
diagnosisPerImage.LocalVidPath = localFile;
diagnosisPerImage.Index = perImage.Index;
diagnosisPerImage.PriorityScore = perImage.PriorityScore;
var diagnosisResults = Newtonsoft.Json.JsonConvert.DeserializeObject>(Newtonsoft.Json.JsonConvert.SerializeObject(perImage.DiagResultsForEachOrgan));
diagnosisPerImage.DiagResultsForEachOrgan = diagnosisResults;
recordResults.Add(diagnosisPerImage);
}
}
}
switch (request.Organ)
{
case DiagnosisOrganEnum.Breast:
return new BreastDiagnosis(recordResults, request.Organ);
case DiagnosisOrganEnum.Liver:
return new LiverDiagnosis(recordResults, request.Organ);
case DiagnosisOrganEnum.Thyroid://甲状腺
return new ThyroidDiagnosis(recordResults, request.Organ);
default:
throw new Exception($"not support organ type:{request.Organ.ToString()}");
}
}
private async Task CarotidDiagnosis(VinnoImageData imageData)
{
var resampleInputData = new ResampleInputData();
resampleInputData.SurfaceFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.SurfaceFileSuffix);
resampleInputData.MdlFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.ModelFileSuffix);
resampleInputData.MdlZipFilePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.ZipFileSuffix);
resampleInputData.BaseAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix);
resampleInputData.PlaqueAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix);
resampleInputData.YShapeAIImagePath = DiagnosisHelper.GetCacheFilePath(DiagnosisHelper.JPGFileSuffix);
var resampleResult = TryResample(imageData, resampleInputData);
if (resampleResult.ResampleErrorCode == ResampleErrorCode.Success)
{
var carotidAIImageTokens = new List();
if (resampleResult.CarotidAIMeasureResult.IsYImageSuccess)
{
var fileUrl = await UploadFileAsync(resampleInputData.YShapeAIImagePath, Path.GetFileName(resampleInputData.YShapeAIImagePath));
if (!string.IsNullOrWhiteSpace(fileUrl))
{
carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.YShape });
}
File.Delete(resampleInputData.YShapeAIImagePath);
}
if (resampleResult.CarotidAIMeasureResult.IntimaResult.IsSuccess)
{
var fileUrl = await UploadFileAsync(resampleInputData.BaseAIImagePath, Path.GetFileName(resampleInputData.BaseAIImagePath));
if (!string.IsNullOrWhiteSpace(fileUrl))
{
carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.Base });
}
File.Delete(resampleInputData.BaseAIImagePath);
}
if (resampleResult.CarotidAIMeasureResult.PlaqueResult.IsSuccess && resampleResult.CarotidAIMeasureResult.PlaqueResult.PlaqueCountType != WingAIDiagnosisService.Carotid.Utilities.DetectPlaque.PlaqueCountType.NoPlaque)
{
var fileUrl = await UploadFileAsync(resampleInputData.PlaqueAIImagePath, Path.GetFileName(resampleInputData.PlaqueAIImagePath));
if (!string.IsNullOrWhiteSpace(fileUrl))
{
carotidAIImageTokens.Add(new CarotidAIImage() { AIImageToken = fileUrl, AIImageType = CarotidAIImageType.Plaque });
}
File.Delete(resampleInputData.PlaqueAIImagePath);
}
var surfaceImageList = new List();
long surfaceFileSize = 0;
var cdnSurfaceFile = "";
if (!string.IsNullOrWhiteSpace(resampleInputData.SurfaceFilePath) && File.Exists(resampleInputData.SurfaceFilePath))
{
surfaceImageList = await SurfaceFile(resampleInputData.SurfaceFilePath);
var surfaceFileTuple = await UploadFileTupleAsync(resampleInputData.SurfaceFilePath, Path.GetFileName(resampleInputData.SurfaceFilePath));
surfaceFileSize = surfaceFileTuple.Item2;
File.Delete(resampleInputData.SurfaceFilePath);
resampleInputData.SurfaceFilePath = surfaceFileTuple.Item1;
cdnSurfaceFile = surfaceFileTuple.Item1.ToUrlToken();//ToCDN
}
long mdlFileFileSize = 0;
var cdnMdlFile = "";
if (!string.IsNullOrWhiteSpace(resampleInputData.MdlFilePath) && File.Exists(resampleInputData.MdlFilePath))
{
var mdlFileTuple = await UploadFileTupleAsync(resampleInputData.MdlFilePath, Path.GetFileName(resampleInputData.MdlFilePath));
mdlFileFileSize = mdlFileTuple.Item2;
File.Delete(resampleInputData.MdlFilePath);
resampleInputData.MdlFilePath = mdlFileTuple.Item1;
cdnMdlFile = mdlFileTuple.Item1.ToUrlToken();//ToCDN
}
var result = new CarotidResultDTO
{
CarotidScanType = (CarotidScanTypeEnum)resampleInputData.ScanType,
CarotidScanDirection = (CarotidScanDirectionEnum)resampleInputData.CarotidScanDirection,
SurfaceFile = resampleInputData.SurfaceFilePath,
SurfaceFileSize = surfaceFileSize,
CDNSurfaceFile = cdnSurfaceFile,
MdlFileSize = mdlFileFileSize,
MdlFile = resampleInputData.MdlFilePath,
CDNMdlFile = cdnMdlFile,
MeasureImageFiles = carotidAIImageTokens.Select(x => new MeasureImageFileDTO { ImageType = (CarotidAIImageTypeEnum)x.AIImageType, ImageFile = x.AIImageToken }).ToList(),
MeasureResult = JsonConvert.SerializeObject(resampleResult.CarotidAIMeasureResult),
SurfaceImageList = surfaceImageList,
};
return result;
}
return null;
}
private ResampleResult TryResample(VinnoImageData vinnoImageData, ResampleInputData resampleInputData)
{
float scanDistance = 7;
ResampleResult resampleResult = new ResampleResult(ResampleErrorCode.Fail);
var vinnoExtendedData = VinnoCarotidExtendedData.FromBytes(vinnoImageData.ExtendedData);
CarotidScanType scanType = CarotidScanType.CarotidLeft;
CarotidScanDirection direction = CarotidScanDirection.TopToBottom;
if (vinnoExtendedData != null)
{
// Should be 6~9cm normally.
//scanDistance = vinnoExtendedData.ScanDistance;
scanType = vinnoExtendedData.CarotidType == CarotidType.Left ? CarotidScanType.CarotidLeft : CarotidScanType.CarotidRight;
direction = vinnoExtendedData.CarotidDirection == CarotidDirection.BottomToTop ? CarotidScanDirection.BottomToTop
: CarotidScanDirection.TopToBottom;
}
else
{
//This is a walkaround : vid's first frame is black image or depth coordinate is not in the right place.
var middleCount = vinnoImageData.ImageCount / 2;
var vinnoImage = vinnoImageData.GetImage(middleCount);
var imageBuffer = vinnoImage.ImageData;
if (_recognizeCarotidTypeCanBeUse)
{
//转成彩色图像
using var img = new Mat();
CvInvoke.Imdecode(imageBuffer, ImreadModes.Color, img);
using var image = img.ToImage();
//识别左右颈信息
scanType = _recognizeCarotidType.RecognizeType(image);
}
}
resampleInputData.ScanDistance = scanDistance;
resampleInputData.ScanType = scanType;
resampleInputData.CarotidScanDirection = direction;
resampleResult = ResampleModel.Instance.Resample(vinnoImageData, resampleInputData, _workerLevel);
return resampleResult;
}
///
///
///
///
private async Task> SurfaceFile(string filePath)
{
var fileUrls = new List();
using (var stream = new FileStream(filePath, FileMode.Open))
{
var reader = new Carotid3DStreamReader(stream);
var width = reader.ReadInt();
var height = reader.ReadInt();
var depth = reader.ReadInt();
var physicalData = VinnoCarotid3DPhysicalData.FromBytes(reader.ReadBytes());
var imageCount = reader.ReadInt();
var fileName = $"{Guid.NewGuid():N}";
for (var i = 0; i < imageCount; i++)
{
var readBytes = reader.ReadBytes();
if (readBytes.Any())
{
using var img = new Mat();
CvInvoke.Imdecode(readBytes, ImreadModes.Color, img);
var buf = new Emgu.CV.Util.VectorOfByte();
using var image = img.ToImage();
CvInvoke.Imencode(".jpg", image, buf, new KeyValuePair(ImwriteFlags.JpegQuality, 80));
var jpgByte = buf.ToArray();
var localPath = Path.Combine(_tempFolder, $"{fileName}_{i + 1}.jpg");
File.WriteAllBytes(localPath, jpgByte);
var fileUrl = await UploadFileAsync(localPath, Path.GetFileName(localPath));
fileUrls.Add(fileUrl);
File.Delete(localPath);
}
}
}
return fileUrls;
}
private SKBitmap CreateBitmap(VinnoImage vinnoImage)
{
try
{
return SKBitmap.Decode(vinnoImage.ImageData);
}
catch (Exception ex)
{
Logger.WriteLineError($"Create skbitmap by VinnoImage error:{ex}");
}
return null;
}
private EnumColorType MapTo(SKColorType sKColor)
{
switch (sKColor)
{
case SKColorType.Rgba8888:
return EnumColorType.Rgba;
case SKColorType.Rgb888x:
return EnumColorType.Rgba;
case SKColorType.Bgra8888:
return EnumColorType.Bgra;
case SKColorType.Gray8:
return EnumColorType.Gray8;
}
throw new Exception($"AIService not support color type:{sKColor}");
}
private void AIdiagSystem_NotifyError(object sender, ErrorEventArgs e)
{
Logger.WriteLineError("AIdiagSystem_NotifyError:" + e.GetException());
}
private void AIdiagSystem_NotifyLog(object sender, LogEventArgs e)
{
if (e != null && !string.IsNullOrEmpty(e.Msg))
{
switch (e.LogType)
{
case EnumLogType.InfoLog:
Logger.WriteLineInfo($"AIdiagSystem_NotifyLog:{e.Msg}");
break;
case EnumLogType.ErrorLog:
Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}");
break;
case EnumLogType.WarnLog:
Logger.WriteLineWarn($"AIdiagSystem_NotifyLog:{e.Msg}");
break;
case EnumLogType.FatalLog:
Logger.WriteLineError($"AIdiagSystem_NotifyLog:{e.Msg}");
break;
default:
Logger.WriteLineInfo(e.Msg);
break;
}
}
}
private DiagnosisConclusion GetDiagnosisConclusion(List results) where T : AIDiagnosisPerImageModel
{
var diagnosisConclusions = new List();
foreach (var imageResult in results)
{
foreach (var diagnosisResult in imageResult.DiagResultsForEachOrgan)
{
var benignLabels = new List();
var malignantLabels = new List();
if (diagnosisResult.Organ == DiagnosisOrganEnum.Breast)
{
benignLabels = new List { 1, 2, 3 };
malignantLabels = new List { 4, 5, 6, 7 };
}
else if (diagnosisResult.Organ == DiagnosisOrganEnum.Liver)
{
benignLabels = new List { 1, 2, 3, 5, 6, 7, 8 };
malignantLabels = new List { 4 };
}
else if (diagnosisResult.Organ == DiagnosisOrganEnum.Thyroid)
{
benignLabels = new List { 1, 2, 7 };
malignantLabels = new List { 3, 4, 5, 6 };
}
var labels = diagnosisResult.DetectedObjects.Select(x => x.Label);
if (labels.Contains(0))
{
diagnosisConclusions.Add(DiagnosisConclusion.NoObviousLesion);
}
if (labels.Intersect(benignLabels).Any())
{
diagnosisConclusions.Add(DiagnosisConclusion.Benign);
}
if (labels.Intersect(malignantLabels).Any())
{
diagnosisConclusions.Add(DiagnosisConclusion.Malignant);
}
}
}
var containsBenign = diagnosisConclusions.Contains(DiagnosisConclusion.Benign);
var containsMalignant = diagnosisConclusions.Contains(DiagnosisConclusion.Malignant);
var containsNoObviousLesion = diagnosisConclusions.Contains(DiagnosisConclusion.NoObviousLesion);
if (containsBenign && containsMalignant)
{
return DiagnosisConclusion.BenignAndMalignant;
}
else if (containsBenign)
{
return DiagnosisConclusion.Benign;
}
else if (containsMalignant)
{
return DiagnosisConclusion.Malignant;
}
else if (containsNoObviousLesion)
{
return DiagnosisConclusion.NoObviousLesion;
}
else
{
return DiagnosisConclusion.Unrecognized;
}
}
private List GetDiagnosisOrgans(List results)
{
var diagnosisOrgans = new List();
foreach (var imageResult in results)
{
foreach (var diagnosisResult in imageResult.DiagResultsForEachOrgan)
{
var organName = Enum.GetName(typeof(DiagnosisOrganEnum), (int)diagnosisResult.Organ);
if (!string.IsNullOrWhiteSpace(organName) && diagnosisResult.Organ != DiagnosisOrganEnum.Null && diagnosisResult.DetectedObjects?.Any() == true)
{
diagnosisOrgans.Add((DiagnosisOrganEnum)diagnosisResult.Organ);
}
}
}
return diagnosisOrgans.Distinct().ToList();
}
private Dictionary NormalDiagnosis(VinnoImageData imageData)
{
var images = new List();
var results = new Dictionary();
try
{
var totalCount = imageData.ImageCount;
for (var i = 0; i < totalCount; i++)
{
var image = imageData.GetImage(i);
var bitmap = CreateBitmap(image);
if (bitmap != null)
{
images.Add(new RawImage(bitmap.Bytes, bitmap.Width, bitmap.Height, MapTo(bitmap.ColorType)));
}
}
if (images.Count > 0)
{
var excuteCount = images.Count / _batchImageSize;
var remainderCount = images.Count % _batchImageSize;
var diagId = _diagSystem.StartEvalutationOfMultipleImageBatches(images.Count);
for (var j = 0; j < excuteCount; j++)
{
var excuteImages = images.GetRange(j * _batchImageSize, _batchImageSize);
_diagSystem.PushOneBatchOfImagesAsync(diagId, excuteImages);
Thread.Sleep(_sleepTime);
}
if (remainderCount > 0)
{
var excuteImages = images.GetRange(excuteCount * _batchImageSize, remainderCount);
_diagSystem.PushOneBatchOfImagesAsync(diagId, excuteImages);
}
results = _diagSystem.GetEvaluationsOfPushedMultipleImageBatches(diagId);
}
}
catch (Exception ex)
{
Logger.WriteLineWarn($"AIService NormalDiagnosis err, {ex}");
}
finally
{
foreach (var image in images)
{
image?.Dispose();
}
}
return results;
}
///
/// 保存AI诊断结果
///
///
///
///
///
private async Task AddDiagnosisResultInfosAsync(string relationCode, string fileUrl, List diagnosisResultInfos)
{
try
{
var resultInfos = new List();
for (var i = 0; i < diagnosisResultInfos.Count; i++)
{
resultInfos.Add(new DiagnosisResultDTO
{
Index = i,
DiagnosisResult = Newtonsoft.Json.JsonConvert.SerializeObject(diagnosisResultInfos[i]),
});
}
var addRequest = new AddDiagnosisResultInfosDBRequest
{
RelationCode = relationCode,
FileUrl = fileUrl,
DiagnosisResultInfos = resultInfos,
};
_diagnosisResultDBService.AddDiagnosisResultInfosAsync(addRequest);
return true;
}
catch (Exception ex)
{
Logger.WriteLineWarn($"AIDiagnosisService AddDiagnosisResultInfosAsync err, ex:{ex}");
return false;
}
}
/// 下载文件
///
///
private async Task DownloadAsync(string fileUrl)
{
try
{
if (string.IsNullOrEmpty(fileUrl))
{
return string.Empty;
}
if (!Directory.Exists(_tempFolder))
{
Directory.CreateDirectory(_tempFolder);
}
var fileName = Path.GetFileName(fileUrl);
var tempFile = Path.Combine(_tempFolder, fileName);
if (File.Exists(tempFile))
{
return tempFile;
}
long fileSize = 0;
using (var request = new HttpRequestMessage())
{
request.RequestUri = new Uri(fileUrl);
request.Method = HttpMethod.Get;
var response = await _httpClient.SendAsync(request);
if (response != null && response.StatusCode == HttpStatusCode.OK)
{
var contentLength = response.Content.Headers.ContentLength;
fileSize = contentLength == null ? 0 : contentLength.Value;
}
}
if (fileSize <= 0)
{
throw new NotSupportedException($"fileSize is {fileSize}");
}
byte[] bytes = await _httpClient.GetByteArrayAsync(fileUrl);
File.WriteAllBytes(tempFile, bytes);
return tempFile;
}
catch (Exception ex)
{
Logger.WriteLineWarn($"DiagnosisService download file err, url: {fileUrl}, {ex}");
}
finally
{
//Logger.WriteLineInfo($"download file:{fileUrl}");
}
return string.Empty;
}
///
/// 上传文件
///
///
///
///
private async Task UploadFileAsync(string filePath, string fileName)
{
var fileToken = "";
using (var fileStream = new FileStream(filePath, FileMode.Open))
{
var size = fileStream.Length;
byte[] buffer = new byte[fileStream.Length];
fileStream.Read(buffer, 0, buffer.Length);
fileToken = await DoUploadFile(fileName, buffer);
}
return fileToken;
}
///
/// 上传文件
///
///
///
///
private async Task> UploadFileTupleAsync(string filePath, string fileName)
{
var fileToken = "";
long size = 0;
using (var fileStream = new FileStream(filePath, FileMode.Open))
{
size = fileStream.Length;
byte[] buffer = new byte[fileStream.Length];
fileStream.Read(buffer, 0, buffer.Length);
fileToken = await DoUploadFile(fileName, buffer);
}
return new Tuple(fileToken, size);
}
///
/// 上传文件
///
///
///
///
async Task DoUploadFile(string fileName, byte[] fileData)
{
var requestHeads = new Dictionary();
var defaultToken = await _authenticationService.GetServerDefaultTokenAsync();
var authorizationRet = await _storageService.GetAuthorizationAsync(
new FileServiceRequest
{
Token = defaultToken,
FileName = fileName,
});
requestHeads.Add("Authorization", authorizationRet.Authorization);
var fileUrl = authorizationRet.StorageUrl;
Logger.WriteLineInfo($"DoUploadFile fileUrl:{fileUrl}");
using (var request = new HttpRequestMessage())
{
var fileExtension = Path.GetExtension(fileName);
var mimeType = FileHelper.GetMimeType(fileExtension);
var contentType = MediaTypeHeaderValue.Parse(mimeType);
using (UploadContent content = new UploadContent(fileData, contentType))
{
request.RequestUri = new Uri(fileUrl);
request.Method = HttpMethod.Put;
request.Content = content;
foreach (var head in requestHeads)
{
request.Headers.TryAddWithoutValidation(head.Key, head.Value);
}
var result = await ExecuteRequest(request);
if (!result)
{
throw new Exception("Upload file failed!");
}
}
}
return fileUrl;
}
private UploadFileTypeEnum GetUploadFileType(string fileName)
{
if (fileName.EndsWith(".vid"))
{
return UploadFileTypeEnum.VID;
}
else if (fileName.EndsWith(".jpg"))
{
return UploadFileTypeEnum.JPG;
}
else
{
return UploadFileTypeEnum.MP4;
}
}
///
/// 执行请求
///
///
///
///
public async Task ExecuteRequest(HttpRequestMessage httpRequestMessage)
{
try
{
var response = await _httpClient.SendAsync(httpRequestMessage);
if (response != null && response.StatusCode == HttpStatusCode.OK)
{
return true;
}
return false;
}
catch (Exception ex)
{
throw ex;
}
}
private List PointToDto(Point2D[] points)
{
var dstContours = new List();
foreach (var p in points)
{
dstContours.Add(new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D
{
X = p.X,
Y = p.Y
});
}
return dstContours;
}
private List KeyPointToDto(KeyPointInfo[] points)
{
var dstKeys = new List();
foreach (var point in points)
{
dstKeys.Add(new DiagnosisKeyPointDTO
{
Type = (DiagnosisKeyPointType)point.Type,
IndexInContour = point.IndexInContour,
Point = new WingInterfaceLibrary.DTO.Comment.AIDiagnosisPoint2D
{
X = point.Point.X,
Y = point.Point.Y,
},
});
}
return dstKeys;
}
}
}