using System;
using System.Linq;
using System.Runtime.InteropServices;
namespace YOLODetectProcessLib
{
///
/// 模型中图像数据的轴排列顺序
///
public enum EnumAxisOrder
{
CHW,
HWC,
}
///
/// 将原始图像转换为模型输入时所采用的resize方式
///
public enum EnumResizeMode
{
///
/// 以长边为准,将原始图像等比例缩放,短边用固定灰度值填充
///
FitLargeSizeAndPad,
///
/// 以短边为准,将原始图像等比例缩放,长边直接裁切掉不要
///
FitSmallSizeAndCrop,
///
/// 非等比例缩放,长短边各自伸缩到模型所需的尺寸
///
Warp,
}
///
/// 减均值的时候,均值的计算方式
///
public enum EnumMeanValueType
{
///
/// 不用减均值
///
None,
///
/// 减均值(固定值)
///
ConstantMean,
///
/// 减均值(用当前图像的所有灰度计算一个均值)
///
CalcMeanImage,
///
/// 减均值(用当前图像的每个通道单独计算均值,减的时候也是每通道分别减)
///
CalcMeanPerChannel,
}
///
/// 除以scale的时候,scale的计算方式
///
public enum EnumScaleValueType
{
///
/// 不用除以scale
///
None,
///
/// 除以scale(固定值)
///
ConstantScale,
///
/// 除以scale(用当前图像的所有灰度计算一个标准差,作为scale)
///
CalcStdImage,
///
/// 除以scale(用当前图像的每个通道单独计算标准差,除的时候也是每个通道分别除)
///
CalcStdPerChannel,
}
///
/// 归一化的方法
///
public enum EnumNormalizationType
{
///
/// 不用做归一化
///
None,
///
/// 用当前图像的最大最小值做归一化(val'=(val-min)/(max-min))
///
MinMaxScalingImage,
///
/// 用当前图像每个通道的最大最小值做归一化
///
MinMaxScalingPerChannel,
}
///
/// 颜色通道的顺序
///
public enum EnumChannelOrder
{
None,
RGB,
BGR,
}
///
/// 根据模型需要的尺寸和resize参数,处理后的图像
/// 可直接导入模型
///
public class MoldedImage : IDisposable
{
#region private
private readonly int _netImgH;
private readonly int _netImgW;
private readonly int _netImgC;
private readonly EnumResizeMode _resizeMode;
private readonly EnumMeanValueType _meanValueType;
private readonly float _meanR;
private readonly float _meanG;
private readonly float _meanB;
private readonly EnumScaleValueType _scaleValueType;
private readonly float _scaleR;
private readonly float _scaleG;
private readonly float _scaleB;
private readonly EnumNormalizationType _normalizationType;
private readonly bool _reverseInputChannels;
private readonly EnumAxisOrder _axisOrder;
private readonly bool _useContoursAsMask;
private IImage _resizedImage = null;
private IImage _croppedROIImage = null;
private IImage _moldedImage = null;
private byte[] _padValue = null;
private float[] _dataBuffer = null;
#endregion
#region property
///
/// 图像数据
///
public float[] DataBuffer => _dataBuffer;
#endregion
#region public funcs
///
/// 释放
///
public void Dispose()
{
DoDispose();
GC.SuppressFinalize(this);
LogHelper.InfoLog("MoldedImage.Disposed manually.");
}
///
/// 析构函数
///
~MoldedImage()
{
DoDispose();
LogHelper.InfoLog("MoldedImage.Disposed by destructor.");
}
///
/// 对输入图像进行处理
///
///
public MoldedImageMetas Process(InferenceNetworkInputImage image)
{
// _dataBuffer的尺寸是否需要改变
int dataBufferLength = _netImgH * _netImgW * _netImgC;
if (dataBufferLength != _dataBuffer.Length)
{
Array.Resize(ref _dataBuffer, dataBufferLength);
}
// 传入原始图像相关信息
Rect roiRect;
roiRect = image.ROI;
MoldedImageMetas imageMeta = new MoldedImageMetas
{
NetImgInputHeight = _netImgH,
NetImgInputWidth = _netImgW,
NetImgInputChannels = _netImgC,
OrigImgHeight = image.Image.Height,
OrigImgWidth = image.Image.Width,
ROIRectHeight = roiRect.Height,
ROIRectWidth = roiRect.Width,
ROIRectTop = roiRect.Top,
ROIRectLeft = roiRect.Left,
};
// 是否需要用contour作mask
IImage imageToResize;
imageToResize = image.Image;
// 对原始图像进行缩放
CalcResizeImageMetas(ref imageMeta);
DoResizeCppFix(imageToResize, imageMeta);
// 将Resize过后的图像像素转成所需的格式
if (imageMeta.NetImgInputChannels == 1)
{
ExtractRGB24AsGrayCpp(0);
}
else
{
ExtractRGB24AsColorCpp(0);
}
return imageMeta;
}
#endregion
#region constructor
///
/// 构造函数
///
///
///
///
///
///
///
///
///
///
///
///
///
///
///
public MoldedImage(int netImgH, int netImgW, int netImgC, EnumResizeMode resizeMode,
EnumMeanValueType meanType, float meanR, float meanG, float meanB,
EnumScaleValueType scaleType, float scaleR, float scaleG, float scaleB,
EnumNormalizationType normType, EnumChannelOrder channelOrder, EnumAxisOrder axisOrder)
{
if (netImgC != 3 && netImgC != 1)
{
throw new ArgumentOutOfRangeException("netImgC", "The expected channel number is 1 or 3, but got " + netImgC + ".");
}
if (netImgH <= 0)
{
throw new ArgumentOutOfRangeException("netImgH", "The expected network height is greater than 0, but got " + netImgH + ".");
}
if (netImgW <= 0)
{
throw new ArgumentOutOfRangeException("netImgW", "The expected network width is greater than 0, but got " + netImgW + ".");
}
if (scaleType == EnumScaleValueType.ConstantScale && (Math.Abs(scaleR) < 1e-10))
{
throw new ArgumentOutOfRangeException("scaleR", "The expected scaleR is greater than 1e-10 or less than -1e10 when devideScale is true," +
" but got " + scaleR + ".");
}
if (scaleType == EnumScaleValueType.ConstantScale && (Math.Abs(scaleG) < 1e-10))
{
throw new ArgumentOutOfRangeException("scaleG", "The expected scaleG is greater than 1e-10 or less than -1e10 when devideScale is true," +
" but got " + scaleR + ".");
}
if (scaleType == EnumScaleValueType.ConstantScale && (Math.Abs(scaleB) < 1e-10))
{
throw new ArgumentOutOfRangeException("scaleG", "The expected scaleB is greater than 1e-10 or less than -1e10 when devideScale is true," +
" but got " + scaleR + ".");
}
_netImgH = netImgH;
_netImgW = netImgW;
_netImgC = netImgC;
_resizeMode = resizeMode;
_meanValueType = meanType;
_meanR = meanR;
_meanG = meanG;
_meanB = meanB;
_scaleValueType = scaleType;
_scaleR = scaleR;
_scaleG = scaleG;
_scaleB = scaleB;
_normalizationType = normType;
_reverseInputChannels = (channelOrder == EnumChannelOrder.BGR);
_axisOrder = axisOrder;
_resizedImage = new RawImage(new byte[netImgH * netImgH * 3], netImgW, netImgH, 3, netImgW * 3);
_croppedROIImage = new RawImage(new byte[netImgH * netImgW * 3], netImgW, netImgH, 3, netImgW * 3);
_moldedImage = new RawImage(new byte[netImgH * netImgW * 3], netImgW, netImgH, 3, netImgW * 3);
_padValue = new byte[3];
_padValue[0] = (byte)_meanB;
_padValue[1] = (byte)_meanG;
_padValue[2] = (byte)_meanR;
_dataBuffer = new float[netImgH * netImgW * netImgC];
}
#endregion
#region private funcs
///
/// 计算Resize所需的imageMetas
///
///
private void CalcResizeImageMetas(ref MoldedImageMetas imageMetas)
{
// 先求出缩放所需的参数
double scaleW = (double)imageMetas.ROIRectWidth / imageMetas.NetImgInputWidth;
double scaleH = (double)imageMetas.ROIRectHeight / imageMetas.NetImgInputHeight;
int width;
int height;
int startind;
switch (_resizeMode)
{
case EnumResizeMode.FitLargeSizeAndPad:
// 保持原来的长宽比,以长边为准,短边填充
if (scaleW > scaleH)
{
scaleH = scaleW;
width = imageMetas.NetImgInputWidth;
height = (int)Math.Round(imageMetas.ROIRectHeight / scaleH);
}
else
{
scaleW = scaleH;
width = (int)Math.Round(imageMetas.ROIRectWidth / scaleW);
height = imageMetas.NetImgInputHeight;
}
imageMetas.MoldLeft = (imageMetas.NetImgInputWidth - width) / 2;
imageMetas.MoldTop = (imageMetas.NetImgInputHeight - height) / 2;
imageMetas.MoldWidth = width;
imageMetas.MoldHeight = height;
break;
case EnumResizeMode.FitSmallSizeAndCrop:
// 保持原来的长宽比,以短边为准,长边居中裁切
if (scaleW > scaleH)
{
scaleW = scaleH;
int correctedRoIWidth = (int)Math.Round(imageMetas.NetImgInputWidth * scaleW);
startind = (imageMetas.ROIRectWidth - correctedRoIWidth) / 2;
imageMetas.ROIRectLeft = imageMetas.ROIRectLeft + startind;
imageMetas.ROIRectWidth = correctedRoIWidth;
}
else
{
scaleH = scaleW;
int correctedRoIHeight = (int)Math.Round(imageMetas.NetImgInputHeight * scaleH);
startind = (imageMetas.ROIRectHeight - correctedRoIHeight) / 2;
imageMetas.ROIRectTop = imageMetas.ROIRectTop + startind;
imageMetas.ROIRectHeight = correctedRoIHeight;
}
imageMetas.MoldLeft = 0;
imageMetas.MoldTop = 0;
imageMetas.MoldWidth = imageMetas.NetImgInputWidth;
imageMetas.MoldHeight = imageMetas.NetImgInputHeight;
break;
case EnumResizeMode.Warp:
// 不管长宽比,直接长短边各自resize成所需的尺寸
imageMetas.MoldLeft = 0;
imageMetas.MoldTop = 0;
imageMetas.MoldWidth = imageMetas.NetImgInputWidth;
imageMetas.MoldHeight = imageMetas.NetImgInputHeight;
break;
}
}
///
/// 对原始图像进行缩放
///
private void DoResize(IImage image, MoldedImageMetas imageMetas)
{
// 先在resizedImage上填充背景色
ImageUtils.FillPadVal(_resizedImage, _padValue);
// 取出原始图像种的指定范围
var roiRectInSrc = ImageUtils.CropRect(image, new Rect(imageMetas.ROIRectLeft, imageMetas.ROIRectTop,
imageMetas.ROIRectWidth, imageMetas.ROIRectHeight));
// 再将其resize成指定尺寸
var molded = ImageUtils.ResizeBilinear(roiRectInSrc, imageMetas.MoldWidth, imageMetas.MoldHeight);
// 再将其贴到resizedImage上指定位置
ImageUtils.PasteRect(_resizedImage, molded, new Rect(imageMetas.MoldLeft, imageMetas.MoldTop,
imageMetas.MoldWidth, imageMetas.MoldHeight));
// 销毁
roiRectInSrc.Dispose();
molded.Dispose();
}
///
/// 对原始图像进行缩放
///
private void DoResizeFix(RawImage image, MoldedImageMetas imageMetas)
{
// 先在resizedImage上填充背景色
ImageUtils.FillPadVal(_resizedImage, _padValue);
// 取出原始图像种的指定范围
int croppedROIImageDataBufferSize = imageMetas.ROIRectWidth * imageMetas.ROIRectHeight * 3;
if (_croppedROIImage.DataBuffer.Length < croppedROIImageDataBufferSize)
{
_croppedROIImage.SetDataBufferSize(croppedROIImageDataBufferSize);
}
var croppedROIImageData = _croppedROIImage.DataBuffer;
ImageUtils.CropRect(image, new Rect(imageMetas.ROIRectLeft, imageMetas.ROIRectTop,
imageMetas.ROIRectWidth, imageMetas.ROIRectHeight), ref croppedROIImageData);
_croppedROIImage.Width = imageMetas.ROIRectWidth;
_croppedROIImage.Height = imageMetas.ROIRectHeight;
_croppedROIImage.Stride = imageMetas.ROIRectWidth * 3;
_croppedROIImage.DataBufferSize = croppedROIImageDataBufferSize;
// 再将其resize成指定尺寸
int moldedImageDataBufferSize = imageMetas.MoldWidth * imageMetas.MoldHeight * 3;
if (_moldedImage.DataBuffer.Length < moldedImageDataBufferSize)
{
_moldedImage.SetDataBufferSize(moldedImageDataBufferSize);
}
var moldedImageData = _moldedImage.DataBuffer;
ImageUtils.ResizeBilinear(_croppedROIImage, imageMetas.MoldWidth, imageMetas.MoldHeight, ref moldedImageData);
_moldedImage.Width = imageMetas.MoldWidth;
_moldedImage.Height = imageMetas.MoldHeight;
_moldedImage.Stride = imageMetas.MoldWidth * 3;
_moldedImage.DataBufferSize = moldedImageDataBufferSize;
// 再将其贴到resizedImage上指定位置
ImageUtils.PasteRect(_resizedImage, _moldedImage, new Rect(imageMetas.MoldLeft, imageMetas.MoldTop,
imageMetas.MoldWidth, imageMetas.MoldHeight));
}
///
/// 对原始图像进行缩放(用cpp dll中提供的函数)
///
///
///
private void DoResizeCpp(IImage image, MoldedImageMetas imageMetas)
{
// 先在resizedImage上填充背景色
ImageUtils.FillPadValWithCpp(_resizedImage, _padValue);
// 取出原始图像种的指定范围
var roiRectInSrc = ImageUtils.CropRectWithCpp(image, new Rect(imageMetas.ROIRectLeft, imageMetas.ROIRectTop,
imageMetas.ROIRectWidth, imageMetas.ROIRectHeight));
// 再将其resize成指定尺寸
var molded = ImageUtils.ResizeBilinearWithCpp(roiRectInSrc, imageMetas.MoldWidth, imageMetas.MoldHeight);
// 再将其贴到resizedImage上指定位置
ImageUtils.PasteRectWithCpp(_resizedImage, molded, new Rect(imageMetas.MoldLeft, imageMetas.MoldTop,
imageMetas.MoldWidth, imageMetas.MoldHeight));
// 销毁
roiRectInSrc.Dispose();
molded.Dispose();
}
///
/// 对原始图像进行缩放(用cpp dll中提供的函数,且固定使用某些内存区域)
///
///
///
private void DoResizeCppFix(IImage image, MoldedImageMetas imageMetas)
{
// 先在resizedImage上填充背景色
ImageUtils.FillPadValWithCpp(_resizedImage, _padValue);
// 取出原始图像种的指定范围
// 注意:由于CalcResizeParams中,当ResizeMode为FitSmallSizeAndCrop时,ROIRect的位置可能会发生改变
// 所以g.DrawImage时srcRect不要用InputImage里的ROIRect,而是现在的
byte[] croppedROIImageData = _croppedROIImage.DataBuffer;
int newSizeROI = imageMetas.ROIRectWidth * imageMetas.ROIRectHeight * 3;
if (croppedROIImageData == null || croppedROIImageData.Length < newSizeROI)
{
_croppedROIImage.SetDataBufferSize(newSizeROI);
croppedROIImageData = _croppedROIImage.DataBuffer;
}
ImageUtils.CropRectWithCpp(image, new Rect(imageMetas.ROIRectLeft, imageMetas.ROIRectTop,
imageMetas.ROIRectWidth, imageMetas.ROIRectHeight), ref croppedROIImageData);
_croppedROIImage.Width = imageMetas.ROIRectWidth;
_croppedROIImage.Height = imageMetas.ROIRectHeight;
// 再将其resize成指定尺寸
byte[] moldedImageData = _moldedImage.DataBuffer;
int newSizeMolded = imageMetas.MoldWidth * imageMetas.MoldHeight * 3;
if (moldedImageData == null || moldedImageData.Length < newSizeMolded)
{
_moldedImage.SetDataBufferSize(newSizeMolded);
moldedImageData = _moldedImage.DataBuffer;
}
ImageUtils.ResizeBilinearWithCpp(_croppedROIImage, imageMetas.MoldWidth, imageMetas.MoldHeight, ref moldedImageData);
_moldedImage.Width = imageMetas.MoldWidth;
_moldedImage.Height = imageMetas.MoldHeight;
// 再将其贴到resizedImage上指定位置
ImageUtils.PasteRectWithCpp(_resizedImage, _moldedImage, new Rect(imageMetas.MoldLeft, imageMetas.MoldTop,
imageMetas.MoldWidth, imageMetas.MoldHeight));
}
///
/// 将原始图像转成单通道的数组(用cpp dll中提供的函数)
/// (如果需要做减均值和Scale等操作,也在这一步完成)
///
private void ExtractRGB24AsGrayCpp(int startInd)
{
GCHandle hObjectSrc = GCHandle.Alloc(_resizedImage.DataBuffer, GCHandleType.Pinned);
IntPtr pDataSrc = hObjectSrc.AddrOfPinnedObject();
StructImageInfo srcImgInfo = new StructImageInfo
{
width = _resizedImage.Width,
height = _resizedImage.Height,
channel = _resizedImage.Channel,
pData = pDataSrc,
};
GCHandle hObjectDst = GCHandle.Alloc(_dataBuffer, GCHandleType.Pinned);
IntPtr pDataDst = hObjectDst.AddrOfPinnedObject();
pDataDst = IntPtr.Add(pDataDst, startInd * sizeof(float));
StructExtractInfo extractInfo = new StructExtractInfo
{
meanValueType = (int)_meanValueType,
meanR = _meanR,
meanG = _meanG,
meanB = _meanB,
scaleValueType = (int)_scaleValueType,
scaleR = _scaleR,
scaleG = _scaleG,
scaleB = _scaleB,
normalizationType = (int)_normalizationType,
reverseInputChannels = _reverseInputChannels,
axisOrder = (int)_axisOrder,
};
var ret = ImageUtils.ExtractRGB24AsGray(srcImgInfo, pDataDst, extractInfo);
if (hObjectSrc.IsAllocated)
{
hObjectSrc.Free();
}
if (hObjectDst.IsAllocated)
{
hObjectDst.Free();
}
if (ret != 0)
{
throw new Exception("Failed at calling cpp func: ExtractRGB24AsGray.(error code: " + ret + ").");
}
}
///
/// 将原始图像转成三维数组(用cpp dll中提供的函数)
/// 根据axisOrder决定到底是CHW还是HWC的数据排列顺序
/// (如果需要做减均值和Scale等操作,也在这一步完成)
///
private void ExtractRGB24AsColorCpp(int startInd)
{
GCHandle hObjectSrc = GCHandle.Alloc(_resizedImage.DataBuffer, GCHandleType.Pinned);
IntPtr pDataSrc = hObjectSrc.AddrOfPinnedObject();
StructImageInfo srcImgInfo = new StructImageInfo
{
width = _resizedImage.Width,
height = _resizedImage.Height,
channel = _resizedImage.Channel,
pData = pDataSrc,
};
GCHandle hObjectDst = GCHandle.Alloc(_dataBuffer, GCHandleType.Pinned);
IntPtr pDataDst = hObjectDst.AddrOfPinnedObject();
pDataDst = IntPtr.Add(pDataDst, startInd * sizeof(float));
StructExtractInfo extractInfo = new StructExtractInfo
{
meanValueType = (int)_meanValueType,
meanR = _meanR,
meanG = _meanG,
meanB = _meanB,
scaleValueType = (int)_scaleValueType,
scaleR = _scaleR,
scaleG = _scaleG,
scaleB = _scaleB,
normalizationType = (int)_normalizationType,
reverseInputChannels = _reverseInputChannels,
axisOrder = (int)_axisOrder,
};
var ret = ImageUtils.ExtractRGB24AsColor(srcImgInfo, pDataDst, extractInfo);
if (hObjectSrc.IsAllocated)
{
hObjectSrc.Free();
}
if (hObjectDst.IsAllocated)
{
hObjectDst.Free();
}
if (ret != 0)
{
throw new Exception("Failed at calling cpp func: ExtractRGB24AsColor.(error code: " + ret + ").");
}
}
///
/// 销毁
///
private void DoDispose()
{
_resizedImage?.Dispose();
_resizedImage = null;
_croppedROIImage?.Dispose();
_croppedROIImage = null;
_moldedImage?.Dispose();
_moldedImage = null;
}
///
/// 计算均值
///
///
///
///
///
private void CalcMean(out float meanR, out float meanG, out float meanB, out float meanImg)
{
meanR = 0;
meanG = 0;
meanB = 0;
int imgSize = _resizedImage.Width * _resizedImage.Height;
byte[] resizedImageDataBuffer = _resizedImage.DataBuffer;
for (int ni = 0; ni < imgSize; ni++)
{
int offset = 3 * ni;
meanB += resizedImageDataBuffer[offset];
meanG += resizedImageDataBuffer[offset + 1];
meanR += resizedImageDataBuffer[offset + 2];
}
meanR /= imgSize;
meanG /= imgSize;
meanB /= imgSize;
meanImg = PixelRGBToGray(meanR, meanG, meanB);
}
///
/// 计算方差
///
///
///
///
///
private void CalcStd(float meanR, float meanG, float meanB, float meanImg,
out float stdR, out float stdG, out float stdB, out float stdImg)
{
double sumOfSquareR = 0;
double sumOfSquareG = 0;
double sumOfSquareB = 0;
double sumOfSquareImg = 0;
int imgSize = _resizedImage.Width * _resizedImage.Height;
byte[] resizedImageDataBuffer = _resizedImage.DataBuffer;
for (int ni = 0; ni < imgSize; ni++)
{
int offset = 3 * ni;
byte b = resizedImageDataBuffer[offset];
byte g = resizedImageDataBuffer[offset + 1];
byte r = resizedImageDataBuffer[offset + 2];
sumOfSquareB += Math.Pow(b - meanB, 2);
sumOfSquareG += Math.Pow(g - meanG, 2);
sumOfSquareR += Math.Pow(r - meanR, 2);
sumOfSquareImg += (Math.Pow(r - meanImg, 2) + Math.Pow(g - meanImg, 2) + Math.Pow(b - meanImg, 2));
}
stdR = (float)Math.Sqrt(sumOfSquareR / imgSize);
stdG = (float)Math.Sqrt(sumOfSquareG / imgSize);
stdB = (float)Math.Sqrt(sumOfSquareB / imgSize);
stdImg = (float)Math.Sqrt(sumOfSquareImg / (imgSize * 3));
}
///
/// 计算最大最小值
///
///
///
///
///
///
///
///
///
private void CalcMinMax(out float minR, out float maxR, out float minG, out float maxG,
out float minB, out float maxB, out float minImg, out float maxImg)
{
minR = float.MaxValue;
minG = float.MaxValue;
minB = float.MaxValue;
maxR = float.MinValue;
maxG = float.MinValue;
maxB = float.MinValue;
int imgSize = _resizedImage.Width * _resizedImage.Height;
byte[] resizedImageDataBuffer = _resizedImage.DataBuffer;
for (int ni = 0; ni < imgSize; ni++)
{
int offset = 3 * ni;
byte b = resizedImageDataBuffer[offset];
byte g = resizedImageDataBuffer[offset + 1];
byte r = resizedImageDataBuffer[offset + 2];
if (r < minR)
{
minR = r;
}
if (r > maxR)
{
maxR = r;
}
if (g < minG)
{
minG = g;
}
if (g > maxG)
{
maxG = g;
}
if (b < minB)
{
minB = b;
}
if (b > maxB)
{
maxB = b;
}
}
minImg = resizedImageDataBuffer.Min();
maxImg = resizedImageDataBuffer.Max();
}
///
/// 将单个像素的RGB值转换为灰度值
///
///
///
///
///
private float PixelRGBToGray(float Rvalue, float Gvalue, float Bvalue)
{
return (float)0.299 * Rvalue + (float)0.587 * Gvalue + (float)0.114 * Bvalue;
}
#endregion
}
}