FISLib/Vinno.vCloud.FIS.CrossPlatform.Android/LiveVideo/CombineImageSender.cs

using Android.Graphics;
using System;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Vinno.vCloud.FIS.CrossPlatform.Common;
using Vinno.vCloud.FIS.CrossPlatform.Common.Enum;
using Vinno.vCloud.FIS.CrossPlatform.Common.LiveVideo;

namespace Vinno.vCloud.FIS.CrossPlatform.Android.LiveVideo
{
    internal class CombineImageSender : IImageSender
    {
        private TerminalImageSender _deviceImageSender;
        private CameraHelper _cameraHelper;
        private Action<EnumLiveStates> _updateState;
        private byte[] _screenTempBuffer;
        private byte[] _screenTempPushBuffer;
        private byte[] _cameraTempBuffer;
        private byte[] _cameraTempPushBuffer;
        private byte[] _combineTempPushBuffer;
        private byte[] _combineTempPushBufferI420;

        private int _channelCount;

        private Bitmap _tempScreenBitmap;
        private Bitmap _tempCameraBitmap;
        private Bitmap _tempCombineBitmap;
        private Canvas _tempCanvas;

        private int _outputScreenHeight;
        private int _outputScreenWidth;
        private int _cameraHeight;
        private int _cameraWidth;

        private string _cameraId;
        private bool _isStarted;

        private object _screenLock = new object();
        private object _cameraLock = new object();
        private int _lastSendDataTime = 0;
        private int _finalOutputHeight;
        private int _finalOutputWidth;
        private EnumLiveProtocol _liveProtocol;

        /// <summary>
        /// 实际上ARGB用于RTMP,而I420用于RTC
        /// </summary>
        public AndroidImageType ImageType { get; private set; }

        public event EventHandler<CPVideoFrameData> ImageSended;

        public CombineImageSender(int outputScreenWidth, int outputScreenHeight, int cameraWidth, int cameraHeight, string cameraId, EnumLiveProtocol liveProtocol, Action<EnumLiveStates> updateState = null, AndroidImageType imageType = AndroidImageType.ARGB)
        {
            _channelCount = 4;
            ImageType = imageType;
            _outputScreenHeight = outputScreenHeight;
            _outputScreenWidth = outputScreenWidth;
            _cameraWidth = cameraWidth;
            _cameraHeight = cameraHeight;
            _updateState = updateState;
            _cameraId = cameraId;
            _liveProtocol = liveProtocol;
            switch (ImageType)
            {
                case AndroidImageType.ARGB:
                    _finalOutputWidth = outputScreenWidth + cameraWidth;
                    _finalOutputHeight = outputScreenHeight > cameraHeight ? outputScreenHeight : cameraHeight;
                    _tempCameraBitmap = Bitmap.CreateBitmap(_cameraWidth, _cameraHeight, Bitmap.Config.Argb8888);
                    _tempScreenBitmap = Bitmap.CreateBitmap(_outputScreenWidth, _outputScreenHeight, Bitmap.Config.Argb8888);
                    _tempCombineBitmap = Bitmap.CreateBitmap(_finalOutputWidth, _finalOutputHeight, Bitmap.Config.Argb8888);
                    _tempCanvas = new Canvas(_tempCombineBitmap);
                    _tempCanvas.Save();
                    ResizeBuffer(ref _cameraTempBuffer, _cameraHeight * _cameraWidth * 4);
                    ResizeBuffer(ref _combineTempPushBuffer, _finalOutputWidth * _finalOutputHeight * _channelCount);
                    break;

                case AndroidImageType.I420:
                    _finalOutputHeight = 720;
                    _finalOutputWidth = 1280;
                    _tempCameraBitmap = Bitmap.CreateBitmap(_cameraWidth, _cameraHeight, Bitmap.Config.Argb8888);
                    _tempScreenBitmap = Bitmap.CreateBitmap(_outputScreenWidth, _outputScreenHeight, Bitmap.Config.Argb8888);
                    _tempCombineBitmap = Bitmap.CreateBitmap(_finalOutputWidth, _finalOutputHeight, Bitmap.Config.Argb8888);
                    _tempCanvas = new Canvas(_tempCombineBitmap);
                    _tempCanvas.Save();
                    ResizeBuffer(ref _cameraTempBuffer, _cameraHeight * _cameraWidth * 4);
                    ResizeBuffer(ref _combineTempPushBuffer, _finalOutputWidth * _finalOutputHeight * _channelCount);
                    ResizeBuffer(ref _combineTempPushBufferI420, _finalOutputWidth * _finalOutputHeight * 3 / 2);
                    break;
            }
        }

        public void SetImageType(AndroidImageType imageType)
        {
            ImageType = imageType;
        }

        public async void Start()
        {
            try
            {
                if (!_isStarted)
                {
                    if (_cameraHelper == null)
                    {
                        _cameraHelper = new CameraHelper(_cameraWidth, _cameraHeight, _cameraId, _updateState);
                    }
                    if (_cameraHelper != null)
                    {
                        _cameraHelper.ImageSended += OnCaptureImageChanged;
                        _cameraHelper.Start();
                    }
                    if (_deviceImageSender == null)
                    {
                        _deviceImageSender = new TerminalImageSender(_outputScreenWidth, _outputScreenHeight, _liveProtocol, _updateState);
                    }
                    if (_deviceImageSender != null)
                    {
                        _deviceImageSender.ImageSended += OnDeviceImageSended;
                        _deviceImageSender.Start();
                    }
                    _isStarted = true;
                    await Task.Run(() =>
                    {
                        while (_isStarted)
                        {
                            try
                            {
                                if (Math.Abs(_lastSendDataTime - DateTime.Now.Millisecond) <= 40)
                                {
                                    continue;
                                }
                                else
                                {
                                    _lastSendDataTime = DateTime.Now.Millisecond;
                                }
                                lock (_screenLock)
                                {
                                    if (_screenTempBuffer == null)
                                    {
                                        Thread.Sleep(10);
                                        continue;
                                    }
                                    _tempCanvas.Save();
                                    ResizeBuffer(ref _screenTempPushBuffer, _screenTempBuffer.Length);
                                    Buffer.BlockCopy(_screenTempBuffer, 0, _screenTempPushBuffer, 0, _screenTempBuffer.Length);
                                }
                                lock (_cameraLock)
                                {
                                    ResizeBuffer(ref _cameraTempPushBuffer, _cameraTempBuffer.Length);
                                    Buffer.BlockCopy(_cameraTempBuffer, 0, _cameraTempPushBuffer, 0, _cameraTempBuffer.Length);
                                }
                                switch (ImageType)
                                {
                                    case AndroidImageType.ARGB:
                                        Marshal.Copy(_screenTempPushBuffer, 0, _tempScreenBitmap.LockPixels(), _screenTempPushBuffer.Length);
                                        _tempScreenBitmap.UnlockPixels();
                                        Marshal.Copy(_cameraTempPushBuffer, 0, _tempCameraBitmap.LockPixels(), _cameraTempPushBuffer.Length);
                                        _tempCameraBitmap.UnlockPixels();
                                        _tempCanvas.DrawBitmap(_tempScreenBitmap, 0, 0, new Paint());
                                        _tempCanvas.DrawBitmap(_tempCameraBitmap, _outputScreenWidth, 0, new Paint());
                                        Marshal.Copy(_tempCombineBitmap.LockPixels(), _combineTempPushBuffer, 0, _combineTempPushBuffer.Length);
                                        _tempCombineBitmap.UnlockPixels();
                                        _tempCanvas.Restore();
                                        ImageSended?.Invoke(this, new CPVideoFrameData(_finalOutputWidth, _finalOutputHeight, _combineTempPushBuffer) { ImageData = new CPVideoFrameData(_cameraWidth, _cameraHeight, _cameraTempPushBuffer) });
                                        break;

                                    case AndroidImageType.I420:
                                        Marshal.Copy(_screenTempPushBuffer, 0, _tempScreenBitmap.LockPixels(), _screenTempPushBuffer.Length);
                                        _tempScreenBitmap.UnlockPixels();
                                        _tempCanvas.DrawBitmap(_tempScreenBitmap, 0, 0, new Paint());
                                        Marshal.Copy(_cameraTempPushBuffer, 0, IntPtr.Add(_tempCombineBitmap.LockPixels(), 1280 * _outputScreenHeight * _channelCount), _cameraTempPushBuffer.Length); //640*480
                                        Marshal.Copy(_tempCombineBitmap.LockPixels(), _combineTempPushBuffer, 0, _combineTempPushBuffer.Length);
                                        _tempCombineBitmap.UnlockPixels();
                                        _tempCanvas.Restore();
                                        YuvHelper.ConvertABGRToI420(_combineTempPushBuffer, _finalOutputWidth, _finalOutputHeight, _combineTempPushBufferI420);
                                        ImageSended?.Invoke(this, new CPVideoFrameData(_finalOutputWidth, _finalOutputHeight, _combineTempPushBufferI420) { ImageData = new CPVideoFrameData(_cameraWidth, _cameraHeight, _cameraTempPushBuffer) });
                                        break;
                                }
                            }
                            catch (Exception ex)
                            {
                                CrossPlatformHelper.Instance.LogWriter?.WriteLineError($"CombineImageSended Error:{ex}");
                            }
                        }
                    });
                }
            }
            catch (Exception ex)
            {
                CrossPlatformHelper.Instance.LogWriter?.WriteLineError($"Start Combine Image Sender Error:{ex}");
            }
        }

        private void ResizeBuffer(ref byte[] buffer, int size)
        {
            if (buffer == null)
            {
                buffer = new byte[size];
            }
            else if (buffer.Length != size)
            {
                Array.Resize(ref buffer, size);
            }
        }

        private void OnDeviceImageSended(object sender, CPVideoFrameData e)
        {
            try
            {
                if (e == null || e.Width != _outputScreenWidth || e.Height != _outputScreenHeight)
                {
                    throw new ArgumentOutOfRangeException($"OutputScreenWidth Should be {_outputScreenWidth},Actually:{e?.Width},OutputScreenHeight Should be {_outputScreenHeight},Actually:{e?.Height}");
                }

                var dataSize = _outputScreenHeight * _outputScreenWidth * 4;
                lock (_screenLock)
                {
                    ResizeBuffer(ref _screenTempBuffer, dataSize);
                    Buffer.BlockCopy(e.Data, 0, _screenTempBuffer, 0, dataSize);
                }
            }
            catch (Exception ex)
            {
                _updateState?.Invoke(EnumLiveStates.TerminalPushFailed);
                CrossPlatformHelper.Instance.LogWriter?.WriteLineError($" OnDeviceImageSended error, ex:{ex}");
            }
        }

        private void OnCaptureImageChanged(object sender, CPVideoFrameData e)
        {
            try
            {
                if (e == null || e.Width != _cameraWidth || e.Height != _cameraHeight)
                {
                    throw new ArgumentOutOfRangeException($"CameraWidth Should be {_cameraWidth},Actually:{e?.Width},CameraHeight Should be {_cameraHeight},Actually:{e?.Height}");
                }

                var dataSize = _cameraHeight * _cameraWidth * 4;
                lock (_cameraLock)
                {
                    ResizeBuffer(ref _cameraTempBuffer, dataSize);
                    Buffer.BlockCopy(e.Data, 0, _cameraTempBuffer, 0, dataSize);
                }
            }
            catch (Exception ex)
            {
                _updateState?.Invoke(EnumLiveStates.TerminalPushFailed);
                CrossPlatformHelper.Instance.LogWriter?.WriteLineError($" OnDeviceImageSended error, ex:{ex}");
            }
        }

        public void Stop()
        {
            try
            {
                if (_isStarted)
                {
                    if (_deviceImageSender != null)
                    {
                        _deviceImageSender.ImageSended -= OnDeviceImageSended;
                        _deviceImageSender.Stop();
                        _deviceImageSender = null;
                    }
                    if (_cameraHelper != null)
                    {
                        _cameraHelper.ImageSended -= OnCaptureImageChanged;
                        _cameraHelper.Stop();
                        _cameraHelper = null;
                    }
                    _isStarted = false;
                    CrossPlatformHelper.Instance.LogWriter?.WriteLineInfo($"Combine Image Sender stoped handle");
                }
            }
            catch (Exception ex)
            {
                CrossPlatformHelper.Instance.LogWriter?.WriteLineError($"Stop CombineImageSender Failed:{ex}");
                _isStarted = false;
            }
        }

        public void SwitchCamera()
        {
            if (_cameraHelper != null)
            {
                _cameraHelper.SwitchCamera();
            }
        }
    }
}