vStationURM/Common/Vinno.vCloud.Common.Storage.Standard/Upload/VCloudStorageUploader.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Vinno.IUS.Common.IO;
using Vinno.IUS.Common.Log;
using Vinno.IUS.Common.Network.Leaf;
using Vinno.IUS.Common.Network.Tcp;
using Vinno.IUS.Common.Utilities;
using Vinno.vCloud.Common.Storage.Upload;
using Vinno.vCloud.Protocol.Infrastructures;
using Vinno.vCloud.Protocol.Messages.Client.Storage;

namespace Vinno.vCloud.Common.Storage
{
    public class VCloudStorageUploader : IUploader
    {
        /// <summary>
        ///when  app  running ,set cache folder path 
        /// </summary>        
        private readonly List<IUploadStorageFileInfo> UploadStorageFileInfoList =
            new List<IUploadStorageFileInfo>();
        private const int MinBlockSize = 1048576; //4Mb
        private const int BlockSize = 4194304; //4mb

        /// <summary>
        /// Upload one file and return the file token.
        /// </summary>
        /// <param name="creator">The TcpCrerator for newtwork connection</param>
        /// <param name="fileData">The file data to be uploaded</param>
        /// <param name="progress">The upload progress</param>
        /// <param name="cancelTokenSource">The token to cancel the upload</param>
        /// <returns>The file token in server</returns>
        public string UploadFile(StorageInfo storageInfo, byte[] fileData, Action<double> progress = null, CancellationTokenSource cancelTokenSource = null, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func<string, string> GetAuthentication = null, Func<string, List<KeyValuePair<string, string>>, string> GetCopyAuthentication = null)
        {
            var storageUrl = storageInfo.Url;
            if (string.IsNullOrEmpty(storageUrl)) { throw new Exception("Storage url is empty"); }
            var creator = new TcpCreator(storageUrl);
            var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
            try
            {
                using (var stream = new MemoryStream(fileData))
                {
                    if (stream.Length <= BlockSize)
                    {
                        var size = (int)stream.Length;
                        var data = new byte[size];
                        stream.Read(data, 0, size);
                        var buffer = new ByteBuffer(data);
                        var token = storageLeaf.ApplyToken();
                        var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer });
                        var storeFileSuccess = StoreFileSuccess.Convert(result);
                        if (storeFileSuccess != null)
                        {
                            progress?.Invoke(1.0);
                            return storeFileSuccess.FileToken;
                        }
                    }
                    else
                    {
                        var size = (int)stream.Length;
                        var count = size / BlockSize;
                        var totalCount = size % BlockSize != 0 ? count + 1 : count;
                        for (int i = 0; i < count; i++)
                        {
                            if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                            {
                                return null;
                            }
                            var data = new byte[BlockSize];
                            stream.Read(data, 0, BlockSize);
                            var buffer = new ByteBuffer(data);
                            var token = storageLeaf.ApplyToken();
                            var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount });
                            var storeFileSuccess = StoreFileSuccess.Convert(result);
                            if (storeFileSuccess == null)
                            {
                                throw new InvalidOperationException("Store file error");
                            }
                            progress?.Invoke(((double)i + 1) / totalCount);
                            var isDone = totalCount == count && i == count - 1;
                            size -= BlockSize;
                            if (isDone)
                            {
                                return storeFileSuccess.FileToken;
                            }
                        }
                        if (size > 0)
                        {
                            if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                            {
                                return null;
                            }
                            var data = new byte[size];
                            stream.Read(data, 0, size);
                            var buffer = new ByteBuffer(data);
                            var token = storageLeaf.ApplyToken();
                            var result = storageLeaf.Send(token, new StoreFileRequest
                            { FileData = buffer, Index = totalCount - 1, Count = totalCount });
                            var storeFileSuccess = StoreFileSuccess.Convert(result);
                            if (storeFileSuccess == null)
                            {
                                throw new InvalidOperationException("Store file error");
                            }
                            progress?.Invoke(1.0);
                            return storeFileSuccess.FileToken;
                        }
                    }
                }
            }
            finally
            {
                storageLeaf.Close();
            }
            throw new InvalidOperationException("Store file failed.");
        }

        /// <summary>
        /// new upload file(Breakpoint continuingly)
        /// </summary>
        /// <param name="creator"></param>
        /// <param name="filePath"></param>
        /// <param name="accountId">current account id</param>
        /// <param name="progress"> UI action progress</param>
        /// <param name="cancelTokenSource"></param>
        /// <returns>file token</returns>
        public string UploadFile(StorageInfo storageInfo, string filePath, string accountId,
            Action<double> progress = null,
            CancellationTokenSource cancelTokenSource = null,bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func<string, string> GetAuthentication = null, Func<string, List<KeyValuePair<string, string>>, string> GetCopyAuthentication = null)
        {
            var storageUrl = storageInfo.Url;
            var creator = new TcpCreator(storageUrl);
            var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
            try
            {
                if (!storageLeaf.Online)
                {
                    throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}");
                }
                using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
                {
                    Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin");
                    var uploadFileToken = UploadFileFromStream(stream,
                        accountId, progress, storageLeaf, cancelTokenSource);
                    return uploadFileToken;
                }
            }
            finally
            {
                storageLeaf.Close();
            }
        }

        /// <summary>
        /// new upload file(Breakpoint continuingly)
        /// </summary>
        /// <param name="filePath">The upload file path</param>
        /// <param name="fileName">The file name in file server.</param>
        /// <param name="accountId">The account id</param>
        /// <param name="storageInfo">The storage info.</param>
        /// <param name="progress"> UI action progress</param>
        /// <param name="cancelTokenSource"></param>
        /// <returns>file token</returns>
        public string UploadFile(StorageInfo storageInfo, string filePath, string fileName, string accountId, Action<double> progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl, bool isCopyFileToOtherNode, Func<string, string> GetAuthentication, Func<string, List<KeyValuePair<string, string>>, string> GetCopyAuthentication = null)
        {
            var storageUrl = storageInfo.Url;
            var creator = new TcpCreator(storageUrl);
            var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
            try
            {
                if (!storageLeaf.Online)
                {
                    throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}");
                }
                using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
                {
                    Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin");
                    var uploadFileToken = UploadFileFromStream(fileName,stream,
                         progress, storageLeaf, cancelTokenSource);
                    return uploadFileToken;
                }
            }
            finally
            {
                storageLeaf.Close();
            }
        }
        /// <summary>
        /// Upload file to server by  stream 
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="accountId"></param>
        /// <param name="progress"></param>
        /// <param name="storageLeaf"></param>
        /// <param name="cancelTokenSource"></param>
        /// <returns>file token</returns>
        private string UploadFileFromStream(string fileName,Stream stream,Action<double> progress, ClientLeaf storageLeaf,
            CancellationTokenSource cancelTokenSource = null)
        {
            progress?.Invoke(0.01);
            var fileSize = (int)stream.Length;
            var md5 = MD5.GetHashString(stream);
            var currentUploadFileInfo = new UploadStorageFileInfo()
            {
                CacheFileName = fileName,
                FilePosition = 0,
                FileSize = fileSize,
                Id = string.Empty,
                MD5 = md5
            };
            while (fileSize > currentUploadFileInfo.FilePosition + 1)
            {
                if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                {
                    //upload file canceled 
                    return null;
                }
                var token = storageLeaf.ApplyToken();
                var createuploadRequest = CreateUploadRequest(stream, "", currentUploadFileInfo, fileSize);
                var result = storageLeaf.Send(token, createuploadRequest);
                var storeFileResult = UploadStoreFileResult.Convert(result);

                if (storeFileResult != null)
                {
                    var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
                    currentUploadFileInfo.Id = storeFileInfoMessage.Id;
                    currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
                    currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
                    if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken))
                    {
                        // upload file success
                        currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
                        progress?.Invoke(1.0);
                        return currentUploadFileInfo.FileToken;
                    }

                    progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
                }
                else
                {
                    throw new InvalidOperationException("Store file error");
                }
            }
            return currentUploadFileInfo.FileToken;
        }



        /// <summary>
        /// Upload file to server by  stream 
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="accountId"></param>
        /// <param name="progress"></param>
        /// <param name="storageLeaf"></param>
        /// <param name="cancelTokenSource"></param>
        /// <returns>file token</returns>
        private string UploadFileFromStream(Stream stream,
            string accountId, Action<double> progress, ClientLeaf storageLeaf,
            CancellationTokenSource cancelTokenSource = null)
        {
            progress?.Invoke(0.01);
            var fileSize = (int)stream.Length;
            var md5 = MD5.GetHashString(stream);
            var currentUploadFileInfo = new UploadStorageFileInfo()
            {
                CacheFileName = string.Empty,
                FilePosition = 0,
                FileSize = fileSize,
                Id = string.Empty,
                MD5 = md5
            };
            currentUploadFileInfo.FileSize = fileSize;
            while (fileSize > currentUploadFileInfo.FilePosition + 1)
            {
                if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                {
                    //upload file canceled 
                    return null;
                }
                var token = storageLeaf.ApplyToken();
                var createuploadRequest = CreateUploadRequest(stream, accountId, currentUploadFileInfo, fileSize);
                var result = storageLeaf.Send(token, createuploadRequest);
                var storeFileResult = UploadStoreFileResult.Convert(result);

                if (storeFileResult != null)
                {
                    var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
                    currentUploadFileInfo.Id = storeFileInfoMessage.Id;
                    currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
                    currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
                    if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken))
                    {
                        // upload file success
                        currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
                        progress?.Invoke(1.0);
                        return currentUploadFileInfo.FileToken;
                    }

                    progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
                }
                else
                {
                    throw new InvalidOperationException("Store file error");
                }
            }
            return currentUploadFileInfo.FileToken;
        }

        private async Task<string> UploadFileFromStreamAsync(Stream stream,
            string accountId, Action<double> progress, ClientLeaf storageLeaf,
            CancellationTokenSource cancelTokenSource = null)
        {
            progress?.Invoke(0.01);
            var fileSize = (int)stream.Length;
            var md5 = MD5.GetHashString(stream);
            var currentUploadFileInfo = UploadStorageFileInfoList.FirstOrDefault(v => v.MD5 == md5);
            if (currentUploadFileInfo == null)
            {
                currentUploadFileInfo = new UploadStorageFileInfo()
                {
                    CacheFileName = string.Empty,
                    FilePosition = 0,
                    FileSize = fileSize,
                    Id = string.Empty,
                    MD5 = md5
                };
                UploadStorageFileInfoList.Add(currentUploadFileInfo);
            }

            currentUploadFileInfo.FileSize = fileSize;

            while (fileSize > currentUploadFileInfo.FilePosition + 1)
            {
                if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                {
                    //upload file canceled 
                    return null;
                }
                var token = storageLeaf.ApplyToken();
                var result = await storageLeaf.SendAsync(token, CreateUploadRequest(
                    stream, accountId, currentUploadFileInfo, fileSize));
                var storeFileResult = UploadStoreFileResult.Convert(result);
                if (storeFileResult != null)
                {
                    var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
                    currentUploadFileInfo.Id = storeFileInfoMessage.Id;
                    currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
                    currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
                    currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
                    if (!string.IsNullOrEmpty(currentUploadFileInfo.FileToken))
                    {
                        // upload file success
                        progress?.Invoke(1.0);
                        return currentUploadFileInfo.FileToken;
                    }

                    progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
                }
                else
                {
                    throw new InvalidOperationException("Store file error");
                }
                Thread.Sleep(10);
            }

            return currentUploadFileInfo.FileToken;
        }

        private UploadStoreFileRequest CreateUploadRequest(Stream stream, string accountId,
            IUploadStorageFileInfo currentUploadFileInfo, int fileSize)
        {
            Logger.WriteLineInfo($"UploadStoreFile CreateUploadRequest begin");
            //set this upload file end position byte
            var lastPosition = currentUploadFileInfo.FilePosition + BlockSize - 1;
            var fileSizePostion = fileSize - 1;
            if (lastPosition > fileSizePostion)
            {
                int offet = fileSizePostion - currentUploadFileInfo.FilePosition;
                if (offet > MinBlockSize)
                {
                    lastPosition = currentUploadFileInfo.FilePosition + MinBlockSize;
                }
                else
                {
                    lastPosition = fileSizePostion;
                }
            }

            if (lastPosition == fileSizePostion)
            {
                lastPosition = fileSizePostion;
            }

            //set this upload file byte size
            var blockSize = lastPosition - currentUploadFileInfo.FilePosition + 1;
            stream.Position = currentUploadFileInfo.FilePosition;
            var data = new byte[blockSize];
            stream.Read(data, 0, blockSize);
            var buffer = new ByteBuffer(data);
            Logger.WriteLineInfo($"UploadStoreFile blockSize: {blockSize} filePosition:{currentUploadFileInfo.FilePosition}");
            // upload to server
            var uploadmessage = new ClientUploadStorageFileInfoMessage
            {
                FileData = buffer,
                Id = currentUploadFileInfo.Id,
                FileMd5 = currentUploadFileInfo.MD5,
                AccountId = accountId ?? string.Empty,
                FileSize = currentUploadFileInfo.FileSize,
                FilePosition = currentUploadFileInfo.FilePosition,
                FileToken = currentUploadFileInfo.FileToken,
                CacheFileName = currentUploadFileInfo.CacheFileName
            };
            return new UploadStoreFileRequest { ClientUploadStorageFileInfoMessage = uploadmessage };
        }
        
        /// <summary>
        /// upload file async
        /// </summary>
        /// <param name="storageUrl"></param>
        /// <param name="fileData"></param>
        /// <param name="progress"></param>
        /// <param name="cancelTokenSource"></param>
        /// <returns></returns>
        public async Task<string> UploadFileAsync(StorageInfo storageInfo, byte[] fileData, Action<double> progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func<string, string> GetAuthentication = null, Func<string, List<KeyValuePair<string, string>>, string> GetCopyAuthentication = null)
        {
            var storageUrl = storageInfo.Url;
            if (string.IsNullOrEmpty(storageUrl)) { throw new Exception($"Storage url is empty, {storageInfo}"); }
            var creator = new TcpCreator(storageUrl);
            var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
            try
            {
                using (var stream = new MemoryStream(fileData))
                {
                    if (stream.Length <= BlockSize)
                    {
                        var size = (int)stream.Length;
                        var data = new byte[size];
                        await stream.ReadAsync(data, 0, size);
                        var buffer = new ByteBuffer(data);
                        var token = storageLeaf.ApplyToken();
                        var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer });
                        var storeFileSuccess = StoreFileSuccess.Convert(result);
                        if (storeFileSuccess != null)
                        {
                            progress?.Invoke(1.0);
                            return storeFileSuccess.FileToken;
                        }
                    }
                    else
                    {
                        var size = (int)stream.Length;
                        var count = size / BlockSize;
                        var totalCount = size % BlockSize != 0 ? count + 1 : count;
                        for (int i = 0; i < count; i++)
                        {
                            if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                            {
                                return null;
                            }
                            var data = new byte[BlockSize];
                            await stream.ReadAsync(data, 0, BlockSize);
                            var buffer = new ByteBuffer(data);
                            var token = storageLeaf.ApplyToken();
                            var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount });
                            var storeFileSuccess = StoreFileSuccess.Convert(result);
                            if (storeFileSuccess == null)
                            {
                                throw new InvalidOperationException("Store file error");
                            }
                            progress?.Invoke(((double)i + 1) / totalCount);
                            var isDone = totalCount == count && i == count - 1;
                            size -= BlockSize;
                            if (isDone)
                            {
                                return storeFileSuccess.FileToken;
                            }
                        }
                        if (size > 0)
                        {
                            if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
                            {
                                return null;
                            }
                            var data = new byte[size];
                            stream.Read(data, 0, size);
                            var buffer = new ByteBuffer(data);
                            var token = storageLeaf.ApplyToken();
                            var result = await storageLeaf.SendAsync(token, new StoreFileRequest
                            { FileData = buffer, Index = totalCount - 1, Count = totalCount });
                            var storeFileSuccess = StoreFileSuccess.Convert(result);
                            if (storeFileSuccess == null)
                            {
                                throw new InvalidOperationException("Store file error");
                            }
                            progress?.Invoke(1.0);
                            return storeFileSuccess.FileToken;
                        }
                    }
                }
            }
            finally
            {
                storageLeaf.Close();
            }
            throw new InvalidOperationException("Store file failed.");
        }

        /// <summary>
        /// new upload file(Breakpoint continuingly)
        /// </summary>
        /// <param name="filePath">The upload file path</param>
        /// <param name="fileName">The file name in file server.</param>
        /// <param name="storageInfo">The storage info.</param>
        /// <returns>file token</returns>
        public List<string> CopydFile(StorageInfo storageInfo, string filePath, string fileName, string destinationServer, Func<string, List<KeyValuePair<string, string>>, string> GetCopyAuthentication = null)
        {
            return new List<string>();
        }
    }
}