using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Vinno.IUS.Common.IO; using Vinno.IUS.Common.Log; using Vinno.IUS.Common.Network.Leaf; using Vinno.IUS.Common.Network.Tcp; using Vinno.IUS.Common.Utilities; using Vinno.vCloud.Common.Storage.Upload; using Vinno.vCloud.Protocol.Infrastructures; using Vinno.vCloud.Protocol.Messages.Client.Storage; namespace Vinno.vCloud.Common.Storage { public class VCloudStorageUploader : IUploader { /// ///when app running ,set cache folder path /// private readonly List UploadStorageFileInfoList = new List(); private const int MinBlockSize = 1048576; //4Mb private const int BlockSize = 4194304; //4mb /// /// Upload one file and return the file token. /// /// The TcpCrerator for newtwork connection /// The file data to be uploaded /// The upload progress /// The token to cancel the upload /// The file token in server public string UploadFile(StorageInfo storageInfo, byte[] fileData, Action progress = null, CancellationTokenSource cancelTokenSource = null, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null) { var storageUrl = storageInfo.Url; if (string.IsNullOrEmpty(storageUrl)) { throw new Exception("Storage url is empty"); } var creator = new TcpCreator(storageUrl); var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator); try { using (var stream = new MemoryStream(fileData)) { if (stream.Length <= BlockSize) { var size = (int)stream.Length; var data = new byte[size]; stream.Read(data, 0, size); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess != null) { progress?.Invoke(1.0); return storeFileSuccess.FileToken; } } else { var size = (int)stream.Length; var count = size / BlockSize; var totalCount = size % BlockSize != 0 ? count + 1 : count; for (int i = 0; i < count; i++) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { return null; } var data = new byte[BlockSize]; stream.Read(data, 0, BlockSize); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess == null) { throw new InvalidOperationException("Store file error"); } progress?.Invoke(((double)i + 1) / totalCount); var isDone = totalCount == count && i == count - 1; size -= BlockSize; if (isDone) { return storeFileSuccess.FileToken; } } if (size > 0) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { return null; } var data = new byte[size]; stream.Read(data, 0, size); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer, Index = totalCount - 1, Count = totalCount }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess == null) { throw new InvalidOperationException("Store file error"); } progress?.Invoke(1.0); return storeFileSuccess.FileToken; } } } } finally { storageLeaf.Close(); } throw new InvalidOperationException("Store file failed."); } /// /// new upload file(Breakpoint continuingly) /// /// /// /// current account id /// UI action progress /// /// file token public string UploadFile(StorageInfo storageInfo, string filePath, string accountId, Action progress = null, CancellationTokenSource cancelTokenSource = null,bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null) { var storageUrl = storageInfo.Url; var creator = new TcpCreator(storageUrl); var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator); try { if (!storageLeaf.Online) { throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}"); } using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin"); var uploadFileToken = UploadFileFromStream(stream, accountId, progress, storageLeaf, cancelTokenSource); return uploadFileToken; } } finally { storageLeaf.Close(); } } /// /// new upload file(Breakpoint continuingly) /// /// The upload file path /// The file name in file server. /// The account id /// The storage info. /// UI action progress /// /// file token public string UploadFile(StorageInfo storageInfo, string filePath, string fileName, string accountId, Action progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl, bool isCopyFileToOtherNode, Func GetAuthentication, Func>, string> GetCopyAuthentication = null) { var storageUrl = storageInfo.Url; var creator = new TcpCreator(storageUrl); var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator); try { if (!storageLeaf.Online) { throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}"); } using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin"); var uploadFileToken = UploadFileFromStream(fileName,stream, progress, storageLeaf, cancelTokenSource); return uploadFileToken; } } finally { storageLeaf.Close(); } } /// /// Upload file to server by stream /// /// /// /// /// /// /// file token private string UploadFileFromStream(string fileName,Stream stream,Action progress, ClientLeaf storageLeaf, CancellationTokenSource cancelTokenSource = null) { progress?.Invoke(0.01); var fileSize = (int)stream.Length; var md5 = MD5.GetHashString(stream); var currentUploadFileInfo = new UploadStorageFileInfo() { CacheFileName = fileName, FilePosition = 0, FileSize = fileSize, Id = string.Empty, MD5 = md5 }; while (fileSize > currentUploadFileInfo.FilePosition + 1) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { //upload file canceled return null; } var token = storageLeaf.ApplyToken(); var createuploadRequest = CreateUploadRequest(stream, "", currentUploadFileInfo, fileSize); var result = storageLeaf.Send(token, createuploadRequest); var storeFileResult = UploadStoreFileResult.Convert(result); if (storeFileResult != null) { var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage; currentUploadFileInfo.Id = storeFileInfoMessage.Id; currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName; currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition; if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken)) { // upload file success currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken; progress?.Invoke(1.0); return currentUploadFileInfo.FileToken; } progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize); } else { throw new InvalidOperationException("Store file error"); } } return currentUploadFileInfo.FileToken; } /// /// Upload file to server by stream /// /// /// /// /// /// /// file token private string UploadFileFromStream(Stream stream, string accountId, Action progress, ClientLeaf storageLeaf, CancellationTokenSource cancelTokenSource = null) { progress?.Invoke(0.01); var fileSize = (int)stream.Length; var md5 = MD5.GetHashString(stream); var currentUploadFileInfo = new UploadStorageFileInfo() { CacheFileName = string.Empty, FilePosition = 0, FileSize = fileSize, Id = string.Empty, MD5 = md5 }; currentUploadFileInfo.FileSize = fileSize; while (fileSize > currentUploadFileInfo.FilePosition + 1) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { //upload file canceled return null; } var token = storageLeaf.ApplyToken(); var createuploadRequest = CreateUploadRequest(stream, accountId, currentUploadFileInfo, fileSize); var result = storageLeaf.Send(token, createuploadRequest); var storeFileResult = UploadStoreFileResult.Convert(result); if (storeFileResult != null) { var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage; currentUploadFileInfo.Id = storeFileInfoMessage.Id; currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName; currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition; if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken)) { // upload file success currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken; progress?.Invoke(1.0); return currentUploadFileInfo.FileToken; } progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize); } else { throw new InvalidOperationException("Store file error"); } } return currentUploadFileInfo.FileToken; } private async Task UploadFileFromStreamAsync(Stream stream, string accountId, Action progress, ClientLeaf storageLeaf, CancellationTokenSource cancelTokenSource = null) { progress?.Invoke(0.01); var fileSize = (int)stream.Length; var md5 = MD5.GetHashString(stream); var currentUploadFileInfo = UploadStorageFileInfoList.FirstOrDefault(v => v.MD5 == md5); if (currentUploadFileInfo == null) { currentUploadFileInfo = new UploadStorageFileInfo() { CacheFileName = string.Empty, FilePosition = 0, FileSize = fileSize, Id = string.Empty, MD5 = md5 }; UploadStorageFileInfoList.Add(currentUploadFileInfo); } currentUploadFileInfo.FileSize = fileSize; while (fileSize > currentUploadFileInfo.FilePosition + 1) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { //upload file canceled return null; } var token = storageLeaf.ApplyToken(); var result = await storageLeaf.SendAsync(token, CreateUploadRequest( stream, accountId, currentUploadFileInfo, fileSize)); var storeFileResult = UploadStoreFileResult.Convert(result); if (storeFileResult != null) { var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage; currentUploadFileInfo.Id = storeFileInfoMessage.Id; currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken; currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName; currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition; if (!string.IsNullOrEmpty(currentUploadFileInfo.FileToken)) { // upload file success progress?.Invoke(1.0); return currentUploadFileInfo.FileToken; } progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize); } else { throw new InvalidOperationException("Store file error"); } Thread.Sleep(10); } return currentUploadFileInfo.FileToken; } private UploadStoreFileRequest CreateUploadRequest(Stream stream, string accountId, IUploadStorageFileInfo currentUploadFileInfo, int fileSize) { Logger.WriteLineInfo($"UploadStoreFile CreateUploadRequest begin"); //set this upload file end position byte var lastPosition = currentUploadFileInfo.FilePosition + BlockSize - 1; var fileSizePostion = fileSize - 1; if (lastPosition > fileSizePostion) { int offet = fileSizePostion - currentUploadFileInfo.FilePosition; if (offet > MinBlockSize) { lastPosition = currentUploadFileInfo.FilePosition + MinBlockSize; } else { lastPosition = fileSizePostion; } } if (lastPosition == fileSizePostion) { lastPosition = fileSizePostion; } //set this upload file byte size var blockSize = lastPosition - currentUploadFileInfo.FilePosition + 1; stream.Position = currentUploadFileInfo.FilePosition; var data = new byte[blockSize]; stream.Read(data, 0, blockSize); var buffer = new ByteBuffer(data); Logger.WriteLineInfo($"UploadStoreFile blockSize: {blockSize} filePosition:{currentUploadFileInfo.FilePosition}"); // upload to server var uploadmessage = new ClientUploadStorageFileInfoMessage { FileData = buffer, Id = currentUploadFileInfo.Id, FileMd5 = currentUploadFileInfo.MD5, AccountId = accountId ?? string.Empty, FileSize = currentUploadFileInfo.FileSize, FilePosition = currentUploadFileInfo.FilePosition, FileToken = currentUploadFileInfo.FileToken, CacheFileName = currentUploadFileInfo.CacheFileName }; return new UploadStoreFileRequest { ClientUploadStorageFileInfoMessage = uploadmessage }; } /// /// upload file async /// /// /// /// /// /// public async Task UploadFileAsync(StorageInfo storageInfo, byte[] fileData, Action progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null) { var storageUrl = storageInfo.Url; if (string.IsNullOrEmpty(storageUrl)) { throw new Exception($"Storage url is empty, {storageInfo}"); } var creator = new TcpCreator(storageUrl); var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator); try { using (var stream = new MemoryStream(fileData)) { if (stream.Length <= BlockSize) { var size = (int)stream.Length; var data = new byte[size]; await stream.ReadAsync(data, 0, size); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess != null) { progress?.Invoke(1.0); return storeFileSuccess.FileToken; } } else { var size = (int)stream.Length; var count = size / BlockSize; var totalCount = size % BlockSize != 0 ? count + 1 : count; for (int i = 0; i < count; i++) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { return null; } var data = new byte[BlockSize]; await stream.ReadAsync(data, 0, BlockSize); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess == null) { throw new InvalidOperationException("Store file error"); } progress?.Invoke(((double)i + 1) / totalCount); var isDone = totalCount == count && i == count - 1; size -= BlockSize; if (isDone) { return storeFileSuccess.FileToken; } } if (size > 0) { if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested) { return null; } var data = new byte[size]; stream.Read(data, 0, size); var buffer = new ByteBuffer(data); var token = storageLeaf.ApplyToken(); var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer, Index = totalCount - 1, Count = totalCount }); var storeFileSuccess = StoreFileSuccess.Convert(result); if (storeFileSuccess == null) { throw new InvalidOperationException("Store file error"); } progress?.Invoke(1.0); return storeFileSuccess.FileToken; } } } } finally { storageLeaf.Close(); } throw new InvalidOperationException("Store file failed."); } /// /// new upload file(Breakpoint continuingly) /// /// The upload file path /// The file name in file server. /// The storage info. /// file token public List CopydFile(StorageInfo storageInfo, string filePath, string fileName, string destinationServer, Func>, string> GetCopyAuthentication = null) { return new List(); } } }