using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Vinno.IUS.Common.IO;
using Vinno.IUS.Common.Log;
using Vinno.IUS.Common.Network.Leaf;
using Vinno.IUS.Common.Network.Tcp;
using Vinno.IUS.Common.Utilities;
using Vinno.vCloud.Common.Storage.Upload;
using Vinno.vCloud.Protocol.Infrastructures;
using Vinno.vCloud.Protocol.Messages.Client.Storage;
namespace Vinno.vCloud.Common.Storage
{
public class VCloudStorageUploader : IUploader
{
///
///when app running ,set cache folder path
///
private readonly List UploadStorageFileInfoList =
new List();
private const int MinBlockSize = 1048576; //4Mb
private const int BlockSize = 4194304; //4mb
///
/// Upload one file and return the file token.
///
/// The TcpCrerator for newtwork connection
/// The file data to be uploaded
/// The upload progress
/// The token to cancel the upload
/// The file token in server
public string UploadFile(StorageInfo storageInfo, byte[] fileData, Action progress = null, CancellationTokenSource cancelTokenSource = null, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null)
{
var storageUrl = storageInfo.Url;
if (string.IsNullOrEmpty(storageUrl)) { throw new Exception("Storage url is empty"); }
var creator = new TcpCreator(storageUrl);
var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
try
{
using (var stream = new MemoryStream(fileData))
{
if (stream.Length <= BlockSize)
{
var size = (int)stream.Length;
var data = new byte[size];
stream.Read(data, 0, size);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess != null)
{
progress?.Invoke(1.0);
return storeFileSuccess.FileToken;
}
}
else
{
var size = (int)stream.Length;
var count = size / BlockSize;
var totalCount = size % BlockSize != 0 ? count + 1 : count;
for (int i = 0; i < count; i++)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
return null;
}
var data = new byte[BlockSize];
stream.Read(data, 0, BlockSize);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = storageLeaf.Send(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess == null)
{
throw new InvalidOperationException("Store file error");
}
progress?.Invoke(((double)i + 1) / totalCount);
var isDone = totalCount == count && i == count - 1;
size -= BlockSize;
if (isDone)
{
return storeFileSuccess.FileToken;
}
}
if (size > 0)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
return null;
}
var data = new byte[size];
stream.Read(data, 0, size);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = storageLeaf.Send(token, new StoreFileRequest
{ FileData = buffer, Index = totalCount - 1, Count = totalCount });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess == null)
{
throw new InvalidOperationException("Store file error");
}
progress?.Invoke(1.0);
return storeFileSuccess.FileToken;
}
}
}
}
finally
{
storageLeaf.Close();
}
throw new InvalidOperationException("Store file failed.");
}
///
/// new upload file(Breakpoint continuingly)
///
///
///
/// current account id
/// UI action progress
///
/// file token
public string UploadFile(StorageInfo storageInfo, string filePath, string accountId,
Action progress = null,
CancellationTokenSource cancelTokenSource = null,bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null)
{
var storageUrl = storageInfo.Url;
var creator = new TcpCreator(storageUrl);
var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
try
{
if (!storageLeaf.Online)
{
throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}");
}
using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin");
var uploadFileToken = UploadFileFromStream(stream,
accountId, progress, storageLeaf, cancelTokenSource);
return uploadFileToken;
}
}
finally
{
storageLeaf.Close();
}
}
///
/// new upload file(Breakpoint continuingly)
///
/// The upload file path
/// The file name in file server.
/// The account id
/// The storage info.
/// UI action progress
///
/// file token
public string UploadFile(StorageInfo storageInfo, string filePath, string fileName, string accountId, Action progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl, bool isCopyFileToOtherNode, Func GetAuthentication, Func>, string> GetCopyAuthentication = null)
{
var storageUrl = storageInfo.Url;
var creator = new TcpCreator(storageUrl);
var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
try
{
if (!storageLeaf.Online)
{
throw new Exception($"storageLeaf is offline for upload file,url:{storageInfo}");
}
using (var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
Logger.WriteLineInfo($"UploadStoreFile:filePath{filePath} begin");
var uploadFileToken = UploadFileFromStream(fileName,stream,
progress, storageLeaf, cancelTokenSource);
return uploadFileToken;
}
}
finally
{
storageLeaf.Close();
}
}
///
/// Upload file to server by stream
///
///
///
///
///
///
/// file token
private string UploadFileFromStream(string fileName,Stream stream,Action progress, ClientLeaf storageLeaf,
CancellationTokenSource cancelTokenSource = null)
{
progress?.Invoke(0.01);
var fileSize = (int)stream.Length;
var md5 = MD5.GetHashString(stream);
var currentUploadFileInfo = new UploadStorageFileInfo()
{
CacheFileName = fileName,
FilePosition = 0,
FileSize = fileSize,
Id = string.Empty,
MD5 = md5
};
while (fileSize > currentUploadFileInfo.FilePosition + 1)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
//upload file canceled
return null;
}
var token = storageLeaf.ApplyToken();
var createuploadRequest = CreateUploadRequest(stream, "", currentUploadFileInfo, fileSize);
var result = storageLeaf.Send(token, createuploadRequest);
var storeFileResult = UploadStoreFileResult.Convert(result);
if (storeFileResult != null)
{
var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
currentUploadFileInfo.Id = storeFileInfoMessage.Id;
currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken))
{
// upload file success
currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
progress?.Invoke(1.0);
return currentUploadFileInfo.FileToken;
}
progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
}
else
{
throw new InvalidOperationException("Store file error");
}
}
return currentUploadFileInfo.FileToken;
}
///
/// Upload file to server by stream
///
///
///
///
///
///
/// file token
private string UploadFileFromStream(Stream stream,
string accountId, Action progress, ClientLeaf storageLeaf,
CancellationTokenSource cancelTokenSource = null)
{
progress?.Invoke(0.01);
var fileSize = (int)stream.Length;
var md5 = MD5.GetHashString(stream);
var currentUploadFileInfo = new UploadStorageFileInfo()
{
CacheFileName = string.Empty,
FilePosition = 0,
FileSize = fileSize,
Id = string.Empty,
MD5 = md5
};
currentUploadFileInfo.FileSize = fileSize;
while (fileSize > currentUploadFileInfo.FilePosition + 1)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
//upload file canceled
return null;
}
var token = storageLeaf.ApplyToken();
var createuploadRequest = CreateUploadRequest(stream, accountId, currentUploadFileInfo, fileSize);
var result = storageLeaf.Send(token, createuploadRequest);
var storeFileResult = UploadStoreFileResult.Convert(result);
if (storeFileResult != null)
{
var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
currentUploadFileInfo.Id = storeFileInfoMessage.Id;
currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
if (!string.IsNullOrWhiteSpace(storeFileInfoMessage.FileToken))
{
// upload file success
currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
progress?.Invoke(1.0);
return currentUploadFileInfo.FileToken;
}
progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
}
else
{
throw new InvalidOperationException("Store file error");
}
}
return currentUploadFileInfo.FileToken;
}
private async Task UploadFileFromStreamAsync(Stream stream,
string accountId, Action progress, ClientLeaf storageLeaf,
CancellationTokenSource cancelTokenSource = null)
{
progress?.Invoke(0.01);
var fileSize = (int)stream.Length;
var md5 = MD5.GetHashString(stream);
var currentUploadFileInfo = UploadStorageFileInfoList.FirstOrDefault(v => v.MD5 == md5);
if (currentUploadFileInfo == null)
{
currentUploadFileInfo = new UploadStorageFileInfo()
{
CacheFileName = string.Empty,
FilePosition = 0,
FileSize = fileSize,
Id = string.Empty,
MD5 = md5
};
UploadStorageFileInfoList.Add(currentUploadFileInfo);
}
currentUploadFileInfo.FileSize = fileSize;
while (fileSize > currentUploadFileInfo.FilePosition + 1)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
//upload file canceled
return null;
}
var token = storageLeaf.ApplyToken();
var result = await storageLeaf.SendAsync(token, CreateUploadRequest(
stream, accountId, currentUploadFileInfo, fileSize));
var storeFileResult = UploadStoreFileResult.Convert(result);
if (storeFileResult != null)
{
var storeFileInfoMessage = storeFileResult.ClientUploadStorageFileInfoMessage;
currentUploadFileInfo.Id = storeFileInfoMessage.Id;
currentUploadFileInfo.FileToken = storeFileInfoMessage.FileToken;
currentUploadFileInfo.CacheFileName = storeFileInfoMessage.CacheFileName;
currentUploadFileInfo.FilePosition = storeFileInfoMessage.FilePosition;
if (!string.IsNullOrEmpty(currentUploadFileInfo.FileToken))
{
// upload file success
progress?.Invoke(1.0);
return currentUploadFileInfo.FileToken;
}
progress?.Invoke((double)currentUploadFileInfo.FilePosition / fileSize);
}
else
{
throw new InvalidOperationException("Store file error");
}
Thread.Sleep(10);
}
return currentUploadFileInfo.FileToken;
}
private UploadStoreFileRequest CreateUploadRequest(Stream stream, string accountId,
IUploadStorageFileInfo currentUploadFileInfo, int fileSize)
{
Logger.WriteLineInfo($"UploadStoreFile CreateUploadRequest begin");
//set this upload file end position byte
var lastPosition = currentUploadFileInfo.FilePosition + BlockSize - 1;
var fileSizePostion = fileSize - 1;
if (lastPosition > fileSizePostion)
{
int offet = fileSizePostion - currentUploadFileInfo.FilePosition;
if (offet > MinBlockSize)
{
lastPosition = currentUploadFileInfo.FilePosition + MinBlockSize;
}
else
{
lastPosition = fileSizePostion;
}
}
if (lastPosition == fileSizePostion)
{
lastPosition = fileSizePostion;
}
//set this upload file byte size
var blockSize = lastPosition - currentUploadFileInfo.FilePosition + 1;
stream.Position = currentUploadFileInfo.FilePosition;
var data = new byte[blockSize];
stream.Read(data, 0, blockSize);
var buffer = new ByteBuffer(data);
Logger.WriteLineInfo($"UploadStoreFile blockSize: {blockSize} filePosition:{currentUploadFileInfo.FilePosition}");
// upload to server
var uploadmessage = new ClientUploadStorageFileInfoMessage
{
FileData = buffer,
Id = currentUploadFileInfo.Id,
FileMd5 = currentUploadFileInfo.MD5,
AccountId = accountId ?? string.Empty,
FileSize = currentUploadFileInfo.FileSize,
FilePosition = currentUploadFileInfo.FilePosition,
FileToken = currentUploadFileInfo.FileToken,
CacheFileName = currentUploadFileInfo.CacheFileName
};
return new UploadStoreFileRequest { ClientUploadStorageFileInfoMessage = uploadmessage };
}
///
/// upload file async
///
///
///
///
///
///
public async Task UploadFileAsync(StorageInfo storageInfo, byte[] fileData, Action progress, CancellationTokenSource cancelTokenSource, bool isReturnCDNUrl = true, bool isCopyFileToOtherNode = false, Func GetAuthentication = null, Func>, string> GetCopyAuthentication = null)
{
var storageUrl = storageInfo.Url;
if (string.IsNullOrEmpty(storageUrl)) { throw new Exception($"Storage url is empty, {storageInfo}"); }
var creator = new TcpCreator(storageUrl);
var storageLeaf = new ClientLeaf(new LeafIdContext(), LeafMode.Single, creator);
try
{
using (var stream = new MemoryStream(fileData))
{
if (stream.Length <= BlockSize)
{
var size = (int)stream.Length;
var data = new byte[size];
await stream.ReadAsync(data, 0, size);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess != null)
{
progress?.Invoke(1.0);
return storeFileSuccess.FileToken;
}
}
else
{
var size = (int)stream.Length;
var count = size / BlockSize;
var totalCount = size % BlockSize != 0 ? count + 1 : count;
for (int i = 0; i < count; i++)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
return null;
}
var data = new byte[BlockSize];
await stream.ReadAsync(data, 0, BlockSize);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = await storageLeaf.SendAsync(token, new StoreFileRequest { FileData = buffer, Index = i, Count = totalCount });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess == null)
{
throw new InvalidOperationException("Store file error");
}
progress?.Invoke(((double)i + 1) / totalCount);
var isDone = totalCount == count && i == count - 1;
size -= BlockSize;
if (isDone)
{
return storeFileSuccess.FileToken;
}
}
if (size > 0)
{
if (cancelTokenSource != null && cancelTokenSource.IsCancellationRequested)
{
return null;
}
var data = new byte[size];
stream.Read(data, 0, size);
var buffer = new ByteBuffer(data);
var token = storageLeaf.ApplyToken();
var result = await storageLeaf.SendAsync(token, new StoreFileRequest
{ FileData = buffer, Index = totalCount - 1, Count = totalCount });
var storeFileSuccess = StoreFileSuccess.Convert(result);
if (storeFileSuccess == null)
{
throw new InvalidOperationException("Store file error");
}
progress?.Invoke(1.0);
return storeFileSuccess.FileToken;
}
}
}
}
finally
{
storageLeaf.Close();
}
throw new InvalidOperationException("Store file failed.");
}
///
/// new upload file(Breakpoint continuingly)
///
/// The upload file path
/// The file name in file server.
/// The storage info.
/// file token
public List CopydFile(StorageInfo storageInfo, string filePath, string fileName, string destinationServer, Func>, string> GetCopyAuthentication = null)
{
return new List();
}
}
}