github · ArinGhazarian · Dec 11, 2024 · Dec 6, 2024 · Dec 7, 2024 · Dec 7, 2024
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -1 +1 @@
-
+- Add progress report to `gh [gei|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3
diff --git a/src/Octoshift/Extensions/NumericExtensions.cs b/src/Octoshift/Extensions/NumericExtensions.cs
@@ -0,0 +1,19 @@
+namespace OctoshiftCLI.Extensions;
+
+public static class NumericExtensions
+{
+    public static string ToLogFriendlySize(this long size)
+    {
+        const int kilobyte = 1024;
+        const int megabyte = 1024 * kilobyte;
+        const int gigabyte = 1024 * megabyte;
+
+        return size switch
+        {
+            < kilobyte => $"{size:n0} bytes",
+            < megabyte => $"{size / (double)kilobyte:n0} KB",
+            < gigabyte => $"{size / (double)megabyte:n0} MB",
+            _ => $"{size / (double)gigabyte:n2} GB"
+        };
+    }
+}
diff --git a/src/Octoshift/Services/AwsApi.cs b/src/Octoshift/Services/AwsApi.cs
@@ -13,14 +13,22 @@ namespace OctoshiftCLI.Services;
 public class AwsApi : IDisposable
 {
     private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
+    private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;
 
     private readonly ITransferUtility _transferUtility;
+    private readonly object _mutex = new();
+    private readonly OctoLogger _log;
+    private DateTime _nextProgressReport = DateTime.Now;
 
-    public AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
+    public AwsApi(ITransferUtility transferUtility, OctoLogger log)
+    {
+        _transferUtility = transferUtility;
+        _log = log;
+    }
 
 #pragma warning disable CA2000
-    public AwsApi(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
-        : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)))
+    public AwsApi(OctoLogger log, string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
+        : this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)), log)
 #pragma warning restore CA2000
     {
     }
@@ -51,20 +59,37 @@ public virtual async Task<string> UploadToBucket(string bucketName, string fileN
     {
         try
         {
-            await _transferUtility.UploadAsync(fileName, bucketName, keyName);
+            var uploadRequest = new TransferUtilityUploadRequest
+            {
+                BucketName = bucketName,
+                Key = keyName,
+                FilePath = fileName
+            };
+            return await UploadToBucket(uploadRequest);
         }
         catch (Exception ex) when (ex is TaskCanceledException or TimeoutException)
         {
             throw new OctoshiftCliException($"Upload of archive \"{fileName}\" to AWS timed out", ex);
         }
-
-        return GetPreSignedUrlForFile(bucketName, keyName);
     }
 
     public virtual async Task<string> UploadToBucket(string bucketName, Stream content, string keyName)
     {
-        await _transferUtility.UploadAsync(content, bucketName, keyName);
-        return GetPreSignedUrlForFile(bucketName, keyName);
+        var uploadRequest = new TransferUtilityUploadRequest
+        {
+            BucketName = bucketName,
+            Key = keyName,
+            InputStream = content
+        };
+        return await UploadToBucket(uploadRequest);
+    }
+
+    private async Task<string> UploadToBucket(TransferUtilityUploadRequest uploadRequest)
+    {
+        uploadRequest.UploadProgressEvent += (_, args) => LogProgress(args.PercentDone, args.TransferredBytes, args.TotalBytes);
+        await _transferUtility.UploadAsync(uploadRequest);
+
+        return GetPreSignedUrlForFile(uploadRequest.BucketName, uploadRequest.Key);
     }
 
     private string GetPreSignedUrlForFile(string bucketName, string keyName)
@@ -81,6 +106,25 @@ private string GetPreSignedUrlForFile(string bucketName, string keyName)
         return _transferUtility.S3Client.GetPreSignedURL(urlRequest);
     }
 
+    private void LogProgress(int percentDone, long uploadedBytes, long totalBytes)
+    {
+        lock (_mutex)
+        {
+            if (DateTime.Now < _nextProgressReport)
+            {
+                return;
+            }
+
+            _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
+        }
+
+        var progressMessage = uploadedBytes > 0
+            ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentDone}%) completed"
+            : "";
+
+        _log.LogInformation($"Archive upload in progress{progressMessage}...");
+    }
+
     protected virtual void Dispose(bool disposing)
     {
         if (disposing)

diff --git a/src/Octoshift/Services/AzureApi.cs b/src/Octoshift/Services/AzureApi.cs
@@ -6,6 +6,7 @@
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
 using Azure.Storage.Sas;
+using OctoshiftCLI.Extensions;
 
 namespace OctoshiftCLI.Services;
 
@@ -14,9 +15,12 @@ public class AzureApi
     private readonly HttpClient _client;
     private readonly BlobServiceClient _blobServiceClient;
     private readonly OctoLogger _log;
+    private readonly object _mutex = new();
     private const string CONTAINER_PREFIX = "migration-archives";
     private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
     private const int DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024;
+    private const int UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10;
+    private DateTime _nextProgressReport = DateTime.Now;
 
     public AzureApi(HttpClient client, BlobServiceClient blobServiceClient, OctoLogger log)
     {
@@ -48,16 +52,24 @@ public virtual async Task<Uri> UploadToBlob(string fileName, byte[] content)
 
     public virtual async Task<Uri> UploadToBlob(string fileName, Stream content)
     {
+        ArgumentNullException.ThrowIfNull(fileName, nameof(fileName));
+        ArgumentNullException.ThrowIfNull(content);
+
         var containerClient = await CreateBlobContainerAsync();
         var blobClient = containerClient.GetBlobClient(fileName);
 
+        var progress = new Progress<long>();
+        var archiveSize = content.Length;
+        progress.ProgressChanged += (_, uploadedBytes) => LogProgress(uploadedBytes, archiveSize);
+
         var options = new BlobUploadOptions
         {
             TransferOptions = new Azure.Storage.StorageTransferOptions()
             {
                 InitialTransferSize = DEFAULT_BLOCK_SIZE,
                 MaximumTransferSize = DEFAULT_BLOCK_SIZE
             },
+            ProgressHandler = progress
         };
 
         await blobClient.UploadAsync(content, options);
@@ -89,4 +101,24 @@ private Uri GetServiceSasUriForBlob(BlobClient blobClient)
 
         return blobClient.GenerateSasUri(sasBuilder);
     }
+
+    private void LogProgress(long uploadedBytes, long totalBytes)
+    {
+        lock (_mutex)
+        {
+            if (DateTime.Now < _nextProgressReport)
+            {
+                return;
+            }
+
+            _nextProgressReport = _nextProgressReport.AddSeconds(UPLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS);
+        }
+
+        var percentage = (int)(uploadedBytes * 100L / totalBytes);
+        var progressMessage = uploadedBytes > 0
+            ? $", {uploadedBytes.ToLogFriendlySize()} out of {totalBytes.ToLogFriendlySize()} ({percentage}%) completed"
+            : "";
+
+        _log.LogInformation($"Archive upload in progress{progressMessage}...");
+    }
 }
diff --git a/src/Octoshift/Services/OctoLogger.cs b/src/Octoshift/Services/OctoLogger.cs
@@ -23,6 +23,7 @@ public class OctoLogger
     private readonly string _logFilePath;
     private readonly string _verboseFilePath;
     private readonly bool _debugMode;
+    private readonly object _mutex = new();
 
     private readonly Action<string> _writeToLog;
     private readonly Action<string> _writeToVerboseLog;
@@ -103,20 +104,32 @@ private string Redact(string msg)
         return result;
     }
 
-    public virtual void LogInformation(string msg) => Log(msg, LogLevel.INFO);
+    public virtual void LogInformation(string msg)
+    {
+        lock (_mutex)
+        {
+            Log(msg, LogLevel.INFO);
+        }
+    }
 
     public virtual void LogWarning(string msg)
     {
-        Console.ForegroundColor = ConsoleColor.Yellow;
-        Log(msg, LogLevel.WARNING);
-        Console.ResetColor();
+        lock (_mutex)
+        {
+            Console.ForegroundColor = ConsoleColor.Yellow;
+            Log(msg, LogLevel.WARNING);
+            Console.ResetColor();
+        }
     }
 
     public virtual void LogError(string msg)
     {
-        Console.ForegroundColor = ConsoleColor.Red;
-        Log(msg, LogLevel.ERROR);
-        Console.ResetColor();
+        lock (_mutex)
+        {
+            Console.ForegroundColor = ConsoleColor.Red;
+            Log(msg, LogLevel.ERROR);
+            Console.ResetColor();
+        }
     }
 
     public virtual void LogError(Exception ex)
@@ -126,30 +139,36 @@ public virtual void LogError(Exception ex)
             throw new ArgumentNullException(nameof(ex));
         }
 
-        var verboseMessage = ex is HttpRequestException httpEx ? $"[HTTP ERROR {(int?)httpEx.StatusCode}] {ex}" : ex.ToString();
-        var logMessage = Verbose ? verboseMessage : ex is OctoshiftCliException ? ex.Message : GENERIC_ERROR_MESSAGE;
+        lock (_mutex)
+        {
+            var verboseMessage = ex is HttpRequestException httpEx ? $"[HTTP ERROR {(int?)httpEx.StatusCode}] {ex}" : ex.ToString();
+            var logMessage = Verbose ? verboseMessage : ex is OctoshiftCliException ? ex.Message : GENERIC_ERROR_MESSAGE;
 
-        var output = Redact(FormatMessage(logMessage, LogLevel.ERROR));
+            var output = Redact(FormatMessage(logMessage, LogLevel.ERROR));
 
-        Console.ForegroundColor = ConsoleColor.Red;
-        _writeToConsoleError(output);
-        Console.ResetColor();
+            Console.ForegroundColor = ConsoleColor.Red;
+            _writeToConsoleError(output);
+            Console.ResetColor();
 
-        _writeToLog(output);
-        _writeToVerboseLog(Redact(FormatMessage(verboseMessage, LogLevel.ERROR)));
+            _writeToLog(output);
+            _writeToVerboseLog(Redact(FormatMessage(verboseMessage, LogLevel.ERROR)));
+        }
     }
 
     public virtual void LogVerbose(string msg)
     {
-        if (Verbose)
+        lock (_mutex)
         {
-            Console.ForegroundColor = ConsoleColor.Gray;
-            Log(msg, LogLevel.VERBOSE);
-            Console.ResetColor();
-        }
-        else
-        {
-            _writeToVerboseLog(Redact(FormatMessage(msg, LogLevel.VERBOSE)));
+            if (Verbose)
+            {
+                Console.ForegroundColor = ConsoleColor.Gray;
+                Log(msg, LogLevel.VERBOSE);
+                Console.ResetColor();
+            }
+            else
+            {
+                _writeToVerboseLog(Redact(FormatMessage(msg, LogLevel.VERBOSE)));
+            }
         }
     }
 
@@ -163,9 +182,12 @@ public virtual void LogDebug(string msg)
 
     public virtual void LogSuccess(string msg)
     {
-        Console.ForegroundColor = ConsoleColor.Green;
-        Log(msg, LogLevel.SUCCESS);
-        Console.ResetColor();
+        lock (_mutex)
+        {
+            Console.ForegroundColor = ConsoleColor.Green;
+            Log(msg, LogLevel.SUCCESS);
+            Console.ResetColor();
+        }
     }
 
     public virtual void RegisterSecret(string secret) => _secrets.Add(secret);
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@

		- Add progress report to `gh [gei\|bbs2gh] migrate-repo` command when uploading migration archives to Azure Blob or AWS S3