Some perf improvements...

Author: vplauzon

code/KustoCopyConsole/Kusto/DbCommandClient.cs

@@ -5,6 +5,7 @@
 using KustoCopyConsole.Kusto.Data;
 using System.Collections.Immutable;
 using System.Data;
+using System.Diagnostics;
 
 namespace KustoCopyConsole.Kusto
 {

code/KustoCopyConsole/Kusto/IngestClient.cs

@@ -1,5 +1,6 @@
 using Kusto.Data.Common;
 using Kusto.Ingest;
+using System.Diagnostics;
 
 namespace KustoCopyConsole.Kusto
 {

code/KustoCopyConsole/Runner/AwaitExportedRunner.cs

@@ -17,7 +17,7 @@ internal class AwaitExportedRunner : RunnerBase
         private record ClusterBlocks(Uri ClusterUri, IEnumerable<BlockRowItem> BlockItems);
         #endregion
 
-        private const int MAX_OPERATIONS = 50;
+        private const int MAX_OPERATIONS = 200;
         private static readonly IImmutableSet<string> FAILED_STATUS =
             ImmutableHashSet.Create(
                 [
@@ -33,7 +33,7 @@ public AwaitExportedRunner(
            MainJobParameterization parameterization,
            RowItemGateway rowItemGateway,
            DbClientFactory dbClientFactory)
-           : base(parameterization, rowItemGateway, dbClientFactory, TimeSpan.FromSeconds(5))
+           : base(parameterization, rowItemGateway, dbClientFactory, TimeSpan.FromSeconds(3))
         {
         }
 

code/KustoCopyConsole/Runner/ExportingRunner.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Azure;
 using System;
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Linq;
 
 namespace KustoCopyConsole.Runner
@@ -36,11 +37,20 @@ public async Task RunAsync(CancellationToken ct)
                 var exportLineUp = await GetExportLineUpAsync(capacityMap, ct);
                 var exportCount = await StartExportAsync(exportLineUp, ct);
 
-                //  Sleep
-                await SleepAsync(ct);
+                if (exportCount == 0)
+                {
+                    //  Sleep
+                    await SleepAsync(ct);
+                }
             }
         }
 
+        protected override bool IsWakeUpRelevant(RowItemBase item)
+        {
+            return item is BlockRowItem b
+                && (b.State == BlockState.Planned || b.State == BlockState.Exported);
+        }
+
         private async Task<int> StartExportAsync(
             IEnumerable<ActivityFlatHierarchy> exportLineUp,
             CancellationToken ct)

code/KustoCopyConsole/Runner/QueueIngestRunner.cs

@@ -6,6 +6,7 @@
 using KustoCopyConsole.Kusto;
 using KustoCopyConsole.Storage;
 using System.Collections.Immutable;
+using System.Diagnostics;
 
 namespace KustoCopyConsole.Runner
 {
@@ -29,17 +30,27 @@ public async Task RunAsync(CancellationToken ct)
                     i => i.RowItem.State != IterationState.Completed);
                 var exportedBlocks = allBlocks
                     .Where(h => h.Block.State == BlockState.Exported);
+                var s = new Stopwatch(); s.Start();
                 var ingestionTasks = exportedBlocks
                     .Select(h => QueueIngestBlockAsync(h, ct))
                     .ToImmutableArray();
 
                 await Task.WhenAll(ingestionTasks);
 
-                //  Sleep
-                await SleepAsync(ct);
+                if (!ingestionTasks.Any())
+                {
+                    //  Sleep
+                    await SleepAsync(ct);
+                }
             }
         }
 
+        protected override bool IsWakeUpRelevant(RowItemBase item)
+        {
+            return item is BlockRowItem b
+                && b.State == BlockState.Exported;
+        }
+
         private async Task QueueIngestBlockAsync(ActivityFlatHierarchy item, CancellationToken ct)
         {
             //  It's possible, although unlikely, the temp table hasn't been created yet

code/KustoCopyConsole/Storage/AzureStorage/AzureBlobUriProvider.cs

@@ -7,6 +7,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;