Skip to content

Commit 878273c

Browse files
dsamedsame
committed
Implement processing & monitoring
1 parent 3eeea9c commit 878273c

15 files changed

+176
-30
lines changed

‎az-appservice-dotnet.xUnit/providers/v1/Azure/AzureBlobProvider/StoreBlobAsyncTest.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public async Task Should_ReturnUri()
2323
var provider = _containerFixture.GetProvider();
2424
var fileObject = _fileProviderFixture.GetFileObject("test.bin",1);
2525
// Act
26-
var actual = await provider.StoreBlobAsync(fileObject.Name, fileObject.Path);
26+
var actual = await provider.UploadBlobAsync(fileObject.Name, fileObject.Path);
2727
_containerFixture.DisposableBag.Add(fileObject.Name);
2828
// Assert
2929
var readResponse = await _containerFixture.ContainerClient.GetBlobClient(fileObject.Name).ExistsAsync();
@@ -42,7 +42,7 @@ public async Task Should_ThrowException()
4242
var provider = _containerFixture.GetProvider();
4343
var fileObject = new IFileProviderService.FileObject("test.bin","/nonexistent");
4444
// Act
45-
var exception = await Record.ExceptionAsync(() => provider.StoreBlobAsync(fileObject.Name, fileObject.Path));
45+
var exception = await Record.ExceptionAsync(() => provider.UploadBlobAsync(fileObject.Name, fileObject.Path));
4646
// Assert
4747
Assert.NotNull(exception);
4848
Assert.IsType<AggregateException>(exception);

‎az-appservice-dotnet/Program.cs

+19
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
using az_appservice_dotnet.services.v1;
55
using az_appservice_dotnet.services.v1.Blob;
66
using az_appservice_dotnet.services.v1.Blob.dependencies;
7+
using az_appservice_dotnet.services.v1.ImageProcessing;
8+
using az_appservice_dotnet.services.v1.Monitor;
79
using az_appservice_dotnet.services.v1.State;
810
using az_appservice_dotnet.services.v1.State.dependencies;
911
using az_appservice_dotnet.services.v1.UploadedFiles;
@@ -19,12 +21,16 @@ public static async Task Main(string[] args)
1921
builder.Services.AddSingleton<IBlobProvider, AzureBlobProvider>();
2022
builder.Services.AddSingleton<IPersistProcessingStateProvider, CosmosDbPersistProcessingStateProvider>();
2123
builder.Services.AddSingleton<IPublishProcessingStateProvider, AzureSbPublishProcessingStateProvider>();
24+
builder.Services.AddSingleton<ISubscribeProcessingStateProvider, AzureSbSubscribeProcessingStateProvider>();
2225

2326
builder.Services.AddSingleton<IFileProviderService, FakeFileProviderService>();
2427
builder.Services.AddSingleton<IBlobService, BlobService>();
2528
builder.Services.AddSingleton<IProcessingStateService, ProcessingStateService>();
29+
builder.Services.AddSingleton<IStateMonitor, ConsoleMonitor>();
30+
builder.Services.AddSingleton<IImageProcessorService, NullImageProcessorService>();
2631

2732
builder.Services.AddSingleton<ProducerService>();
33+
builder.Services.AddSingleton<ProcessorService>();
2834

2935
var app = builder.Build();
3036

@@ -37,7 +43,20 @@ public static async Task Main(string[] args)
3743

3844
app.MapGroup("/1")
3945
.MapApi1(app);
46+
47+
var monitorService = app.Services.GetService<IStateMonitor>();
48+
if (monitorService == null)
49+
{
50+
throw new Exception("StateMonitor is not registered");
51+
}
52+
monitorService.StartStateMonitor();
4053

54+
var processorService = app.Services.GetService<ProcessorService>();
55+
if (processorService == null)
56+
{
57+
throw new Exception("ProcessorService is not registered");
58+
}
59+
processorService.StartWaitForImagesToProcess();
4160
await app.RunAsync();
4261
}
4362

‎az-appservice-dotnet/providers/Azure/v1/AzureBlobProvider.cs

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
using az_appservice_dotnet.services;
21
using az_appservice_dotnet.services.v1.Blob.dependencies;
32
using Azure.Storage.Blobs;
43

@@ -23,7 +22,7 @@ public AzureBlobProvider(BlobContainerClient containerClient)
2322
_containerClient = containerClient;
2423
}
2524

26-
public Task<Uri> StoreBlobAsync(string name, string localFilePath)
25+
public Task<Uri> UploadBlobAsync(string name, string localFilePath)
2726
{
2827
var blobClient = _containerClient.GetBlobClient(name);
2928
return blobClient.UploadAsync(localFilePath, true)
@@ -33,4 +32,15 @@ public Task<Uri> StoreBlobAsync(string name, string localFilePath)
3332
return blobClient.Uri;
3433
});
3534
}
35+
36+
public Task<bool> DownloadBlobAsync(string name, string localFilePath)
37+
{
38+
var blobClient = _containerClient.GetBlobClient(name);
39+
return blobClient.DownloadToAsync(localFilePath)
40+
.ContinueWith(task =>
41+
{
42+
if (task.Exception != null) throw task.Exception;
43+
return true;
44+
});
45+
}
3646
}

‎az-appservice-dotnet/providers/Azure/v1/AzureSbSubscribeProcessingStateProvider.cs

+15
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,21 @@
44

55
namespace az_appservice_dotnet.providers.Azure.v1;
66

7+
/**
8+
* <summary>
9+
* Known bug/feature: the provider makes a lookup to IPersistProcessingStateProvider in order
10+
* to find the state by the ID extracted from the message. By the moment the message is
11+
* read from the queue, the state might be already changed in the database.
12+
*
13+
* So the message sent for change to State1 can cause the propagation of the State2. It would not
14+
* be a problem if the State2 had not its own message sent to the queue. In this case the subscriber
15+
* will receive 2 notification for State2 and none for State1.
16+
*
17+
* Generally speaking, it it correct behaviour, but still might cause the problem with handling the
18+
* some states twice and missing the others.
19+
* </summary>
20+
*/
21+
722
public class AzureSbSubscribeProcessingStateProvider : ISubscribeProcessingStateProvider
823
{
924
private readonly ServiceBusProcessor _processor;

‎az-appservice-dotnet/services/v1/Blob/BlobService.cs

+9-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ namespace az_appservice_dotnet.services.v1.Blob;
66
* <summary>
77
* This is a wrapper around a blob provider to implement some possible business logic common
88
* for differnt blob providers.
9+
*
10+
* For ex. logging error handling should be placed here.
911
* </summary>
1012
*/
1113
public class BlobService: IBlobService
@@ -17,8 +19,13 @@ public BlobService(IBlobProvider blobProvider)
1719
_blobProvider = blobProvider;
1820
}
1921

20-
public Task<Uri> StoreBlobAsync(string name, string localFilePath)
22+
public Task<Uri> UploadBlobAsync(string name, string localFilePath)
2123
{
22-
return _blobProvider.StoreBlobAsync(name, localFilePath);
24+
return _blobProvider.UploadBlobAsync(name, localFilePath);
25+
}
26+
27+
public Task<bool> DownloadBlobAsync(string name, string localFilePath)
28+
{
29+
return _blobProvider.DownloadBlobAsync(name, localFilePath);
2330
}
2431
}

‎az-appservice-dotnet/services/v1/Blob/IBlobService.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@ namespace az_appservice_dotnet.services.v1.Blob;
22

33
public interface IBlobService
44
{
5-
Task<Uri> StoreBlobAsync(string name, string localFilePath);
5+
Task<Uri> UploadBlobAsync(string name, string localFilePath);
6+
Task<bool> DownloadBlobAsync(string name, string localFilePath);
67
}

‎az-appservice-dotnet/services/v1/Blob/dependencies/IBlobProvider.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@ namespace az_appservice_dotnet.services.v1.Blob.dependencies;
22

33
public interface IBlobProvider
44
{
5-
Task<Uri> StoreBlobAsync(string name, string localFilePath);
5+
Task<Uri> UploadBlobAsync(string name, string localFilePath);
6+
Task<Boolean> DownloadBlobAsync(string name, string localFilePath);
67
}

‎az-appservice-dotnet/services/v1/ImageProcessing/IImageProcessorService.cs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
namespace az_appservice_dotnet.services.v1.ImageProcessing;
2+
3+
public interface IImageProcessorService
4+
{
5+
Task<string> ProcessImageAsync(string imageFilePath);
6+
}

‎az-appservice-dotnet/services/v1/ImageProcessing/NullImageProcessorService.cs

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
namespace az_appservice_dotnet.services.v1.ImageProcessing;
2+
3+
public class NullImageProcessorService: IImageProcessorService
4+
{
5+
public Task<string> ProcessImageAsync(string imageFilePath)
6+
{
7+
if (!File.Exists(imageFilePath))
8+
{
9+
throw new FileNotFoundException($"NullImageProcessorService: File not found: {imageFilePath}");
10+
}
11+
12+
return Task.Run(() =>
13+
{
14+
// get tmp file path
15+
var tmpFilePath = Path.GetTempFileName();
16+
// copy imageFilePath to tmpFilePath
17+
File.Copy(imageFilePath, tmpFilePath, true);
18+
return tmpFilePath;
19+
});
20+
}
21+
}

‎az-appservice-dotnet/services/v1/Monitor/ConsoleMonitor.cs

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
using az_appservice_dotnet.services.v1.State;
2+
3+
namespace az_appservice_dotnet.services.v1.Monitor;
4+
5+
public class ConsoleMonitor: IStateMonitor
6+
{
7+
private readonly IProcessingStateService _processingStateService;
8+
9+
public ConsoleMonitor(IProcessingStateService processingStateService)
10+
{
11+
_processingStateService = processingStateService;
12+
}
13+
14+
public void StartStateMonitor()
15+
{
16+
_processingStateService.ListenToStateChanges(state =>
17+
{
18+
Console.WriteLine($"State changed to {state.Status}: file={state.FileName}, originalUrl={state.OriginalFileUrl}, processedUrl={state.ProcessedFileUrl}");
19+
});
20+
}
21+
}

‎az-appservice-dotnet/services/v1/Monitor/IStateMonitor.cs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
namespace az_appservice_dotnet.services.v1.Monitor;
2+
3+
public interface IStateMonitor
4+
{
5+
void StartStateMonitor();
6+
}

‎az-appservice-dotnet/services/v1/ProcessorService.cs

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
using az_appservice_dotnet.services.v1.Blob;
2+
using az_appservice_dotnet.services.v1.ImageProcessing;
3+
using az_appservice_dotnet.services.v1.State;
4+
5+
namespace az_appservice_dotnet.services.v1;
6+
7+
public class ProcessorService
8+
{
9+
private IBlobService _blobService;
10+
private IProcessingStateService _processingStateService;
11+
private IImageProcessorService _imageProcessorService;
12+
13+
public ProcessorService(IBlobService blobService, IProcessingStateService processingStateService,
14+
IImageProcessorService imageProcessorService)
15+
{
16+
_imageProcessorService = imageProcessorService;
17+
_blobService = blobService;
18+
_processingStateService = processingStateService;
19+
}
20+
21+
public void StartWaitForImagesToProcess()
22+
{
23+
_processingStateService.ListenToStateChanges(async state =>
24+
{
25+
if (state.Status == IProcessingStateService.Status.WaitingForProcessing)
26+
{
27+
Console.WriteLine(state.OriginalFileUrl);
28+
var tmpFilePath = Path.GetTempFileName();
29+
var stateProcessing = await _processingStateService.MoveToProcessingStateAsync(state);
30+
try
31+
{
32+
var ok = await _blobService.DownloadBlobAsync(state.FileName, tmpFilePath);
33+
var processedFilePath = await _imageProcessorService.ProcessImageAsync(tmpFilePath);
34+
var uploadedUri =
35+
await _blobService.UploadBlobAsync($"processed-{state.FileName}", processedFilePath);
36+
await _processingStateService.MoveToCompletedStateAsync(stateProcessing, uploadedUri.ToString());
37+
}
38+
catch (Exception e)
39+
{
40+
await _processingStateService.MoveToFailedStateAsync(stateProcessing, e.Message);
41+
Console.WriteLine(e);
42+
throw;
43+
}
44+
finally
45+
{
46+
if (File.Exists(tmpFilePath))
47+
{
48+
File.Delete(tmpFilePath);
49+
}
50+
}
51+
}
52+
});
53+
}
54+
}

‎az-appservice-dotnet/services/v1/ProducerService.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public void StartProcessImage(IFileProviderService.FileObject fileObject)
3838
var state2 = await _processingStateService.MoveToUploadingStateAsync(state1);
3939
try
4040
{
41-
var blobUri = await _blobService.StoreBlobAsync(fileObject.Name, fileObject.Path);
41+
var blobUri = await _blobService.UploadBlobAsync(fileObject.Name, fileObject.Path);
4242
// TODO: not sure is it necessary to await the last call
4343
await _processingStateService.MoveToWaitingForProcessingStateAsync(state2, blobUri.ToString());
4444
}

‎az-appservice-dotnet/services/v1/State/IProcessingStateService.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public State WithFailedStatus(string? failureReason)
9999
}
100100
}
101101

102-
public delegate void StateChangeHandler(in State state);
102+
public delegate void StateChangeHandler(State state);
103103

104104
public Task<State> CreateInitialState(in TaskId taskId, in string fileName);
105105
public Task<State> MoveToUploadingStateAsync(in State state);
@@ -109,5 +109,5 @@ public State WithFailedStatus(string? failureReason)
109109
public Task<State> MoveToFailedStateAsync(in State state, string? failureReason);
110110
public Task<ImmutableDictionary<StateId, State>> GetStates();
111111

112-
public void ListenToStateChanges(in StateChangeHandler onStateChange);
112+
public void ListenToStateChanges(StateChangeHandler onStateChange);
113113
}

‎az-appservice-dotnet/services/v1/State/ProcessingStateService.cs

+4-19
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace az_appservice_dotnet.services.v1.State;
55

6-
public class ProcessingStateService : IProcessingStateService, IDisposable
6+
public class ProcessingStateService : IProcessingStateService
77
{
88
private readonly IPublishProcessingStateProvider _publishProcessingStateProvider;
99
private readonly ISubscribeProcessingStateProvider _subscribeProcessingStateProvider;
@@ -83,24 +83,9 @@ private IProcessingStateService.State PublishTask(in Task<IProcessingStateServic
8383
return _persistProcessingStateProvider.ListStatesAsync();
8484
}
8585

86-
87-
private IProcessingStateService.StateChangeHandler? _stateChangeHandler;
88-
public void ListenToStateChanges(in IProcessingStateService.StateChangeHandler onStateChange)
86+
public void ListenToStateChanges(IProcessingStateService.StateChangeHandler onStateChange)
8987
{
90-
if (_stateChangeHandler != null)
91-
{
92-
_subscribeProcessingStateProvider.RemoveStateChangeHandler(_stateChangeHandler);
93-
}
94-
95-
_stateChangeHandler = onStateChange;
96-
_subscribeProcessingStateProvider.AddStateChangeHandler(_stateChangeHandler);
97-
}
98-
99-
public void Dispose()
100-
{
101-
if (_stateChangeHandler != null)
102-
{
103-
_subscribeProcessingStateProvider.RemoveStateChangeHandler(_stateChangeHandler);
104-
}
88+
_subscribeProcessingStateProvider.RemoveStateChangeHandler(onStateChange);
89+
_subscribeProcessingStateProvider.AddStateChangeHandler(onStateChange);
10590
}
10691
}

0 commit comments

Comments
 (0)