From b0b61114dfa0cfd6225dcdc4ac37822a1f97c42e Mon Sep 17 00:00:00 2001 From: Melih Odabas Date: Sat, 8 Jul 2023 11:56:37 +0300 Subject: [PATCH 1/3] -add DeleteStateOnClear option and support to AdoNet grain storage provider. -add delete on clear script for postgresql --- .../Options/AdoNetGrainStorageOptions.cs | 5 +++ .../PostgreSQL-Persistence.sql | 17 ++++++++++ .../Storage/Provider/AdoNetGrainStorage.cs | 31 +++++++++++++++---- .../RelationalStorageProviderQueries.cs | 12 +++++-- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/Options/AdoNetGrainStorageOptions.cs b/src/AdoNet/Orleans.Persistence.AdoNet/Options/AdoNetGrainStorageOptions.cs index b3a7e18f87..22c5decad7 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/Options/AdoNetGrainStorageOptions.cs +++ b/src/AdoNet/Orleans.Persistence.AdoNet/Options/AdoNetGrainStorageOptions.cs @@ -35,6 +35,11 @@ public class AdoNetGrainStorageOptions : IStorageProviderSerializerOptions /// public IGrainStorageSerializer GrainStorageSerializer { get; set; } + + /// + /// Delete record row from db when state is cleared. + /// + public bool DeleteStateOnClear { get; set; } = false; } /// diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql b/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql index 4767a38dff..b3e1526fb8 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql +++ b/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql @@ -180,3 +180,20 @@ VALUES AND Version IS NOT NULL AND Version = @GrainStateVersion AND @GrainStateVersion IS NOT NULL Returning Version as NewGrainStateVersion '); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'DeleteStorageKey',' + DELETE FROM OrleansStorage + WHERE + GrainIdHash = @GrainIdHash AND @GrainIdHash IS NOT NULL + AND GrainTypeHash = @GrainTypeHash AND @GrainTypeHash IS NOT NULL + AND GrainIdN0 = @GrainIdN0 AND @GrainIdN0 IS NOT NULL + AND GrainIdN1 = @GrainIdN1 AND @GrainIdN1 IS NOT NULL + AND GrainTypeString = @GrainTypeString AND @GrainTypeString IS NOT NULL + AND ((@GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString = @GrainIdExtensionString) OR @GrainIdExtensionString IS NULL AND GrainIdExtensionString IS NULL) + AND ServiceId = @ServiceId AND @ServiceId IS NOT NULL + AND Version IS NOT NULL AND Version = @GrainStateVersion AND @GrainStateVersion IS NOT NULL + Returning NULL as NewGrainStateVersion +'); diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/AdoNetGrainStorage.cs b/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/AdoNetGrainStorage.cs index 2479c7c5d7..da3cd421c8 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/AdoNetGrainStorage.cs +++ b/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/AdoNetGrainStorage.cs @@ -108,7 +108,7 @@ public class AdoNetGrainStorage: IGrainStorage, ILifecycleParticipant /// The default query to initialize this structure from the Orleans database. /// - public const string DefaultInitializationQuery = "SELECT QueryKey, QueryText FROM OrleansQuery WHERE QueryKey = 'WriteToStorageKey' OR QueryKey = 'ReadFromStorageKey' OR QueryKey = 'ClearStorageKey'"; + public const string DefaultInitializationQuery = "SELECT QueryKey, QueryText FROM OrleansQuery WHERE QueryKey = 'WriteToStorageKey' OR QueryKey = 'ReadFromStorageKey' OR QueryKey = 'ClearStorageKey' OR QueryKey = 'DeleteStorageKey'"; /// /// The queries currently used. When this is updated, the new queries will take effect immediately. @@ -155,12 +155,13 @@ public async Task ClearStateAsync(string grainType, GrainId grainReference, I { logger.LogTrace( (int)RelationalStorageProviderCodes.RelationalProviderClearing, - "Clearing grain state: ServiceId={ServiceId} ProviderName={Name} GrainType={BaseGrainType} GrainId={GrainId} ETag={ETag}.", + "Clearing grain state: ServiceId={ServiceId} ProviderName={Name} GrainType={BaseGrainType} GrainId={GrainId} ETag={ETag} DeleteStateOnClear={DeleteStateOnClear}.", serviceId, name, baseGrainType, grainId, - grainState.ETag); + grainState.ETag, + options.DeleteStateOnClear); } string storageVersion = null; @@ -168,7 +169,15 @@ public async Task ClearStateAsync(string grainType, GrainId grainReference, I { var grainIdHash = HashPicker.PickHasher(serviceId, this.name, baseGrainType, grainReference, grainState).Hash(grainId.GetHashBytes()); var grainTypeHash = HashPicker.PickHasher(serviceId, this.name, baseGrainType, grainReference, grainState).Hash(Encoding.UTF8.GetBytes(baseGrainType)); - var clearRecord = (await Storage.ReadAsync(CurrentOperationalQueries.ClearState, command => + + var queryText = options.DeleteStateOnClear ? CurrentOperationalQueries.DeleteState : CurrentOperationalQueries.ClearState; + //Backward compatibility checks in Init should handle this case and throw prior to reaching this state. + if (queryText is null) + { + //We should not be here. + throw new UnreachableException($"QueryText is null for options.DeleteStateOnClear={options.DeleteStateOnClear}"); + } + var clearRecord = (await Storage.ReadAsync(queryText, command => { command.AddParameter("GrainIdHash", grainIdHash); command.AddParameter("GrainIdN0", grainId.N0Key); @@ -202,8 +211,9 @@ public async Task ClearStateAsync(string grainType, GrainId grainReference, I throw inconsistentStateException; } + //if delete on clear is set, ETag should be set to null since record has been deleted, but db script returns deleted record version + 1 as storageVersion for CheckVersionInconsistency method. //No errors found, the version of the state held by the grain can be updated and also the state. - grainState.ETag = storageVersion; + grainState.ETag = options.DeleteStateOnClear ? null : storageVersion; grainState.RecordExists = false; if(logger.IsEnabled(LogLevel.Trace)) { @@ -409,10 +419,19 @@ private async Task Init(CancellationToken cancellationToken) return Task.FromResult(Tuple.Create(selector.GetValue("QueryKey"), selector.GetValue("QueryText"))); }).ConfigureAwait(false); + //This check is for backward compatibility: + //1. Some AdoNet storage invariants may not support delete on clear + //2. AdoNet invariant supports delete on clear but updated persistence scripts have not been deployed to management db. + var deleteStateQuery = queries.SingleOrDefault(i => i.Item1 == "DeleteStorageKey")?.Item2; + if (options.DeleteStateOnClear && deleteStateQuery is null) + { + throw new ArgumentException("DeleteStateOnClear=true option is not supported. Use options.DeleteStateOnClear=false instead or check persistence scripts."); + } CurrentOperationalQueries = new RelationalStorageProviderQueries( queries.Single(i => i.Item1 == "WriteToStorageKey").Item2, queries.Single(i => i.Item1 == "ReadFromStorageKey").Item2, - queries.Single(i => i.Item1 == "ClearStorageKey").Item2); + queries.Single(i => i.Item1 == "ClearStorageKey").Item2, + deleteStateQuery); logger.LogInformation( (int)RelationalStorageProviderCodes.RelationalProviderInitProvider, diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/RelationalStorageProviderQueries.cs b/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/RelationalStorageProviderQueries.cs index 6d1b29f264..7e06077d7a 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/RelationalStorageProviderQueries.cs +++ b/src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/RelationalStorageProviderQueries.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace Orleans.Storage @@ -24,6 +24,11 @@ public class RelationalStorageProviderQueries /// public string ClearState { get; set; } + /// + /// The clause to clear the storage, deleting row in db. + /// + public string DeleteState { get; set; } + /// /// Constructor. @@ -31,7 +36,7 @@ public class RelationalStorageProviderQueries /// The clause to write to a storage. /// The clause to read from a storage. /// The clause to clear the storage. - public RelationalStorageProviderQueries(string writeToStorage, string readFromStorage, string clearState) + public RelationalStorageProviderQueries(string writeToStorage, string readFromStorage, string clearState, string deleteState) { if(writeToStorage == null) { @@ -48,9 +53,12 @@ public RelationalStorageProviderQueries(string writeToStorage, string readFromSt throw new ArgumentNullException(nameof(clearState)); } + //no null check on deleteState for backward compatibility + WriteToStorage = writeToStorage; ReadFromStorage = readFromStorage; ClearState = clearState; + DeleteState = deleteState; } } } From bff641c5bfad3bef51b8144b78f4057a57f07a77 Mon Sep 17 00:00:00 2001 From: Melih Odabas Date: Sat, 8 Jul 2023 13:43:20 +0300 Subject: [PATCH 2/3] add delete on clear script for mssql --- .../SQLServer-Persistence.sql | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/SQLServer-Persistence.sql b/src/AdoNet/Orleans.Persistence.AdoNet/SQLServer-Persistence.sql index 4338b99ad1..34533fdf42 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/SQLServer-Persistence.sql +++ b/src/AdoNet/Orleans.Persistence.AdoNet/SQLServer-Persistence.sql @@ -246,3 +246,24 @@ VALUES AND ServiceId = @ServiceId AND @ServiceId IS NOT NULL OPTION(FAST 1, OPTIMIZE FOR(@GrainIdHash UNKNOWN, @GrainTypeHash UNKNOWN));' ); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'DeleteStorageKey', + 'BEGIN TRANSACTION; + SET XACT_ABORT, NOCOUNT ON; + DELETE FROM OrleansStorage OUTPUT DELETED.Version + 1 + WHERE + GrainIdHash = @GrainIdHash AND @GrainIdHash IS NOT NULL + AND GrainTypeHash = @GrainTypeHash AND @GrainTypeHash IS NOT NULL + AND (GrainIdN0 = @GrainIdN0 OR @GrainIdN0 IS NULL) + AND (GrainIdN1 = @GrainIdN1 OR @GrainIdN1 IS NULL) + AND (GrainTypeString = @GrainTypeString OR @GrainTypeString IS NULL) + AND ((@GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString = @GrainIdExtensionString) OR @GrainIdExtensionString IS NULL AND GrainIdExtensionString IS NULL) + AND ServiceId = @ServiceId AND @ServiceId IS NOT NULL + AND Version IS NOT NULL AND Version = @GrainStateVersion AND @GrainStateVersion IS NOT NULL + OPTION(FAST 1, OPTIMIZE FOR(@GrainIdHash UNKNOWN, @GrainTypeHash UNKNOWN)); + + COMMIT TRANSACTION;' +); From 152f004f0a70bb38d3da005a1e1add88e630316a Mon Sep 17 00:00:00 2001 From: Melih Odabas Date: Sat, 8 Jul 2023 14:25:23 +0300 Subject: [PATCH 3/3] fix returned version in postgresql delete on clear state script --- .../Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql b/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql index b3e1526fb8..c52fcc2f20 100644 --- a/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql +++ b/src/AdoNet/Orleans.Persistence.AdoNet/PostgreSQL-Persistence.sql @@ -195,5 +195,5 @@ VALUES AND ((@GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString IS NOT NULL AND GrainIdExtensionString = @GrainIdExtensionString) OR @GrainIdExtensionString IS NULL AND GrainIdExtensionString IS NULL) AND ServiceId = @ServiceId AND @ServiceId IS NOT NULL AND Version IS NOT NULL AND Version = @GrainStateVersion AND @GrainStateVersion IS NOT NULL - Returning NULL as NewGrainStateVersion + Returning Version + 1 as NewGrainStateVersion ');