Silo Metadata and Placement Filtering

src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs

@@ -12,6 +12,9 @@ namespace Orleans.Placement;
 [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
 public abstract class PlacementFilterAttribute : Attribute, IGrainPropertiesProviderAttribute
 {
+    /// <summary>
+    /// Gets the placement filter strategy.
+    /// </summary>
     public PlacementFilterStrategy PlacementFilterStrategy { get; private set; }
 
     protected PlacementFilterAttribute(PlacementFilterStrategy placement)

src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs

@@ -1,11 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using Orleans.Metadata;
 using Orleans.Runtime;
 
 #nullable enable
 namespace Orleans.Placement;
 
+/// <summary>
+/// Represents a strategy for filtering silos which a grain can be placed on.
+/// </summary>
 public abstract class PlacementFilterStrategy
 {
     public int Order { get; private set; }
@@ -36,7 +40,6 @@ public void Initialize(GrainProperties properties)
 
     public virtual void AdditionalInitialize(GrainProperties properties)
     {
-
     }
 
     /// <summary>
@@ -58,7 +61,7 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass,
             properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName;
         }
 
-        properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString();
+        properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(CultureInfo.InvariantCulture);
 
         foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties))
         {

src/Orleans.Core/Runtime/Constants.cs

@@ -12,6 +12,7 @@ internal static class Constants
         public static readonly GrainType DirectoryCacheValidatorType = SystemTargetGrainId.CreateGrainType("dir.cache-validator");
         public static readonly GrainType ClientDirectoryType = SystemTargetGrainId.CreateGrainType("dir.client");
         public static readonly GrainType SiloControlType = SystemTargetGrainId.CreateGrainType("silo-control");
+        public static readonly GrainType SiloMetadataType = SystemTargetGrainId.CreateGrainType("silo-metadata");
         public static readonly GrainType CatalogType = SystemTargetGrainId.CreateGrainType("catalog");
         public static readonly GrainType MembershipServiceType = SystemTargetGrainId.CreateGrainType("clustering");
         public static readonly GrainType SystemMembershipTableType = SystemTargetGrainId.CreateGrainType("clustering.dev");
@@ -27,8 +28,8 @@ internal static class Constants
         public static readonly GrainType ActivationMigratorType = SystemTargetGrainId.CreateGrainType("migrator");
         public static readonly GrainType ActivationRepartitionerType = SystemTargetGrainId.CreateGrainType("repartitioner");
         public static readonly GrainType ActivationRebalancerMonitorType = SystemTargetGrainId.CreateGrainType("rebalancer-monitor");
-        public static readonly GrainType GrainDirectoryPartition = SystemTargetGrainId.CreateGrainType("dir.grain.part");
-        public static readonly GrainType GrainDirectory = SystemTargetGrainId.CreateGrainType("dir.grain");
+        public static readonly GrainType GrainDirectoryPartitionType = SystemTargetGrainId.CreateGrainType("dir.grain.part");
+        public static readonly GrainType GrainDirectoryType = SystemTargetGrainId.CreateGrainType("dir.grain");
 
         public static readonly GrainId SiloDirectConnectionId = GrainId.Create(
             GrainType.Create(GrainTypePrefix.SystemPrefix + "silo"),
@@ -41,6 +42,7 @@ internal static class Constants
             {DirectoryServiceType, "DirectoryService"},
             {DirectoryCacheValidatorType, "DirectoryCacheValidator"},
             {SiloControlType, "SiloControl"},
+            {SiloMetadataType, "SiloMetadata"},
             {ClientDirectoryType, "ClientDirectory"},
             {CatalogType,"Catalog"},
             {MembershipServiceType,"MembershipService"},
@@ -57,7 +59,7 @@ internal static class Constants
             {ActivationMigratorType, "ActivationMigrator"},
             {ActivationRepartitionerType, "ActivationRepartitioner"},
             {ActivationRebalancerMonitorType, "ActivationRebalancerMonitor"},
-            {GrainDirectory, "GrainDirectory"},
+            {GrainDirectoryType, "GrainDirectory"},
         }.ToFrozenDictionary();
 
         public static string SystemTargetName(GrainType id) => SingletonSystemTargetNames.TryGetValue(id, out var name) ? name : id.ToString();

src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs

@@ -84,7 +84,7 @@ public DistributedGrainDirectory(
         ILocalSiloDetails localSiloDetails,
         ILoggerFactory loggerFactory,
         IServiceProvider serviceProvider,
-        IInternalGrainFactory grainFactory) : base(Constants.GrainDirectory, localSiloDetails.SiloAddress, loggerFactory)
+        IInternalGrainFactory grainFactory) : base(Constants.GrainDirectoryType, localSiloDetails.SiloAddress, loggerFactory)
     {
         _serviceProvider = serviceProvider;
         _membershipService = membershipService;

src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs

@@ -29,7 +29,7 @@ internal sealed partial class GrainDirectoryPartition(
     IInternalGrainFactory grainFactory)
     : SystemTarget(CreateGrainId(localSiloDetails.SiloAddress, partitionIndex), localSiloDetails.SiloAddress, loggerFactory), IGrainDirectoryPartition, IGrainDirectoryTestHooks
 {
-    internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartition, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture));
+    internal static SystemTargetGrainId CreateGrainId(SiloAddress siloAddress, int partitionIndex) => SystemTargetGrainId.Create(Constants.GrainDirectoryPartitionType, siloAddress, partitionIndex.ToString(CultureInfo.InvariantCulture));
     private readonly Dictionary<GrainId, GrainAddress> _directory = [];
     private readonly int _partitionIndex = partitionIndex;
     private readonly DistributedGrainDirectory _owner = owner;
@@ -665,7 +665,7 @@ private async IAsyncEnumerable<List<GrainAddress>> GetRegisteredActivations(Dire
         async Task<List<GrainAddress>> GetRegisteredActivationsFromClusterMember(MembershipVersion version, RingRange range, SiloAddress siloAddress, bool isValidation)
         {
             var stopwatch = ValueStopwatch.StartNew();
-            var client = _grainFactory.GetSystemTarget<IGrainDirectoryClient>(Constants.GrainDirectory, siloAddress);
+            var client = _grainFactory.GetSystemTarget<IGrainDirectoryClient>(Constants.GrainDirectoryType, siloAddress);
             var result = await InvokeOnClusterMember(
                 siloAddress,
                 async () =>

src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs

@@ -1,7 +1,8 @@
 #nullable enable
+
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
 public interface ISiloMetadataCache
 {
-    SiloMetadata GetMetadata(SiloAddress siloAddress);
+    SiloMetadata GetSiloMetadata(SiloAddress siloAddress);
 }

src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs

@@ -1,10 +1,9 @@
 using System.Threading.Tasks;
-using Orleans.Services;
 
 #nullable enable
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
-public interface ISiloMetadataClient : IGrainServiceClient<ISiloMetadataGrainService>
+internal interface ISiloMetadataClient
 {
     Task<SiloMetadata> GetSiloMetadata(SiloAddress siloAddress);
 }

src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs → src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataSystemTarget.cs

@@ -1,11 +1,10 @@
 using System.Threading.Tasks;
-using Orleans.Services;
 
 #nullable enable
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
-[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")]
-public interface ISiloMetadataGrainService : IGrainService
+[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataSystemTarget")]
+internal interface ISiloMetadataSystemTarget : ISystemTarget
 {
     [Alias("GetSiloMetadata")]
     Task<SiloMetadata> GetSiloMetadata();

src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs

@@ -5,8 +5,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using Orleans.Configuration;
-using Orleans.Internal;
 
 #nullable enable
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
@@ -22,44 +20,44 @@ internal class SiloMetadataCache(
 
     void ILifecycleParticipant<ISiloLifecycle>.Participate(ISiloLifecycle lifecycle)
     {
-        var tasks = new List<Task>(1);
-        var cancellation = new CancellationTokenSource();
-        Task OnRuntimeInitializeStart(CancellationToken _)
+        Task? task = null;
+        Task OnStart(CancellationToken _)
         {
-            tasks.Add(Task.Run(() => this.ProcessMembershipUpdates(cancellation.Token)));
+            task = Task.Run(() => this.ProcessMembershipUpdates(_cts.Token));
             return Task.CompletedTask;
         }
 
-        async Task OnRuntimeInitializeStop(CancellationToken ct)
+        async Task OnStop(CancellationToken ct)
         {
-            cancellation.Cancel(throwOnFirstException: false);
-            var shutdownGracePeriod = Task.WhenAll(Task.Delay(ClusterMembershipOptions.ClusteringShutdownGracePeriod), ct.WhenCancelled());
-            await Task.WhenAny(shutdownGracePeriod, Task.WhenAll(tasks));
+            await _cts.CancelAsync().ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
+            if (task is not null)
+            {
+                await task.WaitAsync(ct).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
+            }
         }
 
         lifecycle.Subscribe(
             nameof(ClusterMembershipService),
-            ServiceLifecycleStage.RuntimeInitialize,
-            OnRuntimeInitializeStart,
-            OnRuntimeInitializeStop);
+            ServiceLifecycleStage.RuntimeServices,
+            OnStart,
+            OnStop);
     }
 
-
     private async Task ProcessMembershipUpdates(CancellationToken ct)
     {
         try
         {
-            if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates");
+            if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates.");
             await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct))
             {
                 // Add entries for members that aren't already in the cache
-                foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status != SiloStatus.Dead))
+                foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status is SiloStatus.Active or SiloStatus.Joining))
                 {
                     if (!_metadata.ContainsKey(membershipEntry.Key))
                     {
                         try
                         {
-                            var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key);
+                            var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key).WaitAsync(ct);
                             _metadata.TryAdd(membershipEntry.Key, metadata);
                         }
                         catch(Exception exception)
@@ -85,6 +83,10 @@ private async Task ProcessMembershipUpdates(CancellationToken ct)
                 }
             }
         }
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        {
+            // Ignore and continue shutting down.
+        }
         catch (Exception exception)
         {
             logger.LogError(exception, "Error processing membership updates");
@@ -95,7 +97,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct)
         }
     }
 
-    public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty;
+    public SiloMetadata GetSiloMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty;
 
     public void SetMetadata(SiloAddress siloAddress, SiloMetadata metadata) => _metadata.TryAdd(siloAddress, metadata);
 

src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs

@@ -1,17 +1,14 @@
-using System;
 using System.Threading.Tasks;
-using Orleans.Runtime.Services;
 
 #nullable enable
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
-public class SiloMetadataClient(IServiceProvider serviceProvider)
-    : GrainServiceClient<ISiloMetadataGrainService>(serviceProvider), ISiloMetadataClient
+internal sealed class SiloMetadataClient(IInternalGrainFactory grainFactory) : ISiloMetadataClient
 {
     public async Task<SiloMetadata> GetSiloMetadata(SiloAddress siloAddress)
     {
-        var grainService = GetGrainService(siloAddress);
-        var metadata = await grainService.GetSiloMetadata();
+        var metadataSystemTarget = grainFactory.GetSystemTarget<ISiloMetadataSystemTarget>(Constants.SiloMetadataType, siloAddress);
+        var metadata = await metadataSystemTarget.GetSiloMetadata();
         return metadata;
     }
 }

src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs

@@ -1,23 +1,34 @@
+using System;
+using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 #nullable enable
 namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
-public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService
+internal sealed class SiloMetadataSystemTarget(
+    IOptions<SiloMetadata> siloMetadata,
+    ILocalSiloDetails localSiloDetails,
+    ILoggerFactory loggerFactory,
+    IServiceProvider serviceProvider)
+    : SystemTarget(Constants.SiloMetadataType, localSiloDetails.SiloAddress, loggerFactory), ISiloMetadataSystemTarget, ILifecycleParticipant<ISiloLifecycle>
 {
-    private readonly SiloMetadata _siloMetadata;
+    private readonly SiloMetadata _siloMetadata = siloMetadata.Value;
 
-    public SiloMetadataGrainService(IOptions<SiloMetadata> siloMetadata) : base()
-    {
-        _siloMetadata = siloMetadata.Value;
-    }
+    public Task<SiloMetadata> GetSiloMetadata() => Task.FromResult(_siloMetadata);
 
-    public SiloMetadataGrainService(IOptions<SiloMetadata> siloMetadata, GrainId grainId, Silo silo, ILoggerFactory loggerFactory) : base(grainId, silo, loggerFactory)
+    void ILifecycleParticipant<ISiloLifecycle>.Participate(ISiloLifecycle lifecycle)
     {
-        _siloMetadata = siloMetadata.Value;
-    }
+        lifecycle.Subscribe(nameof(SiloMetadataSystemTarget), ServiceLifecycleStage.RuntimeInitialize, OnRuntimeInitializeStart, OnRuntimeInitializeStop);
 
-    public Task<SiloMetadata> GetSiloMetadata() => Task.FromResult(_siloMetadata);
+        Task OnRuntimeInitializeStart(CancellationToken token)
+        {
+            serviceProvider.GetRequiredService<Catalog>().RegisterSystemTarget(this);
+            return Task.CompletedTask;
+        }
+
+        Task OnRuntimeInitializeStop(CancellationToken token) => Task.CompletedTask;
+    }
 }

src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs

@@ -11,7 +11,6 @@ namespace Orleans.Runtime.MembershipService.SiloMetadata;
 
 public static class SiloMetadataHostingExtensions
 {
-
     /// <summary>
     /// Configure silo metadata from the builder configuration.
     /// </summary>
@@ -58,7 +57,7 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfigura
     {
         var dictionary = configurationSection.Get<Dictionary<string, string>>();
 
-        return builder.UseSiloMetadata(dictionary ?? new Dictionary<string, string>());
+        return builder.UseSiloMetadata(dictionary ?? []);
     }
 
     /// <summary>
@@ -73,16 +72,15 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary
         {
             services
                 .AddOptionsWithValidateOnStart<SiloMetadata>()
-                .Configure(m =>
-                {
-                    m.AddMetadata(metadata);
-                });
+                .Configure(m => m.AddMetadata(metadata));
 
-            services.AddGrainService<SiloMetadataGrainService>();
+            services.AddSingleton<SiloMetadataSystemTarget>();
+            services.AddFromExisting<ILifecycleParticipant<ISiloLifecycle>, SiloMetadataSystemTarget>();
             services.AddSingleton<SiloMetadataCache>();
             services.AddFromExisting<ISiloMetadataCache, SiloMetadataCache>();
             services.AddFromExisting<ILifecycleParticipant<ISiloLifecycle>, SiloMetadataCache>();
             services.AddSingleton<ISiloMetadataClient, SiloMetadataClient>();
+
             // Placement filters
             services.AddPlacementFilter<PreferredMatchSiloMetadataPlacementFilterStrategy, PreferredMatchSiloMetadataPlacementFilterDirector>(ServiceLifetime.Transient);
             services.AddPlacementFilter<RequiredMatchSiloMetadataPlacementFilterStrategy, RequiredMatchSiloMetadataPlacementFilterDirector>(ServiceLifetime.Transient);

src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs

@@ -14,11 +14,11 @@ internal class PreferredMatchSiloMetadataPlacementFilterDirector(
 {
     public IEnumerable<SiloAddress> Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable<SiloAddress> silos)
     {
-        var preferredMatchSiloMetadataPlacementFilterStrategy = (filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy);
+        var preferredMatchSiloMetadataPlacementFilterStrategy = filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy;
         var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1;
         var orderedMetadataKeys = preferredMatchSiloMetadataPlacementFilterStrategy?.OrderedMetadataKeys ?? [];
 
-        var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata;
+        var localSiloMetadata = siloMetadataCache.GetSiloMetadata(localSiloDetails.SiloAddress).Metadata;
 
         if (localSiloMetadata.Count == 0)
         {
@@ -39,7 +39,7 @@ public IEnumerable<SiloAddress> Filter(PlacementFilterStrategy filterStrategy, P
         var scoreCounts = new int[orderedMetadataKeys.Length+1];
         for (var i = 0; i < siloList.Count; i++)
         {
-            var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata;
+            var siloMetadata = siloMetadataCache.GetSiloMetadata(siloList[i]).Metadata;
             var siloScore = 0;
             for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j)
             {

src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using Orleans.Metadata;
 using Orleans.Placement;
 
@@ -23,6 +24,7 @@ public override void AdditionalInitialize(GrainProperties properties)
         {
             throw new ArgumentException("Invalid ordered-metadata-keys property value.");
         }
+
         OrderedMetadataKeys = placementFilterGrainProperty.Split(",");
         var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties);
         if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates))
@@ -37,6 +39,6 @@ protected override IEnumerable<KeyValuePair<string, string>> GetAdditionalGrainP
         IReadOnlyDictionary<string, string> existingProperties)
     {
         yield return new KeyValuePair<string, string>("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys));
-        yield return new KeyValuePair<string, string>("min-candidates", MinCandidates.ToString());
+        yield return new KeyValuePair<string, string>("min-candidates", MinCandidates.ToString(CultureInfo.InvariantCulture));
     }
 }

src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs

@@ -19,12 +19,12 @@ public IEnumerable<SiloAddress> Filter(PlacementFilterStrategy filterStrategy, P
             return silos;
         }
 
-        var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress);
+        var localMetadata = siloMetadataCache.GetSiloMetadata(localSiloDetails.SiloAddress);
         var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys);
 
         return silos.Where(silo =>
         {
-            var remoteMetadata = siloMetadataCache.GetMetadata(silo);
+            var remoteMetadata = siloMetadataCache.GetSiloMetadata(silo);
             return DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys);
         });
     }
@@ -33,7 +33,7 @@ private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata silo
     {
         for (var i = 0; i < metadataKeys.Length; i++)
         {
-            if(localMetadata[i] != siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]))
+            if (localMetadata[i] != siloMetadata.Metadata.GetValueOrDefault(metadataKeys[i]))
             {
                 return false;
             }
@@ -46,8 +46,9 @@ private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata silo
         var result = new string?[metadataKeys.Length];
         for (var i = 0; i < metadataKeys.Length; i++)
         {
-            result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]);
+            result[i] = siloMetadata.Metadata.GetValueOrDefault(metadataKeys[i]);
         }
+
         return result;
     }
 }

src/Orleans.Runtime/Placement/PlacementService.cs

@@ -125,7 +125,6 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target)
 
             var compatibleSilos = silos.Intersect(AllActiveSilos).ToArray();
 
-
             var filters = _filterStrategyResolver.GetPlacementFilterStrategies(grainType);
             if (filters.Length > 0)
             {
@@ -139,7 +138,6 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target)
                 compatibleSilos = filteredSilos.ToArray();
             }
 
-
             if (compatibleSilos.Length == 0)
             {
                 var allWithType = _grainInterfaceVersions.GetSupportedSilos(grainType).Result;

src/Orleans.TestingHost/InProcTestCluster.cs

@@ -10,7 +10,6 @@
 using Orleans.Runtime;
 using Orleans.TestingHost.Utils;
 using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Configuration.Memory;
 using Orleans.Configuration;
 using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Hosting;
@@ -272,14 +271,34 @@ public static TimeSpan GetLivenessStabilizationTime(ClusterMembershipOptions clu
         {
             stabilizationTime += TestingUtils.Multiply(clusterMembershipOptions.TableRefreshTimeout, 2);
         }
+
         return stabilizationTime;
     }
 
     /// <summary>
     /// Start an additional silo, so that it joins the existing cluster.
     /// </summary>
     /// <returns>SiloHandle for the newly started silo.</returns>
-    public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort = false)
+    public InProcessSiloHandle StartAdditionalSilo()
+    {
+        return StartAdditionalSiloAsync().GetAwaiter().GetResult();
+    }
+
+    /// <summary>
+    /// Start an additional silo, so that it joins the existing cluster.
+    /// </summary>
+    /// <returns>SiloHandle for the newly started silo.</returns>
+    public async Task<InProcessSiloHandle> StartAdditionalSiloAsync()
+    {
+        return (await StartSilosAsync(1)).Single();
+    }
+
+    /// <summary>
+    /// Start an additional silo, so that it joins the existing cluster.
+    /// </summary>
+    /// <returns>SiloHandle for the newly started silo.</returns>
+    [Obsolete("Use overload which does not have a 'startAdditionalSiloOnNewPort' parameter.")]
+    public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort)
     {
         return StartAdditionalSiloAsync(startAdditionalSiloOnNewPort).GetAwaiter().GetResult();
     }
@@ -288,9 +307,9 @@ public InProcessSiloHandle StartAdditionalSilo(bool startAdditionalSiloOnNewPort
     /// Start an additional silo, so that it joins the existing cluster.
     /// </summary>
     /// <returns>SiloHandle for the newly started silo.</returns>
-    public async Task<InProcessSiloHandle> StartAdditionalSiloAsync(bool startAdditionalSiloOnNewPort = false)
+    public async Task<InProcessSiloHandle> StartAdditionalSiloAsync(bool startAdditionalSiloOnNewPort)
     {
-        return (await StartSilosAsync(1, startAdditionalSiloOnNewPort)).Single();
+        return (await StartSilosAsync(1)).Single();
     }
 
     /// <summary>
@@ -299,13 +318,24 @@ public async Task<InProcessSiloHandle> StartAdditionalSiloAsync(bool startAdditi
     /// <param name="silosToStart">Number of silos to start.</param>
     /// <param name="startAdditionalSiloOnNewPort"></param>
     /// <returns>List of SiloHandles for the newly started silos.</returns>
-    public async Task<List<InProcessSiloHandle>> StartSilosAsync(int silosToStart, bool startAdditionalSiloOnNewPort = false)
+    [Obsolete("Use overload which does not have a 'startAdditionalSiloOnNewPort' parameter.")]
+    public async Task<List<InProcessSiloHandle>> StartSilosAsync(int silosToStart, bool startAdditionalSiloOnNewPort)
+    {
+        return await StartSilosAsync(silosToStart);
+    }
+
+    /// <summary>
+    /// Start a number of additional silo, so that they join the existing cluster.
+    /// </summary>
+    /// <param name="silosToStart">Number of silos to start.</param>
+    /// <returns>List of SiloHandles for the newly started silos.</returns>
+    public async Task<List<InProcessSiloHandle>> StartSilosAsync(int silosToStart)
     {
         var instances = new List<InProcessSiloHandle>();
         if (silosToStart > 0)
         {
             var siloStartTasks = Enumerable.Range(_startedInstances, silosToStart)
-                .Select(instanceNumber => Task.Run(() => StartSiloAsync((short)instanceNumber, Options, startSiloOnNewPort: startAdditionalSiloOnNewPort))).ToArray();
+                .Select(instanceNumber => Task.Run(() => StartSiloAsync((short)instanceNumber, Options))).ToArray();
 
             try
             {
@@ -622,6 +652,19 @@ public async Task<InProcessSiloHandle> CreateSiloAsync(InProcessTestSiloSpecific
         };
     }
 
+    /// <summary>
+    /// Start a new silo in the target cluster
+    /// </summary>
+    /// <param name="cluster">The InProcessTestCluster in which the silo should be deployed</param>
+    /// <param name="instanceNumber">The instance number to deploy</param>
+    /// <param name="clusterOptions">The options to use.</param>
+    /// <returns>A handle to the silo deployed</returns>
+    public static async Task<InProcessSiloHandle> StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions)
+    {
+        if (cluster == null) throw new ArgumentNullException(nameof(cluster));
+        return await cluster.StartSiloAsync(instanceNumber, clusterOptions);
+    }
+
     /// <summary>
     /// Start a new silo in the target cluster
     /// </summary>
@@ -631,7 +674,8 @@ public async Task<InProcessSiloHandle> CreateSiloAsync(InProcessTestSiloSpecific
     /// <param name="configurationOverrides">Configuration overrides.</param>
     /// <param name="startSiloOnNewPort">Whether we start this silo on a new port, instead of the default one</param>
     /// <returns>A handle to the silo deployed</returns>
-    public static async Task<InProcessSiloHandle> StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList<IConfigurationSource> configurationOverrides = null, bool startSiloOnNewPort = false)
+    [Obsolete("Use the overload which does not have a 'startSiloOnNewPort' parameter.")]
+    public static async Task<InProcessSiloHandle> StartSiloAsync(InProcessTestCluster cluster, int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList<IConfigurationSource> configurationOverrides, bool startSiloOnNewPort)
     {
         if (cluster == null) throw new ArgumentNullException(nameof(cluster));
         return await cluster.StartSiloAsync(instanceNumber, clusterOptions, configurationOverrides, startSiloOnNewPort);
@@ -642,18 +686,30 @@ public static async Task<InProcessSiloHandle> StartSiloAsync(InProcessTestCluste
     /// </summary>
     /// <param name="instanceNumber">The instance number to deploy</param>
     /// <param name="clusterOptions">The options to use.</param>
-    /// <param name="configurationOverrides">Configuration overrides.</param>
-    /// <param name="startSiloOnNewPort">Whether we start this silo on a new port, instead of the default one</param>
     /// <returns>A handle to the deployed silo.</returns>
-    public async Task<InProcessSiloHandle> StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList<IConfigurationSource> configurationOverrides = null, bool startSiloOnNewPort = false)
+    public async Task<InProcessSiloHandle> StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions)
     {
-        var siloOptions = InProcessTestSiloSpecificOptions.Create(this, clusterOptions, instanceNumber, startSiloOnNewPort);
+        var siloOptions = InProcessTestSiloSpecificOptions.Create(this, clusterOptions, instanceNumber, assignNewPort: true);
         var handle = await CreateSiloAsync(siloOptions);
         handle.InstanceNumber = (short)instanceNumber;
         Interlocked.Increment(ref _startedInstances);
         return handle;
     }
 
+    /// <summary>
+    /// Starts a new silo.
+    /// </summary>
+    /// <param name="instanceNumber">The instance number to deploy</param>
+    /// <param name="clusterOptions">The options to use.</param>
+    /// <param name="configurationOverrides">Configuration overrides.</param>
+    /// <param name="startSiloOnNewPort">Whether we start this silo on a new port, instead of the default one</param>
+    /// <returns>A handle to the deployed silo.</returns>
+    [Obsolete("Use the overload which does not have a 'startSiloOnNewPort' parameter.")]
+    public async Task<InProcessSiloHandle> StartSiloAsync(int instanceNumber, InProcessTestClusterOptions clusterOptions, IReadOnlyList<IConfigurationSource> configurationOverrides, bool startSiloOnNewPort)
+    {
+        return await StartSiloAsync(instanceNumber, clusterOptions);
+    }
+
     private async Task StopSiloAsync(InProcessSiloHandle instance, bool stopGracefully)
     {
         try

test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs

@@ -2,38 +2,45 @@
 using Orleans.Runtime.MembershipService.SiloMetadata;
 using Orleans.Runtime.Placement.Filtering;
 using Orleans.TestingHost;
-using TestExtensions;
 using Xunit;
 
 namespace UnitTests.PlacementFilterTests;
 
 [TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")]
-public class SiloMetadataPlacementFilterTests : TestClusterPerTest
+public class SiloMetadataPlacementFilterTests(SiloMetadataPlacementFilterTests.Fixture fixture) : IClassFixture<SiloMetadataPlacementFilterTests.Fixture>
 {
-    protected override void ConfigureTestCluster(TestClusterBuilder builder)
+    public class Fixture : IAsyncLifetime
     {
-        builder.AddSiloBuilderConfigurator<SiloConfigurator>();
-    }
+        public InProcessTestCluster Cluster { get; private set; }
+        public async Task DisposeAsync()
+        {
+            if (Cluster is { } cluster)
+            {
+                await cluster.DisposeAsync();
+            }
+        }
 
-    private class SiloConfigurator : ISiloConfigurator
-    {
-        public void Configure(ISiloBuilder hostBuilder)
+        public async Task InitializeAsync()
         {
-            hostBuilder.UseSiloMetadata(new Dictionary<string, string>
+            var builder = new InProcessTestClusterBuilder(3);
+            builder.ConfigureSilo((options, siloBuilder) => siloBuilder.UseSiloMetadata(new Dictionary<string, string>
             {
                 {"first", "1"},
                 {"second", "2"},
                 {"third", "3"},
                 {"unique", Guid.NewGuid().ToString()}
-            });
+            }));
+
+            Cluster = builder.Build();
+            await Cluster.DeployAsync();
+            await Cluster.WaitForLivenessToStabilizeAsync();
         }
     }
 
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_GrainWithoutFilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        var managementGrain = Client.GetGrain<IManagementGrain>(0);
+        var managementGrain = fixture.Cluster.Client.GetGrain<IManagementGrain>(0);
         var silos = await managementGrain.GetHosts(true);
         Assert.NotNull(silos);
     }
@@ -45,14 +52,13 @@ public async Task PlacementFilter_GrainWithoutFilterCanBeCalled()
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_RequiredFilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IUniqueRequiredMatchFilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();
@@ -69,14 +75,13 @@ public async Task PlacementFilter_RequiredFilterCanBeCalled()
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_PreferredFilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IPreferredMatchFilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();
@@ -93,19 +98,19 @@ public async Task PlacementFilter_PreferredFilterCanBeCalled()
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_PreferredMin2FilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             var dict = new Dictionary<SiloAddress, int>();
-            foreach (var clusterSilo in HostedCluster.Silos)
+            foreach (var clusterSilo in fixture.Cluster.Silos)
             {
                 dict[clusterSilo.SiloAddress] = 0;
             }
+
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IPreferredMatchMin2FilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();
@@ -127,19 +132,19 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled()
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             var dict = new Dictionary<SiloAddress, int>();
-            foreach (var clusterSilo in HostedCluster.Silos)
+            foreach (var clusterSilo in fixture.Cluster.Silos)
             {
                 dict[clusterSilo.SiloAddress] = 0;
             }
+
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IPreferredMatchMultipleFilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();
@@ -161,21 +166,18 @@ public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled()
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        await HostedCluster.StartAdditionalSiloAsync();
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             var dict = new Dictionary<SiloAddress, int>();
-            foreach (var clusterSilo in HostedCluster.Silos)
+            foreach (var clusterSilo in fixture.Cluster.Silos)
             {
                 dict[clusterSilo.SiloAddress] = 0;
             }
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IPreferredMatchMin2FilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();
@@ -197,21 +199,18 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste
     [Fact, TestCategory("Functional")]
     public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        await HostedCluster.StartAdditionalSiloAsync();
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
         var id = 0;
-        foreach (var hostedClusterSilo in HostedCluster.Silos)
+        foreach (var hostedClusterSilo in fixture.Cluster.Silos)
         {
             var dict = new Dictionary<SiloAddress, int>();
-            foreach (var clusterSilo in HostedCluster.Silos)
+            foreach (var clusterSilo in fixture.Cluster.Silos)
             {
                 dict[clusterSilo.SiloAddress] = 0;
             }
             for (var i = 0; i < 50; i++)
             {
                 ++id;
-                var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
+                var firstSp = fixture.Cluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress);
                 var firstSiloMetadataCache = firstSp.GetRequiredService<IClusterClient>();
                 var managementGrain = firstSiloMetadataCache.GetGrain<IPreferredMatchNoMetadataFilteredGrain>(id);
                 var hostingSilo = await managementGrain.GetHostingSilo();

test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs

@@ -11,5 +11,5 @@ public TestSiloMetadataCache(Dictionary<SiloAddress, SiloMetadata> metadata)
         _metadata = metadata;
     }
 
-    public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? new SiloMetadata();
+    public SiloMetadata GetSiloMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty;
 }

test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs

@@ -2,142 +2,133 @@
 using Microsoft.Extensions.DependencyInjection;
 using Orleans.Runtime.MembershipService.SiloMetadata;
 using Orleans.TestingHost;
-using TestExtensions;
 using Xunit;
 
 namespace UnitTests.SiloMetadataTests;
 
-
 [TestCategory("SiloMetadata")]
-public class SiloMetadataConfigTests : TestClusterPerTest
+public class SiloMetadataTests(SiloMetadataTests.Fixture fixture) : IClassFixture<SiloMetadataTests.Fixture>
 {
-    protected override void ConfigureTestCluster(TestClusterBuilder builder)
-    {
-        builder.AddSiloBuilderConfigurator<SiloConfigurator>();
-    }
-
-    private class SiloConfigurator : ISiloConfigurator
-    {
-        public static readonly List<KeyValuePair<string, string>> Metadata =
+    private static readonly List<KeyValuePair<string, string>> Metadata =
         [
             new("Orleans:Metadata:first", "1"),
             new("Orleans:Metadata:second", "2"),
             new("Orleans:Metadata:third", "3")
         ];
 
-        public void Configure(ISiloBuilder hostBuilder)
+    public class Fixture : IAsyncLifetime
+    {
+        public InProcessTestCluster Cluster { get; private set; }
+        public async Task DisposeAsync()
+        {
+            if (Cluster is { } cluster)
+            {
+                await cluster.DisposeAsync();
+            }
+        }
+
+        public async Task InitializeAsync()
         {
-            var config = new ConfigurationBuilder()
-                .AddInMemoryCollection(Metadata)
-                .Build();
-            hostBuilder.UseSiloMetadata(config);
+            var builder = new InProcessTestClusterBuilder(3);
+            builder.ConfigureSiloHost((options, hostBuilder) =>
+            {
+                hostBuilder.Configuration.AddInMemoryCollection(Metadata);
+            });
+
+            builder.ConfigureSilo((options, siloBuilder) =>
+            {
+                siloBuilder
+                .UseSiloMetadata()
+                .UseSiloMetadata(new Dictionary<string, string>
+                {
+                    {"host.id", Guid.NewGuid().ToString()}
+                });
+            });
+
+            Cluster = builder.Build();
+            await Cluster.DeployAsync();
+            await Cluster.WaitForLivenessToStabilizeAsync();
         }
     }
 
     [Fact, TestCategory("Functional")]
-    public async Task SiloMetadata_CanBeSetAndRead()
+    public void SiloMetadata_FromConfiguration_CanBeSetAndRead()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray());
+        fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray());
     }
 
     [Fact, TestCategory("Functional")]
-    public async Task SiloMetadata_HasConfiguredValues()
+    public void SiloMetadata_HasConfiguredValues()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-
-        var first = HostedCluster.Silos.First();
-        var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress);
+        var first = fixture.Cluster.Silos.First();
+        var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress);
         var firstSiloMetadataCache = firstSp.GetRequiredService<ISiloMetadataCache>();
-        var metadata = firstSiloMetadataCache.GetMetadata(first.SiloAddress);
+        var metadata = firstSiloMetadataCache.GetSiloMetadata(first.SiloAddress);
         Assert.NotNull(metadata);
         Assert.NotNull(metadata.Metadata);
-        Assert.Equal(SiloConfigurator.Metadata.Count, metadata.Metadata.Count);
-        foreach (var kv in SiloConfigurator.Metadata)
+        Assert.True(metadata.Metadata.Count >= Metadata.Count);
+        foreach (var kv in Metadata)
         {
             Assert.Equal(kv.Value, metadata.Metadata[kv.Key.Split(':').Last()]);
         }
     }
-}
-
-[TestCategory("SiloMetadata")]
-public class SiloMetadataTests : TestClusterPerTest
-{
-    protected override void ConfigureTestCluster(TestClusterBuilder builder)
-    {
-        builder.AddSiloBuilderConfigurator<SiloConfigurator>();
-    }
-
-    private class SiloConfigurator : ISiloConfigurator
-    {
-        public void Configure(ISiloBuilder hostBuilder)
-        {
-            hostBuilder.UseSiloMetadata(new Dictionary<string, string>
-            {
-                {"host.id", Guid.NewGuid().ToString()}
-            });
-        }
-    }
 
     [Fact, TestCategory("Functional")]
-    public async Task SiloMetadata_CanBeSetAndRead()
+    public void SiloMetadata_CanBeSetAndRead()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
+        fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
     }
 
     [Fact, TestCategory("Functional")]
     public async Task SiloMetadata_NewSilosHaveMetadata()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        await HostedCluster.StartAdditionalSiloAsync();
-        HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
+        await fixture.Cluster.StartAdditionalSiloAsync();
+        await fixture.Cluster.WaitForLivenessToStabilizeAsync();
+        fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
     }
 
     [Fact, TestCategory("Functional")]
     public async Task SiloMetadata_RemovedSiloHasNoMetadata()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
-        var first = HostedCluster.Silos.First();
-        var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress);
+        fixture.Cluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]);
+        var first = fixture.Cluster.Silos.First();
+        var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress);
         var firstSiloMetadataCache = firstSp.GetRequiredService<ISiloMetadataCache>();
 
-        var second = HostedCluster.Silos.Skip(1).First();
-        var metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress);
+        var second = fixture.Cluster.Silos.Skip(1).First();
+        var metadata = firstSiloMetadataCache.GetSiloMetadata(second.SiloAddress);
         Assert.NotNull(metadata);
         Assert.NotEmpty(metadata.Metadata);
 
-        await HostedCluster.StopSiloAsync(second);
-        metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress);
+        await fixture.Cluster.StopSiloAsync(second);
+        metadata = firstSiloMetadataCache.GetSiloMetadata(second.SiloAddress);
         Assert.NotNull(metadata);
         Assert.Empty(metadata.Metadata);
     }
 
     [Fact, TestCategory("Functional")]
-    public async Task SiloMetadata_BadSiloAddressHasNoMetadata()
+    public void SiloMetadata_BadSiloAddressHasNoMetadata()
     {
-        await HostedCluster.WaitForLivenessToStabilizeAsync();
-        var first = HostedCluster.Silos.First();
-        var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress);
+        var first = fixture.Cluster.Silos.First();
+        var firstSp = fixture.Cluster.GetSiloServiceProvider(first.SiloAddress);
         var firstSiloMetadataCache = firstSp.GetRequiredService<ISiloMetadataCache>();
-        var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero);
+        var metadata = firstSiloMetadataCache.GetSiloMetadata(SiloAddress.Zero);
         Assert.NotNull(metadata);
         Assert.Empty(metadata.Metadata);
     }
 }
 
 public static class SiloMetadataTestExtensions
 {
-    public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hostedCluster, string[] expectedKeys)
+    public static void AssertAllSiloMetadataMatchesOnAllSilos(this InProcessTestCluster hostedCluster, string[] expectedKeys)
     {
         var exampleSiloMetadata = new Dictionary<SiloAddress, SiloMetadata>();
         var first = hostedCluster.Silos.First();
         var firstSp = hostedCluster.GetSiloServiceProvider(first.SiloAddress);
         var firstSiloMetadataCache = firstSp.GetRequiredService<ISiloMetadataCache>();
         foreach (var otherSilo in hostedCluster.Silos)
         {
-            var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress);
+            var metadata = firstSiloMetadataCache.GetSiloMetadata(otherSilo.SiloAddress);
             Assert.NotNull(metadata);
             Assert.NotNull(metadata.Metadata);
             foreach (var expectedKey in expectedKeys)
@@ -154,7 +145,7 @@ public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hoste
             var remoteMetadata = new Dictionary<SiloAddress, SiloMetadata>();
             foreach (var otherSilo in hostedCluster.Silos)
             {
-                var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress);
+                var metadata = siloMetadataCache.GetSiloMetadata(otherSilo.SiloAddress);
                 Assert.NotNull(metadata);
                 Assert.NotNull(metadata.Metadata);
                 foreach (var expectedKey in expectedKeys)