-
Notifications
You must be signed in to change notification settings - Fork 543
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Enable replicaSet support for MongoDb #5712
base: main
Are you sure you want to change the base?
Changes from all commits
350ce9f
3b483d8
70e1b29
a3c4fd2
f34d9bb
727264e
6d4ac1d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
@@ -1,6 +1,7 @@ | ||||||||||||||
// Licensed to the .NET Foundation under one or more agreements. | ||||||||||||||
// The .NET Foundation licenses this file to you under the MIT license. | ||||||||||||||
|
||||||||||||||
using System.Text; | ||||||||||||||
using Aspire.Hosting.ApplicationModel; | ||||||||||||||
using Aspire.Hosting.MongoDB; | ||||||||||||||
using Aspire.Hosting.Utils; | ||||||||||||||
|
@@ -94,7 +95,8 @@ public static IResourceBuilder<T> WithMongoExpress<T>(this IResourceBuilder<T> b | |||||||||||||
.WithImageRegistry(MongoDBContainerImageTags.MongoExpressRegistry) | ||||||||||||||
.WithEnvironment(context => ConfigureMongoExpressContainer(context, builder.Resource)) | ||||||||||||||
.WithHttpEndpoint(targetPort: 8081, name: "http") | ||||||||||||||
.ExcludeFromManifest(); | ||||||||||||||
.ExcludeFromManifest() | ||||||||||||||
.WaitFor(builder); | ||||||||||||||
|
||||||||||||||
configureContainer?.Invoke(resourceBuilder); | ||||||||||||||
|
||||||||||||||
|
@@ -161,11 +163,96 @@ public static IResourceBuilder<MongoDBServerResource> WithInitBindMount(this IRe | |||||||||||||
return builder.WithBindMount(source, "/docker-entrypoint-initdb.d", isReadOnly); | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
/// <summary> | ||||||||||||||
/// Adds a replica set to the MongoDB server resource. | ||||||||||||||
/// </summary> | ||||||||||||||
/// <param name="builder">The MongoDB server resource.</param> | ||||||||||||||
/// <param name="replicaSetName">The name of the replica set. If not provided, defaults to <c>rs0</c>.</param> | ||||||||||||||
/// <returns>A reference to the <see cref="IResourceBuilder{T}"/>.</returns> | ||||||||||||||
public static IResourceBuilder<MongoDBServerResource> WithReplicaSet(this IResourceBuilder<MongoDBServerResource> builder, string? replicaSetName = null) | ||||||||||||||
{ | ||||||||||||||
if (builder.Resource.TryGetLastAnnotation<MongoDbReplicaSetAnnotation>(out _)) | ||||||||||||||
{ | ||||||||||||||
throw new InvalidOperationException("A replica set has already been added to the MongoDB server resource."); | ||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why throw instead of noop? Because of the name? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah. I can check if there's already one with the same name and return in those cases if you want |
||||||||||||||
} | ||||||||||||||
|
||||||||||||||
replicaSetName ??= "rs0"; | ||||||||||||||
|
||||||||||||||
var port = SetPortAndTargetToBeSame(builder); | ||||||||||||||
|
||||||||||||||
// Add a container that initializes the replica set | ||||||||||||||
var init = builder.ApplicationBuilder | ||||||||||||||
.AddDockerfile("replicaset-init", GetReplicaSetInitDockerfileDir(replicaSetName, builder.Resource.Name, port)) | ||||||||||||||
|
||||||||||||||
// We don't want to wait for the healthchecks to be successful since the initialization is required for that. However, we also don't want this to start | ||||||||||||||
// up until the database itself is ready | ||||||||||||||
.WaitFor(builder, includeHealthChecks: false); | ||||||||||||||
|
||||||||||||||
return builder | ||||||||||||||
.WithAnnotation(new MongoDbReplicaSetAnnotation(replicaSetName, init)) | ||||||||||||||
.WithArgs("--replSet", replicaSetName, "--bind_ip_all", "--port", $"{port}"); | ||||||||||||||
|
||||||||||||||
static int SetPortAndTargetToBeSame(IResourceBuilder<MongoDBServerResource> builder) | ||||||||||||||
{ | ||||||||||||||
foreach (var endpoint in builder.Resource.Annotations.OfType<EndpointAnnotation>()) | ||||||||||||||
{ | ||||||||||||||
if (endpoint.Name == MongoDBServerResource.PrimaryEndpointName) | ||||||||||||||
{ | ||||||||||||||
if (endpoint.Port is { } port) | ||||||||||||||
{ | ||||||||||||||
endpoint.TargetPort = port; | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
if (endpoint.TargetPort is not { } targetPort) | ||||||||||||||
{ | ||||||||||||||
throw new InvalidOperationException("Target port is not set."); | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
// In the case of replica sets, the port and target port should be the same and is not proxied | ||||||||||||||
endpoint.IsProxied = false; | ||||||||||||||
|
||||||||||||||
return targetPort; | ||||||||||||||
} | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
throw new InvalidOperationException("No endpoint found for the MongoDB server resource."); | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
// See the conversation about setting up replica sets in Docker here: https://github.com/docker-library/mongo/issues/246 | ||||||||||||||
static string GetReplicaSetInitDockerfileDir(string replicaSet, string host, int port) | ||||||||||||||
{ | ||||||||||||||
var dir = Directory.CreateTempSubdirectory("aspire.mongo").FullName; | ||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A couple notes here:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is there a pattern that already exists to handle creating/cleaning up directories? Maybe another reason as @davidfowl mentioned above to maybe wait to exec into a container There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So after some thought, I think a pattern we can use here is:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With #7136, we have now introduced a new "IAspireStore" API: aspire/src/Aspire.Hosting/ApplicationModel/IAspireStore.cs Lines 17 to 22 in acdf501
This file could be written there and my above concerns would be resolved. |
||||||||||||||
|
||||||||||||||
var rsInitContents = $$"""rs.initiate({ _id:'{{replicaSet}}', members:[{_id:0,host:'localhost:{{port}}'}]})"""; | ||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @davidfowl I'd prefer to be able to just docker exec into the container after it started, but couldn't figure out how to do that so I'm running a container and configuring things to ensure it completes before anything else needs to use the db. If there is a better way to do this with aspire things, let me know There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We're adding support for this in Aspire 9 very soon. Might be better to wait until that support is available in the app model. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. cool - is there a tracking issue for that? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @davidfowl any progress on being able to exec into the container? |
||||||||||||||
var init = Path.Combine(dir, "rs.js"); | ||||||||||||||
File.WriteAllText(init, rsInitContents); | ||||||||||||||
|
||||||||||||||
var dockerfile = Path.Combine(dir, "Dockerfile"); | ||||||||||||||
File.WriteAllText(dockerfile, $""" | ||||||||||||||
FROM {MongoDBContainerImageTags.Image}:{MongoDBContainerImageTags.Tag} | ||||||||||||||
WORKDIR /rsinit | ||||||||||||||
ADD rs.js rs.js | ||||||||||||||
ENTRYPOINT ["mongosh", "--port", "{port}", "--host", "{host}", "rs.js"] | ||||||||||||||
"""); | ||||||||||||||
return dir; | ||||||||||||||
} | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
private static void ConfigureMongoExpressContainer(EnvironmentCallbackContext context, MongoDBServerResource resource) | ||||||||||||||
{ | ||||||||||||||
// Mongo Exporess assumes Mongo is being accessed over a default Aspire container network and hardcodes the resource address | ||||||||||||||
var sb = new StringBuilder($"mongodb://{resource.Name}:{resource.PrimaryEndpoint.TargetPort}/?directConnection=true"); | ||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ReferenceExpressionBuilder |
||||||||||||||
|
||||||||||||||
if (resource.TryGetLastAnnotation<MongoDbReplicaSetAnnotation>(out var replica)) | ||||||||||||||
{ | ||||||||||||||
sb.Append('&'); | ||||||||||||||
sb.Append(MongoDbReplicaSetAnnotation.QueryName); | ||||||||||||||
sb.Append('='); | ||||||||||||||
sb.Append(replica.ReplicaSetName); | ||||||||||||||
} | ||||||||||||||
|
||||||||||||||
// Mongo Express assumes Mongo is being accessed over a default Aspire container network and hardcodes the resource address | ||||||||||||||
// This will need to be refactored once updated service discovery APIs are available | ||||||||||||||
context.EnvironmentVariables.Add("ME_CONFIG_MONGODB_URL", $"mongodb://{resource.Name}:{resource.PrimaryEndpoint.TargetPort}/?directConnection=true"); | ||||||||||||||
context.EnvironmentVariables.Add("ME_CONFIG_MONGODB_URL", sb.ToString()); | ||||||||||||||
context.EnvironmentVariables.Add("ME_CONFIG_BASICAUTH", "false"); | ||||||||||||||
} | ||||||||||||||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
// Licensed to the .NET Foundation under one or more agreements. | ||
// The .NET Foundation licenses this file to you under the MIT license. | ||
|
||
namespace Aspire.Hosting.ApplicationModel; | ||
|
||
internal sealed record MongoDbReplicaSetAnnotation(string ReplicaSetName, IResourceBuilder<ContainerResource> InitContainer) : IResourceAnnotation | ||
{ | ||
internal const string QueryName = "replicaSet"; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,2 @@ | ||
#nullable enable | ||
|
||
static Aspire.Hosting.MongoDBBuilderExtensions.WithReplicaSet(this Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.MongoDBServerResource!>! builder, string? replicaSetName = null) -> Aspire.Hosting.ApplicationModel.IResourceBuilder<Aspire.Hosting.ApplicationModel.MongoDBServerResource!>! |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -596,6 +596,37 @@ public static IResourceBuilder<T> ExcludeFromManifest<T>(this IResourceBuilder<T | |
/// </code> | ||
/// </example> | ||
public static IResourceBuilder<T> WaitFor<T>(this IResourceBuilder<T> builder, IResourceBuilder<IResource> dependency) where T : IResource | ||
=> builder.WaitFor(dependency, includeHealthChecks: true); | ||
|
||
/// <summary> | ||
/// Waits for the dependency resource to enter the Running state before starting the resource. | ||
/// </summary> | ||
/// <typeparam name="T">The type of the resource.</typeparam> | ||
/// <param name="builder">The resource builder for the resource that will be waiting.</param> | ||
/// <param name="dependency">The resource builder for the dependency resource.</param> | ||
/// <param name="includeHealthChecks">Optionally includes checking the health checks.</param> | ||
/// <returns>The resource builder.</returns> | ||
/// <remarks> | ||
/// <para>This method is useful when a resource should wait until another has started running. This can help | ||
/// reduce errors in logs during local development where dependency resources.</para> | ||
/// <para>Some resources automatically register health checks with the application host container. For these | ||
/// resources, calling <see cref="WaitFor{T}(IResourceBuilder{T}, IResourceBuilder{IResource})"/> also results | ||
/// in the resource being blocked from starting until the health checks associated with the dependency resource | ||
/// return <see cref="HealthStatus.Healthy"/>.</para> | ||
/// <para>The <see cref="WithHealthCheck{T}(IResourceBuilder{T}, string)"/> method can be used to associate | ||
/// additional health checks with a resource.</para> | ||
/// </remarks> | ||
/// <example> | ||
/// Start message queue before starting the worker service. | ||
/// <code lang="C#"> | ||
/// var builder = DistributedApplication.CreateBuilder(args); | ||
/// var messaging = builder.AddRabbitMQ("messaging"); | ||
/// builder.AddProject<Projects.MyApp>("myapp") | ||
/// .WithReference(messaging) | ||
/// .WaitFor(messaging); | ||
/// </code> | ||
/// </example> | ||
public static IResourceBuilder<T> WaitFor<T>(this IResourceBuilder<T> builder, IResourceBuilder<IResource> dependency, bool includeHealthChecks) where T : IResource | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if |
||
{ | ||
builder.ApplicationBuilder.Eventing.Subscribe<BeforeResourceStartedEvent>(builder.Resource, async (e, ct) => | ||
{ | ||
|
@@ -632,7 +663,7 @@ public static IResourceBuilder<T> WaitFor<T>(this IResourceBuilder<T> builder, I | |
|
||
// If our dependency resource has health check annotations we want to wait until they turn healthy | ||
// otherwise we don't care about their health status. | ||
if (dependency.Resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var _)) | ||
if (includeHealthChecks && dependency.Resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var _)) | ||
twsouthwick marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
resourceLogger.LogInformation("Waiting for resource '{Name}' to become healthy.", dependency.Resource.Name); | ||
await rns.WaitForResourceAsync(dependency.Resource.Name, re => re.Snapshot.HealthStatus == HealthStatus.Healthy, cancellationToken: ct).ConfigureAwait(false); | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm not sure about using
WaitFor
invisibly like this. I can see the benefit but if one of the mongo db instances files to start then you won't be able to inspect any of the other databases.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I added that because once you have a replica set, the existing connection string may cause express to fail if it tries to connect before the replica set is initialized (a race condition) and it doesn't recover from it.