feat: add support for otel collector environment variables

CHANGELOG.md

@@ -15,6 +15,7 @@ Main (unreleased)
 - Add `otelcol.receiver.awscloudwatch` component to receive logs from AWS CloudWatch and forward them to other `otelcol.*` components. (@wildum)
 - Add `loki.enrich` component to enrich logs using labels from `discovery.*` components. (@v-zhuravlev)
 - Add string concatenation for secrets type (@ravishankar15)
+- Add support for environment variables to OpenTelemetry Collector config. (@jharvey10)
 
 ### Enhancements
 

docs/sources/set-up/migrate/from-otelcol.md

@@ -214,6 +214,7 @@ The following list is specific to the convert command and not {{< param "PRODUCT
   Make sure that you use the new metric names, for example, in your alerts and dashboards queries.
 * The logs produced by {{< param "PRODUCT_NAME" >}} differ from those produced by OpenTelemetry Collector.
 * {{< param "FULL_PRODUCT_NAME" >}} exposes the {{< param "PRODUCT_NAME" >}} [UI][].
+* Environment variables with a scheme other than `env` are not supported. Environment variables with no scheme will default to `env`.
 
 [OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/configuration/
 [debugging]: #debugging

internal/converter/internal/otelcolconvert/envprovider/env_provider.go

@@ -0,0 +1,39 @@
+package envprovider
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go.opentelemetry.io/collector/confmap"
+)
+
+const SchemeName = "env"
+
+// provider is a custom environment variable provider
+type provider struct{}
+
+// NewFactory returns a custom environment provider factory
+func NewFactory() confmap.ProviderFactory {
+	return confmap.NewProviderFactory(newProvider)
+}
+
+func newProvider(confmap.ProviderSettings) confmap.Provider {
+	return &provider{}
+}
+
+func (s *provider) Retrieve(_ context.Context, val string, _ confmap.WatcherFunc) (*confmap.Retrieved, error) {
+	if !strings.HasPrefix(val, s.Scheme()+":") {
+		return nil, fmt.Errorf("%q environment variable scheme is not supported by %q provider", val, s.Scheme())
+	}
+
+	return confmap.NewRetrieved(fmt.Sprintf("$${%s}", val))
+}
+
+func (*provider) Scheme() string {
+	return SchemeName
+}
+
+func (s *provider) Shutdown(context.Context) error {
+	return nil
+}

internal/converter/internal/otelcolconvert/otelcolconvert.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/grafana/alloy/internal/converter/diag"
 	"github.com/grafana/alloy/internal/converter/internal/common"
+	"github.com/grafana/alloy/internal/converter/internal/otelcolconvert/envprovider"
 	"github.com/grafana/alloy/syntax/token/builder"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componentstatus"
@@ -40,6 +42,13 @@ import (
 //   	    addConverter(converterCOMPONENT{})
 //      }
 
+// envvarRegexp matches envvar-like strings in the form of ${env:ENV_NAME} or ${env:ENV_NAME:-DEFAULT_VALUE}.
+//
+// See: https://opentelemetry.io/docs/specs/otel/configuration/data-model/#environment-variable-substitution
+var envvarRegexp *regexp.Regexp = regexp.MustCompile(
+	`"\$\{(?:env:)?(?<ENV_NAME>[a-zA-Z_][a-zA-Z0-9_]*)(:-(?<DEFAULT_VALUE>[^\n]*))?\}"`,
+)
+
 // Convert implements an Opentelemetry Collector config converter.
 //
 // For compatibility with other converters, the extraArgs paramater is defined
@@ -78,16 +87,32 @@ func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) {
 		return nil, diags
 	}
 
-	prettyByte, newDiags := common.PrettyPrint(buf.Bytes())
+	converted := convertEnvvars(buf.String())
+
+	prettyByte, newDiags := common.PrettyPrint([]byte(converted))
 	diags.AddAll(newDiags)
 	return prettyByte, diags
 }
 
+// convertEnvvars converts envvar-like strings into alloy sys.env() calls.
+func convertEnvvars(str string) string {
+	// TODO: we can identify certain types of odd configs WRT envvars. Warnings should be emitted to
+	// the console to convey them.
+	return envvarRegexp.ReplaceAllString(
+		str,
+		`coalesce(sys.env("$ENV_NAME"), "$DEFAULT_VALUE")`,
+	)
+}
+
 func readOpentelemetryConfig(in []byte) (*otelcol.Config, error) {
 	configProvider, err := otelcol.NewConfigProvider(otelcol.ConfigProviderSettings{
 		ResolverSettings: confmap.ResolverSettings{
-			URIs:              []string{"yaml:" + string(in)},
-			ProviderFactories: []confmap.ProviderFactory{yamlprovider.NewFactory()},
+			URIs: []string{"yaml:" + string(in)},
+			ProviderFactories: []confmap.ProviderFactory{
+				yamlprovider.NewFactory(),
+				envprovider.NewFactory(),
+			},
+			DefaultScheme: envprovider.SchemeName,
 		},
 	})
 	if err != nil {

internal/converter/internal/otelcolconvert/otelcolconvert_test.go

@@ -20,3 +20,7 @@ func TestConvert(t *testing.T) {
 func TestConvertErrors(t *testing.T) {
 	test_common.TestDirectory(t, "testdata/otelcol_errors", ".yaml", true, []string{}, otelcolconvert.Convert)
 }
+
+func TestConvertEnvvars(t *testing.T) {
+	test_common.TestDirectory(t, "testdata/envvars", ".yaml", true, []string{}, otelcolconvert.Convert)
+}

internal/converter/internal/otelcolconvert/testdata/envvars/default_value_format.alloy

@@ -0,0 +1,19 @@
+otelcol.receiver.otlp "default" {
+	grpc {
+		endpoint = coalesce(sys.env("NORMAL_FORMAT"), "default value")
+	}
+
+	http {
+		endpoint = "localhost:4318"
+	}
+
+	output {
+		metrics = [otelcol.exporter.otlp.default.input]
+	}
+}
+
+otelcol.exporter.otlp "default" {
+	client {
+		endpoint = "database:4317"
+	}
+}

internal/converter/internal/otelcolconvert/testdata/envvars/default_value_format.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${env:NORMAL_FORMAT:-default value}
+      http:
+
+exporters:
+  otlp:
+    endpoint: database:4317
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlp]

internal/converter/internal/otelcolconvert/testdata/envvars/escaped_format.alloy

@@ -0,0 +1,19 @@
+otelcol.receiver.otlp "default" {
+	grpc {
+		endpoint = coalesce(sys.env("ESCAPED_FORMAT"), "")
+	}
+
+	http {
+		endpoint = "localhost:4318"
+	}
+
+	output {
+		metrics = [otelcol.exporter.otlp.default.input]
+	}
+}
+
+otelcol.exporter.otlp "default" {
+	client {
+		endpoint = "database:4317"
+	}
+}

internal/converter/internal/otelcolconvert/testdata/envvars/escaped_format.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${env:ESCAPED_FORMAT}
+      http:
+
+exporters:
+  otlp:
+    endpoint: database:4317
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlp]

internal/converter/internal/otelcolconvert/testdata/envvars/no_scheme.alloy

@@ -0,0 +1,19 @@
+otelcol.receiver.otlp "default" {
+	grpc {
+		endpoint = coalesce(sys.env("NO_SCHEME"), "")
+	}
+
+	http {
+		endpoint = "localhost:4318"
+	}
+
+	output {
+		metrics = [otelcol.exporter.otlp.default.input]
+	}
+}
+
+otelcol.exporter.otlp "default" {
+	client {
+		endpoint = "database:4317"
+	}
+}

internal/converter/internal/otelcolconvert/testdata/envvars/no_scheme.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${NO_SCHEME}
+      http:
+
+exporters:
+  otlp:
+    endpoint: database:4317
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlp]

internal/converter/internal/otelcolconvert/testdata/envvars/normal_format.alloy

@@ -0,0 +1,19 @@
+otelcol.receiver.otlp "default" {
+	grpc {
+		endpoint = coalesce(sys.env("NORMAL_FORMAT"), "")
+	}
+
+	http {
+		endpoint = "localhost:4318"
+	}
+
+	output {
+		metrics = [otelcol.exporter.otlp.default.input]
+	}
+}
+
+otelcol.exporter.otlp "default" {
+	client {
+		endpoint = "database:4317"
+	}
+}

internal/converter/internal/otelcolconvert/testdata/envvars/normal_format.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${env:NORMAL_FORMAT}
+      http:
+
+exporters:
+  otlp:
+    endpoint: database:4317
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlp]

internal/converter/internal/otelcolconvert/testdata/envvars/unknown_scheme.diags

@@ -0,0 +1 @@
+(Critical) failed to get otelcol config: cannot resolve the configuration: scheme "wow" is not supported for uri "wow:NO_SCHEME"

internal/converter/internal/otelcolconvert/testdata/envvars/unknown_scheme.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${wow:NO_SCHEME}
+      http:
+
+exporters:
+  otlp:
+    endpoint: database:4317
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: []
+      exporters: [otlp]