From 2e80f03300b6394bf9d3eac8f43b90ce4a5779e7 Mon Sep 17 00:00:00 2001 From: Jenny Shu <28537278+jenshu@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:13:04 -0500 Subject: [PATCH] Admin server - part 1 (#10460) --- docs/content/static/content/osa_provided.md | 1 - go.mod | 2 +- pkg/utils/glooadminutils/admincli/client.go | 2 +- .../admin/admin_suite_test.go} | 6 +- .../iosnapshot => gateway2/admin}/redact.go | 21 +- .../admin}/resources.go | 20 +- .../iosnapshot => gateway2/admin}/response.go | 18 +- .../admin}/response_test.go | 11 +- .../pkg/servers => gateway2}/admin/server.go | 97 +- projects/gateway2/setup/ggv2setup.go | 4 + projects/gateway2/setup/ggv2setup_test.go | 2 - .../gloo/pkg/servers/iosnapshot/history.go | 332 ------- .../pkg/servers/iosnapshot/history_test.go | 857 ------------------ .../pkg/syncer/envoy_translator_syncer.go | 59 -- projects/gloo/pkg/syncer/setup/extensions.go | 8 - .../gloo/pkg/syncer/setup/extensions_test.go | 10 +- .../gloo/pkg/syncer/setup/setup_syncer.go | 56 +- projects/gloo/pkg/syncer/setup/start_func.go | 85 -- projects/gloo/pkg/syncer/translator_syncer.go | 19 +- .../gloo/pkg/syncer/translator_syncer_test.go | 19 +- test/helpers/kube_dump.go | 2 +- test/kubernetes/testutils/assertions/gloo.go | 2 +- 22 files changed, 112 insertions(+), 1521 deletions(-) rename projects/{gloo/pkg/servers/iosnapshot/iosnapshot_suite_test.go => gateway2/admin/admin_suite_test.go} (56%) rename projects/{gloo/pkg/servers/iosnapshot => gateway2/admin}/redact.go (61%) rename projects/{gloo/pkg/servers/iosnapshot => gateway2/admin}/resources.go (84%) rename projects/{gloo/pkg/servers/iosnapshot => gateway2/admin}/response.go (79%) rename projects/{gloo/pkg/servers/iosnapshot => gateway2/admin}/response_test.go (85%) rename projects/{gloo/pkg/servers => gateway2}/admin/server.go (55%) delete mode 100644 projects/gloo/pkg/servers/iosnapshot/history.go delete mode 100644 projects/gloo/pkg/servers/iosnapshot/history_test.go delete mode 100644 projects/gloo/pkg/syncer/setup/start_func.go diff --git a/docs/content/static/content/osa_provided.md b/docs/content/static/content/osa_provided.md index 43d64ec40aa..51cfe011d5c 100644 --- a/docs/content/static/content/osa_provided.md +++ b/docs/content/static/content/osa_provided.md @@ -28,7 +28,6 @@ Name|Version|License [google/go-github](https://github.com/google/go-github)|v17.0.0+incompatible|BSD 3-clause "New" or "Revised" License [go-github/v32](https://github.com/google/go-github)|v32.0.0|BSD 3-clause "New" or "Revised" License [google/uuid](https://github.com/google/uuid)|v1.6.0|BSD 3-clause "New" or "Revised" License -[gorilla/mux](https://github.com/gorilla/mux)|v1.8.1|BSD 3-clause "New" or "Revised" License [grpc-ecosystem/go-grpc-middleware](https://github.com/grpc-ecosystem/go-grpc-middleware)|v1.4.0|Apache License 2.0 [hinshun/vt10x](https://github.com/hinshun/vt10x)|v0.0.0-20180809195222-d55458df857c|MIT License [imdario/mergo](https://github.com/imdario/mergo)|v0.3.16|BSD 3-clause "New" or "Revised" License diff --git a/go.mod b/go.mod index 08792023dfb..f4f9907d23e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v32 v32.0.0 - github.com/gorilla/mux v1.8.1 + github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/hashicorp/consul/api v1.28.2 github.com/hashicorp/go-multierror v1.1.1 diff --git a/pkg/utils/glooadminutils/admincli/client.go b/pkg/utils/glooadminutils/admincli/client.go index 7eee99091fa..59b56b4b371 100644 --- a/pkg/utils/glooadminutils/admincli/client.go +++ b/pkg/utils/glooadminutils/admincli/client.go @@ -8,7 +8,7 @@ import ( "github.com/rotisserie/eris" "github.com/solo-io/gloo/pkg/utils/cmdutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" + "github.com/solo-io/gloo/projects/gateway2/admin" "github.com/solo-io/go-utils/threadsafe" ) diff --git a/projects/gloo/pkg/servers/iosnapshot/iosnapshot_suite_test.go b/projects/gateway2/admin/admin_suite_test.go similarity index 56% rename from projects/gloo/pkg/servers/iosnapshot/iosnapshot_suite_test.go rename to projects/gateway2/admin/admin_suite_test.go index 0551f8aaaf7..0b7b9b958fb 100644 --- a/projects/gloo/pkg/servers/iosnapshot/iosnapshot_suite_test.go +++ b/projects/gateway2/admin/admin_suite_test.go @@ -1,4 +1,4 @@ -package iosnapshot +package admin import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestIoSnapshot(t *testing.T) { +func TestAdmin(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "IOSnapshot Suite") + RunSpecs(t, "Admin Suite") } diff --git a/projects/gloo/pkg/servers/iosnapshot/redact.go b/projects/gateway2/admin/redact.go similarity index 61% rename from projects/gloo/pkg/servers/iosnapshot/redact.go rename to projects/gateway2/admin/redact.go index 632d29f4ee6..4e07fe5cc6c 100644 --- a/projects/gloo/pkg/servers/iosnapshot/redact.go +++ b/projects/gateway2/admin/redact.go @@ -1,7 +1,6 @@ -package iosnapshot +package admin import ( - gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -34,24 +33,6 @@ func redactKubeSecretData(obj client.Object) { redactAnnotations(secret.GetAnnotations()) } -// redactGlooSecretData modifies the secret to remove any sensitive information -// The structure of a Secret in Gloo Gateway does not lend itself to easily redact data in different places. -// As a result, we perform a primitive redaction method, where we maintain the metadata, and remove the entire spec -func redactGlooSecretData(element *gloov1.Secret) { - element.Kind = nil - - redactAnnotations(element.GetMetadata().GetAnnotations()) -} - -// redactGlooArtifactData modifies the artifact to remove any sensitive information -func redactGlooArtifactData(element *gloov1.Artifact) { - for k := range element.GetData() { - element.GetData()[k] = redactedString - } - - redactAnnotations(element.GetMetadata().GetAnnotations()) -} - // redactGlooResourceMetadata modifies the metadata to remove any sensitive information // ref: https://github.com/solo-io/skv2/blob/1583cb716c04eb3f8d01ecb179b0deeabaa6e42b/contrib/pkg/snapshot/redact.go#L20-L26 func redactAnnotations(annotations map[string]string) { diff --git a/projects/gloo/pkg/servers/iosnapshot/resources.go b/projects/gateway2/admin/resources.go similarity index 84% rename from projects/gloo/pkg/servers/iosnapshot/resources.go rename to projects/gateway2/admin/resources.go index 65062cf444b..0fd62fd4f97 100644 --- a/projects/gloo/pkg/servers/iosnapshot/resources.go +++ b/projects/gateway2/admin/resources.go @@ -1,4 +1,4 @@ -package iosnapshot +package admin import ( "slices" @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +// TODO: these need to be updated var ( KubernetesCoreGVKs = []schema.GroupVersionKind{ wellknownkube.SecretGVK, @@ -33,14 +34,6 @@ var ( gatewayv1.RouteOptionGVK, } - EdgeGatewayGVKs = []schema.GroupVersionKind{ - gatewayv1.GatewayGVK, - gatewayv1.MatchableHttpGatewayGVK, - gatewayv1.MatchableTcpGatewayGVK, - gatewayv1.VirtualServiceGVK, - gatewayv1.RouteTableGVK, - } - KubernetesGatewayGVKs = []schema.GroupVersionKind{ wellknown.GatewayClassGVK, wellknown.GatewayGVK, @@ -56,16 +49,11 @@ var ( gatewayv1.HttpListenerOptionGVK, } - EdgeOnlyInputSnapshotGVKs = slices.Concat( + // CompleteInputSnapshotGVKs is the list of GVKs that will be returned by the InputSnapshot API + CompleteInputSnapshotGVKs = slices.Concat( KubernetesCoreGVKs, GlooGVKs, PolicyGVKs, - EdgeGatewayGVKs, - ) - - // CompleteInputSnapshotGVKs is the list of GVKs that will be returned by the InputSnapshot API - CompleteInputSnapshotGVKs = slices.Concat( - EdgeOnlyInputSnapshotGVKs, KubernetesGatewayGVKs, KubernetesGatewayIntegrationPolicyGVKs, ) diff --git a/projects/gloo/pkg/servers/iosnapshot/response.go b/projects/gateway2/admin/response.go similarity index 79% rename from projects/gloo/pkg/servers/iosnapshot/response.go rename to projects/gateway2/admin/response.go index 6e789bc5f70..a7c26108c17 100644 --- a/projects/gloo/pkg/servers/iosnapshot/response.go +++ b/projects/gateway2/admin/response.go @@ -1,8 +1,12 @@ -package iosnapshot +package admin import ( + "cmp" "encoding/json" "fmt" + "slices" + + "sigs.k8s.io/controller-runtime/pkg/client" ) // SnapshotResponseData is the data that is returned by Getter methods on the History object @@ -76,6 +80,18 @@ func formatOutput(format OutputFormat, genericOutput interface{}) ([]byte, error } } +// sortResources sorts resources by gvk, namespace, and name +func sortResources(resources []client.Object) { + slices.SortStableFunc(resources, func(a, b client.Object) int { + return cmp.Or( + cmp.Compare(a.GetObjectKind().GroupVersionKind().Version, b.GetObjectKind().GroupVersionKind().Version), + cmp.Compare(a.GetObjectKind().GroupVersionKind().Kind, b.GetObjectKind().GroupVersionKind().Kind), + cmp.Compare(a.GetNamespace(), b.GetNamespace()), + cmp.Compare(a.GetName(), b.GetName()), + ) + }) +} + func completeSnapshotResponse(data interface{}) SnapshotResponseData { return SnapshotResponseData{ Data: data, diff --git a/projects/gloo/pkg/servers/iosnapshot/response_test.go b/projects/gateway2/admin/response_test.go similarity index 85% rename from projects/gloo/pkg/servers/iosnapshot/response_test.go rename to projects/gateway2/admin/response_test.go index 7a8b41a66b6..6253194389a 100644 --- a/projects/gloo/pkg/servers/iosnapshot/response_test.go +++ b/projects/gateway2/admin/response_test.go @@ -1,9 +1,10 @@ -package iosnapshot +package admin_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rotisserie/eris" + "github.com/solo-io/gloo/projects/gateway2/admin" crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -11,24 +12,24 @@ import ( var _ = Describe("SnapshotResponseData", func() { DescribeTable("MarshalJSONString", - func(response SnapshotResponseData, expectedString string) { + func(response admin.SnapshotResponseData, expectedString string) { responseStr := response.MarshalJSONString() Expect(responseStr).To(Equal(expectedString)) }, Entry("successful response can be formatted as json", - SnapshotResponseData{ + admin.SnapshotResponseData{ Data: "my data", Error: nil, }, "{\"data\":\"my data\",\"error\":\"\"}"), Entry("errored response can be formatted as json", - SnapshotResponseData{ + admin.SnapshotResponseData{ Data: "", Error: eris.New("one error"), }, "{\"data\":\"\",\"error\":\"one error\"}"), Entry("CR list can be formatted as json", - SnapshotResponseData{ + admin.SnapshotResponseData{ Data: []crdv1.Resource{ { ObjectMeta: metav1.ObjectMeta{ diff --git a/projects/gloo/pkg/servers/admin/server.go b/projects/gateway2/admin/server.go similarity index 55% rename from projects/gloo/pkg/servers/admin/server.go rename to projects/gateway2/admin/server.go index d298d9ebd0d..63591385cd2 100644 --- a/projects/gloo/pkg/servers/admin/server.go +++ b/projects/gateway2/admin/server.go @@ -8,8 +8,12 @@ import ( "net/http" "sort" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" + "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + "github.com/rotisserie/eris" + "github.com/solo-io/gloo/projects/gateway2/controller" "github.com/solo-io/go-utils/contextutils" + "github.com/solo-io/go-utils/stats" "istio.io/istio/pkg/kube/krt" ) @@ -17,41 +21,47 @@ const ( AdminPort = 9095 ) -// ServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux +func RunAdminServer(ctx context.Context, setupOpts *controller.SetupOpts) error { + // serverHandlers defines the custom handlers that the Admin Server will support + serverHandlers := getServerHandlers(ctx, setupOpts.KrtDebugger, setupOpts.Cache) + + stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) { + // let people know these moved + profiles[fmt.Sprintf("http://localhost:%d/snapshots/", AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", AdminPort) + }) + startHandlers(ctx, serverHandlers) + + return nil +} + +// getServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux // These endpoints serve as the basis for an Admin Interface for the Control Plane (https://github.com/solo-io/gloo/issues/6494) -func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.DebugHandler) func(mux *http.ServeMux, profiles map[string]string) { +func getServerHandlers(ctx context.Context, dbg *krt.DebugHandler, cache envoycache.SnapshotCache) func(mux *http.ServeMux, profiles map[string]string) { return func(m *http.ServeMux, profiles map[string]string) { - // The Input Snapshot is intended to return a list of resources that are persisted in the Kubernetes DB, etcD - m.HandleFunc("/snapshots/input", func(w http.ResponseWriter, request *http.Request) { - response := history.GetInputSnapshot(ctx) - respondJson(w, response) - }) - profiles["/snapshots/input"] = "Input Snapshot" - - // The Edge Snapshot is intended to return a representation of the ApiSnapshot object that the Control Plane - // manages internally. This is not intended to be consumed by users, but instead be a mechanism to feed this - // data into future unit tests - m.HandleFunc("/snapshots/edge", func(w http.ResponseWriter, request *http.Request) { - response := history.GetEdgeApiSnapshot(ctx) - respondJson(w, response) - }) - profiles["/snapshots/edge"] = "Edge Snapshot" - - // The Proxy Snapshot is intended to return a representation of the Proxies within the ApiSnapshot object. - // Proxies may either be persisted in etcD or in-memory, so this Api provides a single mechansim to access - // these resources. - m.HandleFunc("/snapshots/proxies", func(w http.ResponseWriter, r *http.Request) { - response := history.GetProxySnapshot(ctx) - respondJson(w, response) - }) - profiles["/snapshots/proxies"] = "Proxy Snapshot" + /* + // The Input Snapshot is intended to return a list of resources that are persisted in the Kubernetes DB, etcD + m.HandleFunc("/snapshots/input", func(w http.ResponseWriter, request *http.Request) { + response := history.GetInputSnapshot(ctx) + respondJson(w, response) + }) + profiles["/snapshots/input"] = "Input Snapshot" + + // The Proxy Snapshot is intended to return a representation of the Proxies within the ApiSnapshot object. + // Proxies may either be persisted in etcD or in-memory, so this Api provides a single mechansim to access + // these resources. + m.HandleFunc("/snapshots/proxies", func(w http.ResponseWriter, r *http.Request) { + response := history.GetProxySnapshot(ctx) + respondJson(w, response) + }) + profiles["/snapshots/proxies"] = "Proxy Snapshot" + */ // The xDS Snapshot is intended to return the full in-memory xDS cache that the Control Plane manages // and serves up to running proxies. m.HandleFunc("/snapshots/xds", func(w http.ResponseWriter, r *http.Request) { - response := history.GetXdsSnapshot(ctx) - respondJson(w, response) + response := getXdsSnapshotDataFromCache(cache) + writeJSON(w, response, r) }) profiles["/snapshots/xds"] = "XDS Snapshot" @@ -62,7 +72,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De } } -func respondJson(w http.ResponseWriter, response iosnapshot.SnapshotResponseData) { +func respondJson(w http.ResponseWriter, response SnapshotResponseData) { w.Header().Set("Content-Type", getContentType("json")) _, _ = fmt.Fprintf(w, "%+v", response.MarshalJSONString()) @@ -106,7 +116,7 @@ func writeJSON(w http.ResponseWriter, obj any, req *http.Request) { } } -func StartHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error { +func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error { mux := new(http.ServeMux) profileDescriptions := map[string]string{} for _, addHandler := range addHandlers { @@ -171,3 +181,28 @@ func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r w.Write(buf.Bytes()) } } + +func getXdsSnapshotDataFromCache(xdsCache cache.SnapshotCache) SnapshotResponseData { + cacheKeys := xdsCache.GetStatusKeys() + cacheEntries := make(map[string]interface{}, len(cacheKeys)) + + for _, k := range cacheKeys { + xdsSnapshot, err := getXdsSnapshot(xdsCache, k) + if err != nil { + cacheEntries[k] = err.Error() + } else { + cacheEntries[k] = xdsSnapshot + } + } + + return completeSnapshotResponse(cacheEntries) +} + +func getXdsSnapshot(xdsCache cache.SnapshotCache, k string) (cache cache.ResourceSnapshot, err error) { + defer func() { + if r := recover(); r != nil { + err = eris.New(fmt.Sprintf("panic occurred while getting xds snapshot: %v", r)) + } + }() + return xdsCache.GetSnapshot(k) +} diff --git a/projects/gateway2/setup/ggv2setup.go b/projects/gateway2/setup/ggv2setup.go index 810d64fdd92..b8a2f3f4c83 100644 --- a/projects/gateway2/setup/ggv2setup.go +++ b/projects/gateway2/setup/ggv2setup.go @@ -12,6 +12,7 @@ import ( "github.com/solo-io/gloo/pkg/utils/kubeutils" "github.com/solo-io/gloo/pkg/utils/namespaces" "github.com/solo-io/gloo/pkg/utils/setuputils" + "github.com/solo-io/gloo/projects/gateway2/admin" "github.com/solo-io/gloo/projects/gateway2/controller" extensionsplug "github.com/solo-io/gloo/projects/gateway2/extensions2/plugin" "github.com/solo-io/gloo/projects/gateway2/krtcollections" @@ -171,6 +172,9 @@ func StartGGv2WithConfig(ctx context.Context, setupOpts *controller.SetupOpts, kubeClient.RunAndWait(ctx.Done()) setting.Synced().WaitUntilSynced(ctx.Done()) + logger.Info("starting admin server") + go admin.RunAdminServer(ctx, setupOpts) + logger.Info("starting controller") return c.Start(ctx) } diff --git a/projects/gateway2/setup/ggv2setup_test.go b/projects/gateway2/setup/ggv2setup_test.go index 6260538f1f1..b762a4b2524 100644 --- a/projects/gateway2/setup/ggv2setup_test.go +++ b/projects/gateway2/setup/ggv2setup_test.go @@ -288,10 +288,8 @@ func testScenario(t *testing.T, ctx context.Context, kdbg *krt.DebugHandler, dump := dumper.Dump(t, ctx) if len(dump.Listeners) == 0 { - // xdsDump := iosnapshot.GetXdsSnapshotDataFromCache(snapCache).MarshalJSONString() j, _ := kdbg.MarshalJSON() t.Logf("timed out waiting - krt state for test: %s %s", t.Name(), string(j)) - // t.Logf("timed out waiting - xds state for test: %s %s", t.Name(), xdsDump) t.Fatalf("timed out waiting for listeners") } if write { diff --git a/projects/gloo/pkg/servers/iosnapshot/history.go b/projects/gloo/pkg/servers/iosnapshot/history.go deleted file mode 100644 index ae1009926b4..00000000000 --- a/projects/gloo/pkg/servers/iosnapshot/history.go +++ /dev/null @@ -1,332 +0,0 @@ -package iosnapshot - -import ( - "cmp" - "context" - "encoding/json" - "fmt" - "slices" - "sync" - - "github.com/rotisserie/eris" - "github.com/solo-io/gloo/pkg/schemes" - "github.com/solo-io/gloo/pkg/utils/kubeutils" - crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/hashicorp/go-multierror" - - gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" - v1snap "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/gloosnapshot" - "github.com/solo-io/solo-kit/pkg/api/v1/control-plane/cache" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// History represents an object that maintains state about the running system -// The ControlPlane will use the Setters to update the last known state, -// and the Getters will be used by the Admin Server -type History interface { - // SetApiSnapshot sets the latest Edge ApiSnapshot. - SetApiSnapshot(latestInput *v1snap.ApiSnapshot) - - // GetInputSnapshot returns all resources in the Edge input snapshot, and if Kubernetes - // Gateway integration is enabled, it additionally returns all resources on the cluster - // with types specified by `inputSnapshotGvks`. - GetInputSnapshot(ctx context.Context) SnapshotResponseData - - // GetEdgeApiSnapshot returns all resources in the Edge input snapshot - GetEdgeApiSnapshot(ctx context.Context) SnapshotResponseData - - // GetProxySnapshot returns the Proxies generated for all components. - GetProxySnapshot(ctx context.Context) SnapshotResponseData - - // GetXdsSnapshot returns the entire cache of xDS snapshots - // NOTE: This contains sensitive data, as it is the exact inputs that used by Envoy - GetXdsSnapshot(ctx context.Context) SnapshotResponseData -} - -// HistoryFactoryParameters are the inputs used to create a History object -type HistoryFactoryParameters struct { - Settings *gloov1.Settings - Cache cache.SnapshotCache -} - -// HistoryFactory is a function that produces a History object -type HistoryFactory func(params HistoryFactoryParameters) History - -// GetHistoryFactory returns a default HistoryFactory implementation -func GetHistoryFactory() HistoryFactory { - return func(params HistoryFactoryParameters) History { - var kubeClient client.Client - - cfg, err := kubeutils.GetRestConfigWithKubeContext("") - if err == nil { - cli, err := client.New(cfg, client.Options{ - Scheme: schemes.DefaultScheme(), - }) - if err == nil { - kubeClient = cli - } - } - - gvks := CompleteInputSnapshotGVKs - - return NewHistory(params.Cache, params.Settings, kubeClient, gvks) - } -} - -// NewHistory returns an implementation of the History interface -// - `cache` is the control plane's xDS snapshot cache -// - `settings` specifies the Settings for this control plane instance -// - `inputSnapshotGvks` specifies the list of resource types to return in the input snapshot when -// Kubernetes Gateway integration is enabled. For example, this may include Gateway API -// resources, Portal resources, or other resources specific to the Kubernetes Gateway integration. -// If not set, then only Edge ApiSnapshot resources will be returned from `GetInputSnapshot`. -func NewHistory(cache cache.SnapshotCache, settings *gloov1.Settings, kubeClient client.Client, kubeGatewayGvks []schema.GroupVersionKind) History { - return &historyImpl{ - latestApiSnapshot: nil, - xdsCache: cache, - settings: settings, - inputSnapshotClient: kubeClient, - inputSnapshotGvks: kubeGatewayGvks, - } -} - -type historyImpl struct { - // TODO: - // We rely on a mutex to prevent races reading/writing the data for this object - // We should instead use channels to coordinate this - sync.RWMutex - latestApiSnapshot *v1snap.ApiSnapshot - xdsCache cache.SnapshotCache - settings *gloov1.Settings - - // The InputSnapshot API is really a pass through to the Kubernetes API Server - // Below are properties that are used to configure the behavior of GetInputSnapshot - - inputSnapshotClient client.Client - inputSnapshotGvks []schema.GroupVersionKind -} - -// SetApiSnapshot sets the latest input ApiSnapshot -func (h *historyImpl) SetApiSnapshot(latestApiSnapshot *v1snap.ApiSnapshot) { - // Setters are called by the running Control Plane, so we perform the update in a goroutine to prevent - // any contention/issues, from impacting the runtime of the system - go func() { - h.Lock() - defer h.Unlock() - - h.latestApiSnapshot = latestApiSnapshot - }() -} - -func (h *historyImpl) GetEdgeApiSnapshot(_ context.Context) SnapshotResponseData { - snap := h.getRedactedApiSnapshot() - return completeSnapshotResponse(snap) -} - -// GetInputSnapshot gets the input snapshot for all components. -func (h *historyImpl) GetInputSnapshot(ctx context.Context) SnapshotResponseData { - if h.inputSnapshotClient == nil { - return errorSnapshotResponse(eris.New("No kubernetes Client found for InputSnapshot")) - } - - var objects []client.Object - var errs *multierror.Error - for _, gvk := range h.inputSnapshotGvks { - gvkResources, err := h.listObjectsForGvk(ctx, h.inputSnapshotClient, gvk) - if err != nil { - // We intentionally aggregate the errors so that we can return a "best effort" set of - // resources, and one error doesn't lead to the entire set of GVKs being short-circuited - errs = multierror.Append(errs, err) - } - objects = append(objects, gvkResources...) - } - sortResources(objects) - - return SnapshotResponseData{ - Data: objects, - Error: errs.ErrorOrNil(), - } -} - -func (h *historyImpl) GetProxySnapshot(_ context.Context) SnapshotResponseData { - snap := h.getRedactedApiSnapshot() - - var resources []crdv1.Resource - var errs *multierror.Error - - for _, proxy := range snap.Proxies { - kubeProxy, err := gloov1.ProxyCrd.KubeResource(proxy) - if err != nil { - // We intentionally aggregate the errors so that we can return a "best effort" set of - // resources, and one error doesn't lead to the entire set of GVKs being short-circuited - errs = multierror.Append(errs, err) - } - resources = append(resources, *kubeProxy) - } - - return SnapshotResponseData{ - Data: resources, - Error: errs.ErrorOrNil(), - } -} - -// GetXdsSnapshot returns the entire cache of xDS snapshots -// NOTE: This contains sensitive data, as it is the exact inputs that used by Envoy -func (h *historyImpl) GetXdsSnapshot(_ context.Context) SnapshotResponseData { - return GetXdsSnapshotDataFromCache(h.xdsCache) -} - -func GetXdsSnapshotDataFromCache(xdsCache cache.SnapshotCache) SnapshotResponseData { - cacheKeys := xdsCache.GetStatusKeys() - cacheEntries := make(map[string]interface{}, len(cacheKeys)) - - for _, k := range cacheKeys { - xdsSnapshot, err := getXdsSnapshot(xdsCache, k) - if err != nil { - cacheEntries[k] = err.Error() - } else { - cacheEntries[k] = xdsSnapshot - } - } - - return completeSnapshotResponse(cacheEntries) -} - -func getXdsSnapshot(xdsCache cache.SnapshotCache, k string) (cache cache.Snapshot, err error) { - defer func() { - if r := recover(); r != nil { - err = eris.New(fmt.Sprintf("panic occurred while getting xds snapshot: %v", r)) - } - }() - return xdsCache.GetSnapshot(k) -} - -// getRedactedApiSnapshot gets an in-memory copy of the ApiSnapshot -// Any sensitive data contained in the Snapshot will either be explicitly redacted -// or entirely excluded -// NOTE: Redaction is somewhat of an expensive operation, so we have a few options for how to approach it: -// -// 1. Perform it when a new ApiSnapshot is received from the Control Plane -// -// 2. Perform it on demand, when an ApiSnapshot is requested -// -// 3. Perform it on demand, when an ApiSnapshot is requested, but store a local cache for future requests. -// That cache would be invalidated each time a new ApiSnapshot is received. -// -// Given that the rate of requests for the ApiSnapshot <<< the frequency of updates of an ApiSnapshot by the Control Plane, -// in this first pass we opt to take approach #2. -func (h *historyImpl) getRedactedApiSnapshot() *v1snap.ApiSnapshot { - snap := h.getApiSnapshotSafe() - - redactApiSnapshot(snap) - return snap -} - -// getApiSnapshotSafe gets a clone of the latest ApiSnapshot -func (h *historyImpl) getApiSnapshotSafe() *v1snap.ApiSnapshot { - h.RLock() - defer h.RUnlock() - if h.latestApiSnapshot == nil { - return &v1snap.ApiSnapshot{} - } - - // This clone is critical!! - // We do this to ensure the following cases: - // 1. Modifications to this snapshot, by the admin server, DO NOT impact the Control Plane - // 2. Modifications to this snapshot by a single request, DO NOT interfere with other requests - clone := h.latestApiSnapshot.Clone() - return &clone -} - -func (h *historyImpl) listObjectsForGvk(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind) ([]client.Object, error) { - var objects []client.Object - - // populate an unstructured list for each resource type - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(gvk) - err := cli.List(ctx, list) - if err != nil { - return nil, err - } - - var errs *multierror.Error - - // convert each Unstructured to a client.Object - for _, uns := range list.Items { - realObj, err := cli.Scheme().New(gvk) - if err != nil { - errs = multierror.Append(errs, err) - continue - } - clientObj, ok := realObj.(client.Object) - if !ok { - errs = multierror.Append(errs, eris.New(fmt.Sprintf("%s could not be converted into client.Object", gvk))) - continue - } - - err = runtime.DefaultUnstructuredConverter.FromUnstructured(uns.Object, clientObj) - if err != nil { - errs = multierror.Append(errs, err) - continue - } - - redactClientObject(clientObj) - objects = append(objects, clientObj) - } - - return objects, errs.ErrorOrNil() -} - -// redactApiSnapshot accepts an ApiSnapshot, and mutates it to remove sensitive data. -// It is critical that data which is exposed by this component is redacted, -// so that customers can feel comfortable sharing the results with us. -// -// NOTE: This is an extremely naive implementation. It is intended as a first pass to get this API -// into the hands of the field.As we iterate on this component, we can use some of the redaction -// utilities in `/pkg/utils/syncutil`. -func redactApiSnapshot(snap *v1snap.ApiSnapshot) { - snap.Secrets.Each(func(element *gloov1.Secret) { - redactGlooSecretData(element) - }) - - // See `pkg/utils/syncutil/log_redactor.StringifySnapshot` for an explanation for - // why we redact Artifacts - snap.Artifacts.Each(func(element *gloov1.Artifact) { - redactGlooArtifactData(element) - }) -} - -// sortResources sorts resources by gvk, namespace, and name -func sortResources(resources []client.Object) { - slices.SortStableFunc(resources, func(a, b client.Object) int { - return cmp.Or( - cmp.Compare(a.GetObjectKind().GroupVersionKind().Version, b.GetObjectKind().GroupVersionKind().Version), - cmp.Compare(a.GetObjectKind().GroupVersionKind().Kind, b.GetObjectKind().GroupVersionKind().Kind), - cmp.Compare(a.GetNamespace(), b.GetNamespace()), - cmp.Compare(a.GetName(), b.GetName()), - ) - }) -} - -// apiSnapshotToGenericMap converts an ApiSnapshot into a generic map -// Since maps do not guarantee ordering, we do not attempt to sort these resources, as we do four []crdv1.Resource -func apiSnapshotToGenericMap(snap *v1snap.ApiSnapshot) (map[string]interface{}, error) { - genericMap := map[string]interface{}{} - - if snap == nil { - return genericMap, nil - } - - jsn, err := json.Marshal(snap) - if err != nil { - return nil, err - } - if err := json.Unmarshal(jsn, &genericMap); err != nil { - return nil, err - } - return genericMap, nil -} diff --git a/projects/gloo/pkg/servers/iosnapshot/history_test.go b/projects/gloo/pkg/servers/iosnapshot/history_test.go deleted file mode 100644 index f5c87189c08..00000000000 --- a/projects/gloo/pkg/servers/iosnapshot/history_test.go +++ /dev/null @@ -1,857 +0,0 @@ -package iosnapshot_test - -import ( - "context" - "fmt" - "time" - - gomegatypes "github.com/onsi/gomega/types" - "github.com/solo-io/gloo/pkg/schemes" - gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/apis/gloo.solo.io/v1" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" - apiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - wellknownkube "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/wellknown" - - corev1 "k8s.io/api/core/v1" - - skmatchers "github.com/solo-io/solo-kit/test/matchers" - - "github.com/onsi/gomega/gstruct" - "github.com/solo-io/gloo/test/gomega/matchers" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/types" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1" - gatewaykubev1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1/kube/apis/gateway.solo.io/v1" - "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1" - "github.com/solo-io/gloo/projects/gateway2/wellknown" - ratelimitv1alpha1 "github.com/solo-io/gloo/projects/gloo/pkg/api/external/solo/ratelimit" - v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" - extauthv1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1" - extauthkubev1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1/kube/apis/enterprise.gloo.solo.io/v1" - graphqlv1beta1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/graphql/v1beta1" - graphqlkubev1beta1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/graphql/v1beta1/kube/apis/graphql.gloo.solo.io/v1beta1" - v1snap "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/gloosnapshot" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" - "github.com/solo-io/gloo/projects/gloo/pkg/xds" - rlv1alpha1 "github.com/solo-io/solo-apis/pkg/api/ratelimit.solo.io/v1alpha1" - crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1" - "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - apiv1 "sigs.k8s.io/gateway-api/apis/v1" -) - -var _ = Describe("History", func() { - - var ( - ctx context.Context - - clientBuilder *fake.ClientBuilder - history iosnapshot.History - - historyFactorParams iosnapshot.HistoryFactoryParameters - ) - - BeforeEach(func() { - ctx = context.Background() - clientBuilder = fake.NewClientBuilder().WithScheme(schemes.DefaultScheme()) - - historyFactorParams = iosnapshot.HistoryFactoryParameters{ - Settings: &v1.Settings{ - Metadata: &core.Metadata{ - Name: "my-settings", - Namespace: defaults.GlooSystem, - }, - }, - Cache: &xds.MockXdsCache{}, - } - }) - - Context("NewHistory", func() { - - var ( - deploymentGvk = schema.GroupVersionKind{ - Group: appsv1.GroupName, - Version: "v1", - Kind: "Deployment", - } - ) - - When("Deployment GVK is included", func() { - - BeforeEach(func() { - clientObjects := []client.Object{ - &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-deploy", - Namespace: "a", - }, - Spec: appsv1.DeploymentSpec{ - MinReadySeconds: 5, - }, - }, - } - - history = iosnapshot.NewHistory( - historyFactorParams.Cache, - historyFactorParams.Settings, - clientBuilder.WithObjects(clientObjects...).Build(), - append(iosnapshot.CompleteInputSnapshotGVKs, deploymentGvk), // include the Deployment GVK - ) - }) - - It("GetInputSnapshot includes Deployments", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement(matchers.MatchClientObject( - deploymentGvk, - types.NamespacedName{ - Namespace: "a", - Name: "kube-deploy", - }, - gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Spec": Equal(appsv1.DeploymentSpec{ - MinReadySeconds: 5, - }), - })), - )), "we should now see the deployment in the input snapshot results") - }) - - }) - - When("Deployment GVK is excluded", func() { - - BeforeEach(func() { - clientObjects := []client.Object{ - &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-deploy", - Namespace: "a", - }, - }, - } - - history = iosnapshot.NewHistory(&xds.MockXdsCache{}, - &v1.Settings{ - Metadata: &core.Metadata{ - Name: "my-settings", - Namespace: defaults.GlooSystem, - }, - }, - clientBuilder.WithObjects(clientObjects...).Build(), - iosnapshot.CompleteInputSnapshotGVKs, // do not include the Deployment GVK - ) - }) - - It("GetInputSnapshot excludes Deployments", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).NotTo(ContainElement( - matchers.MatchClientObjectGvk(deploymentGvk), - ), "snapshot should not include the deployment") - }) - }) - - }) - - Context("GetInputSnapshot", func() { - - BeforeEach(func() { - clientObjects := []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-secret", - Namespace: "secret", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - Data: map[string][]byte{ - "key": []byte("sensitive-data"), - }, - }, - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-configmap", - Namespace: "configmap", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - Data: map[string]string{ - "key": "value", - }, - }, - &apiv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-gw", - Namespace: "a", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &apiv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-gw-class", - Namespace: "c", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &apiv1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-http-route", - Namespace: "b", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &apiv1beta1.ReferenceGrant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-ref-grant", - Namespace: "d", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &v1alpha1.GatewayParameters{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-gwp", - Namespace: "e", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.ListenerOption{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-lo", - Namespace: "f", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.HttpListenerOption{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-hlo", - Namespace: "g", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.VirtualHostOption{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-vho", - Namespace: "i", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.RouteOption{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-rto", - Namespace: "h", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &extauthkubev1.AuthConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-ac", - Namespace: "j", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &rlv1alpha1.RateLimitConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-rlc", - Namespace: "k", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &graphqlkubev1beta1.GraphQLApi{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-graphql", - Namespace: "graphql", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gloov1.Settings{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-settings", - Namespace: "settings", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gloov1.Upstream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-upstream", - Namespace: "upstream", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gloov1.UpstreamGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-upstreamgroup", - Namespace: "upstreamgroup", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gloov1.Proxy{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-proxy", - Namespace: "proxy", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-edgegateway", - Namespace: "edgegateway", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.MatchableHttpGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-httpgateway", - Namespace: "httpgateway", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.MatchableTcpGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-tcpgateway", - Namespace: "tcpgateway", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.VirtualService{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-virtualservice", - Namespace: "virtualservice", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - &gatewaykubev1.RouteTable{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-routetable", - Namespace: "routetable", - ManagedFields: []metav1.ManagedFieldsEntry{{ - Manager: "manager", - }}, - }, - }, - } - - history = iosnapshot.NewHistory( - historyFactorParams.Cache, - historyFactorParams.Settings, - clientBuilder.WithObjects(clientObjects...).Build(), - iosnapshot.CompleteInputSnapshotGVKs) - }) - - Context("Kubernetes Core Resources", func() { - - It("Includes Secrets (redacted)", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - matchers.MatchClientObject( - wellknownkube.SecretGVK, - types.NamespacedName{ - Name: "kube-secret", - Namespace: "secret", - }, - gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "ObjectMeta": matchers.HaveNilManagedFields(), - "Data": HaveKeyWithValue("key", []byte("")), - })), - ), - ), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.SecretGVK, "secret", "kube-secret")) - }) - - It("Includes ConfigMaps", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - matchers.MatchClientObject( - wellknownkube.ConfigMapGVK, - types.NamespacedName{ - Name: "kube-configmap", - Namespace: "configmap", - }, - gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "ObjectMeta": matchers.HaveNilManagedFields(), - "Data": HaveKeyWithValue("key", "value"), - })), - ), - ), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.ConfigMapGVK, "configmap", "kube-configmap")) - }) - - }) - - Context("Kubernetes Gateway API Resources", func() { - - It("Includes Gateways (Kubernetes API)", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(wellknown.GatewayGVK, types.NamespacedName{ - Name: "kube-gw", - Namespace: "a", - }), - ), fmt.Sprintf("results should contain %v %s.%s", wellknown.GatewayGVK, "a", "kube-gw")) - - }) - - It("Includes GatewayClasses", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(wellknown.GatewayClassGVK, types.NamespacedName{ - Name: "kube-gw-class", - Namespace: "c", - }), - ), fmt.Sprintf("results should contain %v %s.%s", wellknown.GatewayClassGVK, "c", "kube-gw-class")) - }) - - It("Includes HTTPRoutes", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(wellknown.HTTPRouteGVK, types.NamespacedName{ - Name: "kube-http-route", - Namespace: "b", - }), - ), fmt.Sprintf("results should contain %v %s.%s", wellknown.HTTPRouteGVK, "b", "kube-http-route")) - }) - - It("Includes ReferenceGrants", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(wellknown.ReferenceGrantGVK, types.NamespacedName{ - Name: "kube-ref-grant", - Namespace: "d", - }), - ), fmt.Sprintf("results should contain %v %s.%s", wellknown.ReferenceGrantGVK, "d", "kube-ref-grant")) - }) - - }) - - Context("Gloo Kubernetes Gateway Integration Resources", func() { - - It("Includes GatewayParameters", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(v1alpha1.GatewayParametersGVK, types.NamespacedName{ - Name: "kube-gwp", - Namespace: "e", - }), - ), fmt.Sprintf("results should contain %v %s.%s", v1alpha1.GatewayParametersGVK, "e", "kube-gwp")) - }) - - It("Includes ListenerOptions", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.ListenerOptionGVK, types.NamespacedName{ - Name: "kube-lo", - Namespace: "f", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.ListenerOptionGVK, "f", "kube-lo")) - }) - - It("Includes HttpListenerOptions", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.HttpListenerOptionGVK, types.NamespacedName{ - Name: "kube-hlo", - Namespace: "g", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.HttpListenerOptionGVK, "g", "kube-hlo")) - }) - - }) - - Context("Gloo Gateway Policy Resources", func() { - - It("Includes VirtualHostOptions", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.VirtualHostOptionGVK, types.NamespacedName{ - Name: "kube-vho", - Namespace: "i", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.VirtualHostOptionGVK, "i", "kube-vho")) - }) - - It("Includes RouteOptions", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.RouteOptionGVK, types.NamespacedName{ - Name: "kube-rto", - Namespace: "h", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.RouteOptionGVK, "h", "kube-rto")) - }) - - }) - - Context("Enterprise Extension Resources", func() { - - It("Excludes AuthConfigs", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).NotTo(ContainElement( - matchers.MatchClientObjectGvk(extauthv1.AuthConfigGVK), - ), fmt.Sprintf("results should not contain %v", extauthv1.AuthConfigGVK)) - }) - - It("Excludes RateLimitConfigs", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).NotTo(ContainElement( - matchers.MatchClientObjectGvk(ratelimitv1alpha1.RateLimitConfigGVK), - ), fmt.Sprintf("results should not contain %v", ratelimitv1alpha1.RateLimitConfigGVK)) - }) - - It("Excludes GraphQLApis", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).NotTo(ContainElement( - matchers.MatchClientObjectGvk(graphqlv1beta1.GraphQLApiGVK), - ), fmt.Sprintf("results should not contain %v", graphqlv1beta1.GraphQLApiGVK)) - }) - - }) - - Context("Gloo Resources", func() { - - It("Includes Settings", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(v1.SettingsGVK, types.NamespacedName{ - Name: "kube-settings", - Namespace: "settings", - }), - ), fmt.Sprintf("results should contain %v %s.%s", v1.SettingsGVK, "settings", "kube-settings")) - }) - - It("Excludes Endpoints", func() { - // Endpoints are a type that are stored in-memory, but the ControlPlane - // As a result, GetInputSnapshot does not attempt to return them - - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).NotTo(ContainElement( - matchers.MatchClientObjectGvk(v1.EndpointGVK), - ), fmt.Sprintf("results should not contain %v", v1.EndpointGVK)) - }) - - It("Includes Upstreams", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(v1.UpstreamGVK, types.NamespacedName{ - Name: "kube-upstream", - Namespace: "upstream", - }), - ), fmt.Sprintf("results should contain %v %s.%s", v1.UpstreamGVK, "upstream", "kube-upstream")) - }) - - It("Includes UpstreamGroups", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(v1.UpstreamGroupGVK, types.NamespacedName{ - Name: "kube-upstreamgroup", - Namespace: "upstreamgroup", - }), - ), fmt.Sprintf("results should contain %v %s.%s", v1.UpstreamGroupGVK, "upstreamgroup", "kube-upstreamgroup")) - }) - - It("Includes Proxies", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(v1.ProxyGVK, types.NamespacedName{ - Name: "kube-proxy", - Namespace: "proxy", - }), - ), fmt.Sprintf("results should contain %v %s.%s", v1.ProxyGVK, "proxy", "kube-proxy")) - }) - - }) - - Context("Edge Gateway API Resources", func() { - - It("Includes Gateways", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.GatewayGVK, types.NamespacedName{ - Name: "kube-edgegateway", - Namespace: "edgegateway", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.GatewayGVK, "edgegateway", "kube-edgegateway")) - }) - - It("Includes HttpGateways", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.MatchableHttpGatewayGVK, types.NamespacedName{ - Name: "kube-httpgateway", - Namespace: "httpgateway", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.MatchableHttpGatewayGVK, "httpgateway", "kube-httpgateway")) - }) - - It("Includes TcpGateways", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.MatchableTcpGatewayGVK, types.NamespacedName{ - Name: "kube-tcpgateway", - Namespace: "tcpgateway", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.MatchableTcpGatewayGVK, "tcpgateway", "kube-tcpgateway")) - }) - - It("Includes VirtualServices", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.VirtualServiceGVK, types.NamespacedName{ - Name: "kube-virtualservice", - Namespace: "virtualservice", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.VirtualServiceGVK, "virtualservice", "kube-virtualservice")) - }) - - It("Includes RouteTables", func() { - returnedResources := getInputSnapshotObjects(ctx, history) - Expect(returnedResources).To(ContainElement( - simpleObjectMatcher(gatewayv1.RouteTableGVK, types.NamespacedName{ - Name: "kube-routetable", - Namespace: "routetable", - }), - ), fmt.Sprintf("results should contain %v %s.%s", gatewayv1.RouteTableGVK, "routetable", "kube-routetable")) - }) - - }) - - }) - - Context("GetEdgeApiSnapshot", func() { - - BeforeEach(func() { - history = iosnapshot.NewHistory( - historyFactorParams.Cache, - historyFactorParams.Settings, - clientBuilder.Build(), // no objects, because this API doesn't rely on the kube client - iosnapshot.CompleteInputSnapshotGVKs, - ) - }) - - It("returns ApiSnapshot", func() { - setSnapshotOnHistory(ctx, history, &v1snap.ApiSnapshot{ - Proxies: v1.ProxyList{ - {Metadata: &core.Metadata{Name: "proxy", Namespace: defaults.GlooSystem}}, - }, - Upstreams: v1.UpstreamList{ - {Metadata: &core.Metadata{Name: "upstream", Namespace: defaults.GlooSystem}}, - }, - Artifacts: v1.ArtifactList{ - { - Metadata: &core.Metadata{ - Name: "artifact", - Namespace: defaults.GlooSystem, - Annotations: map[string]string{ - corev1.LastAppliedConfigAnnotation: "last-applied-configuration", - "safe-annotation": "safe-annotation-value", - }, - }, - Data: map[string]string{ - "key": "sensitive-data", - }, - }, - }, - Secrets: v1.SecretList{ - { - Metadata: &core.Metadata{ - Name: "secret", - Namespace: defaults.GlooSystem, - Annotations: map[string]string{ - corev1.LastAppliedConfigAnnotation: "last-applied-configuration", - "safe-annotation": "safe-annotation-value", - }, - }, - Kind: &v1.Secret_Tls{ - Tls: &v1.TlsSecret{ - CertChain: "cert-chain", - PrivateKey: "private-key", - RootCa: "root-ca", - OcspStaple: nil, - }, - }, - }, - }, - }) - - snap := getEdgeApiSnapshot(ctx, history) - Expect(snap.Proxies).To(ContainElement( - skmatchers.MatchProto(&v1.Proxy{Metadata: &core.Metadata{Name: "proxy", Namespace: defaults.GlooSystem}}), - )) - Expect(snap.Upstreams).To(ContainElement( - skmatchers.MatchProto(&v1.Upstream{Metadata: &core.Metadata{Name: "upstream", Namespace: defaults.GlooSystem}}), - )) - Expect(snap.Artifacts).To(ContainElement( - skmatchers.MatchProto(&v1.Artifact{ - Metadata: &core.Metadata{ - Name: "artifact", - Namespace: defaults.GlooSystem, - Annotations: map[string]string{ - corev1.LastAppliedConfigAnnotation: "", - "safe-annotation": "safe-annotation-value", - }, - }, - Data: map[string]string{ - "key": "", - }, - }), - ), "artifacts are included and redacted") - Expect(snap.Secrets).To(ContainElement( - skmatchers.MatchProto(&v1.Secret{ - Metadata: &core.Metadata{ - Name: "secret", - Namespace: defaults.GlooSystem, - Annotations: map[string]string{ - corev1.LastAppliedConfigAnnotation: "", - "safe-annotation": "safe-annotation-value", - }, - }, - Kind: nil, - }), - ), "secrets are included and redacted") - }) - - }) - - Context("GetProxySnapshot", func() { - - BeforeEach(func() { - history = iosnapshot.NewHistory( - historyFactorParams.Cache, - historyFactorParams.Settings, - clientBuilder.Build(), // no objects, because this API doesn't rely on the kube client - iosnapshot.CompleteInputSnapshotGVKs, - ) - }) - - It("returns ApiSnapshot with _only_ Proxies", func() { - setSnapshotOnHistory(ctx, history, &v1snap.ApiSnapshot{ - Proxies: v1.ProxyList{ - {Metadata: &core.Metadata{Name: "proxy", Namespace: defaults.GlooSystem}}, - }, - Upstreams: v1.UpstreamList{ - {Metadata: &core.Metadata{Name: "upstream", Namespace: defaults.GlooSystem}}, - }, - }) - - returnedResources := getProxySnapshotResources(ctx, history) - Expect(returnedResources).To(And( - matchers.ContainCustomResource( - matchers.MatchTypeMeta(v1.ProxyGVK), - matchers.MatchObjectMeta(types.NamespacedName{ - Namespace: defaults.GlooSystem, - Name: "proxy", - }), - gstruct.Ignore(), - ), - )) - Expect(returnedResources).NotTo(matchers.ContainCustomResourceType(v1.UpstreamGVK), "non-proxy resources should be excluded") - }) - - }) - -}) - -func getInputSnapshotObjects(ctx context.Context, history iosnapshot.History) []client.Object { - snapshotResponse := history.GetInputSnapshot(ctx) - Expect(snapshotResponse.Error).NotTo(HaveOccurred()) - - responseObjects, ok := snapshotResponse.Data.([]client.Object) - Expect(ok).To(BeTrue()) - - return responseObjects -} - -func getProxySnapshotResources(ctx context.Context, history iosnapshot.History) []crdv1.Resource { - snapshotResponse := history.GetProxySnapshot(ctx) - Expect(snapshotResponse.Error).NotTo(HaveOccurred()) - - responseObjects, ok := snapshotResponse.Data.([]crdv1.Resource) - Expect(ok).To(BeTrue()) - - return responseObjects -} - -func getEdgeApiSnapshot(ctx context.Context, history iosnapshot.History) *v1snap.ApiSnapshot { - snapshotResponse := history.GetEdgeApiSnapshot(ctx) - Expect(snapshotResponse.Error).NotTo(HaveOccurred()) - - response, ok := snapshotResponse.Data.(*v1snap.ApiSnapshot) - Expect(ok).To(BeTrue()) - - return response -} - -// setSnapshotOnHistory sets the ApiSnapshot on the history, and blocks until it has been processed -// This is a utility method to help developers write tests, without having to worry about the asynchronous -// nature of the `Set` API on the History -func setSnapshotOnHistory(ctx context.Context, history iosnapshot.History, snap *v1snap.ApiSnapshot) { - gwSignal := &gatewayv1.Gateway{ - // We append a custom Gateway to the Snapshot, and then use that object - // to verify the Snapshot has been processed - Metadata: &core.Metadata{Name: "gw-signal", Namespace: defaults.GlooSystem}, - } - - snap.Gateways = append(snap.Gateways, gwSignal) - history.SetApiSnapshot(snap) - - Eventually(func(g Gomega) { - apiSnapshot := getEdgeApiSnapshot(ctx, history) - g.Expect(apiSnapshot.Gateways).To(ContainElement(skmatchers.MatchProto(gwSignal))) - }). - WithPolling(time.Millisecond*100). - WithTimeout(time.Second*5). - Should(Succeed(), fmt.Sprintf("snapshot should eventually contain resource %v %s", gatewayv1.GatewayGVK, gwSignal.GetMetadata().Ref().String())) -} - -func simpleObjectMatcher(gvk schema.GroupVersionKind, namespacedName types.NamespacedName) gomegatypes.GomegaMatcher { - return matchers.MatchClientObject( - gvk, - namespacedName, - gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "ObjectMeta": matchers.HaveNilManagedFields(), - })), - ) -} diff --git a/projects/gloo/pkg/syncer/envoy_translator_syncer.go b/projects/gloo/pkg/syncer/envoy_translator_syncer.go index aaf947f3331..7ef15727427 100644 --- a/projects/gloo/pkg/syncer/envoy_translator_syncer.go +++ b/projects/gloo/pkg/syncer/envoy_translator_syncer.go @@ -2,11 +2,7 @@ package syncer import ( "context" - "encoding/json" - "fmt" - "net/http" - "github.com/gorilla/mux" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -85,10 +81,6 @@ func (s *translatorSyncer) syncEnvoy(ctx context.Context, snap *v1snap.ApiSnapsh stopwatch.Start() defer stopwatch.Stop(ctx) - // store snap for debug tooling - s.snapshotHistory.SetApiSnapshot(snap) - s.latestSnap = snap - var nonKubeProxies v1.ProxyList for _, proxy := range snap.Proxies { proxyType := utils.GetTranslatorValue(proxy.GetMetadata()) @@ -214,54 +206,3 @@ func (s *translatorSyncer) syncEnvoy(ctx context.Context, snap *v1snap.ApiSnapsh logger.Debugf("gloo reports to be written: %v", allReports) } - -// ServeXdsSnapshots exposes Gloo configuration as an API when `devMode` in Settings is True. -// Deprecated: https://github.com/solo-io/gloo/issues/6494 -// Prefer to use the iosnapshot.History and pkg/servers/admin -func (s *translatorSyncer) ServeXdsSnapshots() error { - return s.ContextuallyServeXdsSnapshots(context.Background()) -} - -// ContextuallyServeXdsSnapshots exposes Gloo configuration as an API when `devMode` in Settings is True. -// Deprecated: https://github.com/solo-io/gloo/issues/6494 -// Prefer to use the iosnapshot.History and pkg/servers/admin -func (s *translatorSyncer) ContextuallyServeXdsSnapshots(ctx context.Context) error { - - r := mux.NewRouter() - - r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "%v", "Developer API") - }) - r.HandleFunc("/xds", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "%+v", prettify(s.xdsCache.GetStatusKeys())) - }) - r.HandleFunc("/xds/{key}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - xdsCacheKey := vars["key"] - - xdsSnapshot, _ := s.xdsCache.GetSnapshot(xdsCacheKey) - _, _ = fmt.Fprintf(w, "%+v", prettify(xdsSnapshot)) - }) - r.HandleFunc("/api", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "%+v", prettify(s.latestSnap)) - }) - - server := &http.Server{Addr: fmt.Sprintf(":%d", devModePort), Handler: r} - - go func() { - <-ctx.Done() - _ = server.Close() - }() - - return server.ListenAndServe() - -} - -func prettify(original interface{}) string { - b, err := json.MarshalIndent(original, "", " ") - if err != nil { - return "" - } - - return string(b) -} diff --git a/projects/gloo/pkg/syncer/setup/extensions.go b/projects/gloo/pkg/syncer/setup/extensions.go index e2c25a58b9c..c7a25a6ece9 100644 --- a/projects/gloo/pkg/syncer/setup/extensions.go +++ b/projects/gloo/pkg/syncer/setup/extensions.go @@ -3,7 +3,6 @@ package setup import ( errors "github.com/rotisserie/eris" "github.com/solo-io/gloo/projects/gloo/pkg/plugins" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/gloo/projects/gloo/pkg/syncer" xdsserver "github.com/solo-io/solo-kit/pkg/api/v1/control-plane/server" ) @@ -30,18 +29,11 @@ type Extensions struct { // ApiEmitterChannel is a channel that forces the API Emitter to emit a new API Snapshot ApiEmitterChannel chan struct{} - - // SnapshotHistoryFactory is the factory function which will produce a History object - // This history object is used by the ControlPlane to track internal state - SnapshotHistoryFactory iosnapshot.HistoryFactory } // Validate returns an error if the Extensions are invalid, nil otherwise func (e Extensions) Validate() error { - if e.SnapshotHistoryFactory == nil { - return ErrNilExtension("SnapshotHistoryFactory") - } if e.PluginRegistryFactory == nil { return ErrNilExtension("PluginRegistryFactory") } diff --git a/projects/gloo/pkg/syncer/setup/extensions_test.go b/projects/gloo/pkg/syncer/setup/extensions_test.go index aadf5a1a8f8..646e09a7e45 100644 --- a/projects/gloo/pkg/syncer/setup/extensions_test.go +++ b/projects/gloo/pkg/syncer/setup/extensions_test.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/types" "github.com/solo-io/gloo/projects/gloo/pkg/plugins" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/gloo/projects/gloo/pkg/syncer" ) @@ -17,15 +16,10 @@ var _ = Describe("Extensions", func() { func(extensions Extensions, expectedError types.GomegaMatcher) { Expect(extensions.Validate()).To(expectedError) }, - Entry("missing SnapshotHistoryFactory", Extensions{ - SnapshotHistoryFactory: nil, - }, MatchError(ErrNilExtension("SnapshotHistoryFactory"))), Entry("missing PluginRegistryFactory", Extensions{ - SnapshotHistoryFactory: iosnapshot.GetHistoryFactory(), - PluginRegistryFactory: nil, + PluginRegistryFactory: nil, }, MatchError(ErrNilExtension("PluginRegistryFactory"))), Entry("missing ApiEmitterChannel", Extensions{ - SnapshotHistoryFactory: iosnapshot.GetHistoryFactory(), PluginRegistryFactory: func(ctx context.Context) plugins.PluginRegistry { // non-nil function return nil @@ -33,7 +27,6 @@ var _ = Describe("Extensions", func() { ApiEmitterChannel: nil, }, MatchError(ErrNilExtension("ApiEmitterChannel"))), Entry("missing SyncerExtensions", Extensions{ - SnapshotHistoryFactory: iosnapshot.GetHistoryFactory(), PluginRegistryFactory: func(ctx context.Context) plugins.PluginRegistry { // non-nil function return nil @@ -42,7 +35,6 @@ var _ = Describe("Extensions", func() { SyncerExtensions: nil, }, MatchError(ErrNilExtension("SyncerExtensions"))), Entry("missing nothing", Extensions{ - SnapshotHistoryFactory: iosnapshot.GetHistoryFactory(), PluginRegistryFactory: func(ctx context.Context) plugins.PluginRegistry { // non-nil function return nil diff --git a/projects/gloo/pkg/syncer/setup/setup_syncer.go b/projects/gloo/pkg/syncer/setup/setup_syncer.go index 98cc942cd24..fcd292c14a4 100644 --- a/projects/gloo/pkg/syncer/setup/setup_syncer.go +++ b/projects/gloo/pkg/syncer/setup/setup_syncer.go @@ -15,7 +15,6 @@ import ( "github.com/solo-io/gloo/pkg/utils/settingsutil" "github.com/solo-io/gloo/pkg/utils/statsutils/metrics" "github.com/solo-io/gloo/projects/gloo/pkg/debug" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/gloo/projects/gloo/pkg/utils" "github.com/solo-io/solo-kit/pkg/api/v1/control-plane/cache" @@ -26,7 +25,6 @@ import ( consulapi "github.com/hashicorp/consul/api" vaultapi "github.com/hashicorp/vault/api" "go.uber.org/zap" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/reflection" "google.golang.org/protobuf/proto" @@ -482,9 +480,8 @@ func RunGloo(opts bootstrap.Opts) error { ratelimitExt.NewTranslatorSyncerExtension, extauthExt.NewTranslatorSyncerExtension, }, - ApiEmitterChannel: make(chan struct{}), - XdsCallbacks: nil, - SnapshotHistoryFactory: iosnapshot.GetHistoryFactory(), + ApiEmitterChannel: make(chan struct{}), + XdsCallbacks: nil, } return RunGlooWithExtensions(opts, glooExtensions) @@ -506,7 +503,6 @@ func RunGlooWithExtensions(opts bootstrap.Opts, extensions Extensions) error { watchOpts.Ctx = contextutils.WithLogger(watchOpts.Ctx, "setup") opts.WatchOpts.Ctx = contextutils.WithLogger(opts.WatchOpts.Ctx, "gloo") - runErrorGroup, _ := errgroup.WithContext(watchOpts.Ctx) logger := contextutils.LoggerFrom(watchOpts.Ctx) // MARK: build resource clients @@ -890,20 +886,6 @@ func RunGlooWithExtensions(opts bootstrap.Opts, extensions Extensions) error { } gwValidationSyncer := gwvalidation.NewValidator(validationConfig) - // startFuncs represents the set of StartFunc that should be executed at startup - // At the moment, the functionality is used minimally. - // Overtime, we should break up this large function into smaller StartFunc - startFuncs := map[string]StartFunc{} - - // snapshotHistory is a utility for managing the state of the input/output snapshots that the Control Plane - // consumes and produces. This object is then used by our Admin Server, to provide this data on demand - snapshotHistory := extensions.SnapshotHistoryFactory(iosnapshot.HistoryFactoryParameters{ - Settings: opts.Settings, - Cache: opts.ControlPlane.SnapshotCache, - }) - - startFuncs["admin-server"] = AdminServerStartFunc(snapshotHistory, opts.KrtDebugger) - if opts.ProxyReconcileQueue != nil { go runQueue(watchOpts.Ctx, opts.ProxyReconcileQueue, opts.WriteNamespace, proxyClient) } @@ -926,7 +908,6 @@ func RunGlooWithExtensions(opts bootstrap.Opts, extensions Extensions) error { proxyClient, opts.WriteNamespace, opts.Identity, - snapshotHistory, ) // MARK: build & run api snap loop @@ -1046,39 +1027,6 @@ func RunGlooWithExtensions(opts bootstrap.Opts, extensions Extensions) error { } } - ExecuteAsynchronousStartFuncs( - watchOpts.Ctx, - opts, - extensions, - startFuncs, - runErrorGroup, - ) - - go func() { - // It is critical that the RunGlooWithExtensions function does not block. - // As a result, we monitor the runErrorGroup and just drop errors on the shared "errs" channel if one occurs - runErr := runErrorGroup.Wait() - if runErr != nil { - errs <- runErr - } - }() - - go func() { - for { - select { - case err, ok := <-errs: - if !ok { - return - } - logger.Errorw("gloo main event loop", zap.Error(err)) - case <-watchOpts.Ctx.Done(): - // think about closing this channel - // close(errs) - return - } - } - }() - logger.Infof("Gloo setup completed successfully") return nil } diff --git a/projects/gloo/pkg/syncer/setup/start_func.go b/projects/gloo/pkg/syncer/setup/start_func.go deleted file mode 100644 index 6cea9c0dd5d..00000000000 --- a/projects/gloo/pkg/syncer/setup/start_func.go +++ /dev/null @@ -1,85 +0,0 @@ -package setup - -import ( - "context" - "fmt" - "net/http" - - "github.com/solo-io/gloo/pkg/utils/envutils" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" - "github.com/solo-io/go-utils/stats" - "istio.io/istio/pkg/kube/krt" - - "golang.org/x/sync/errgroup" - - "github.com/solo-io/go-utils/contextutils" - - "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap" -) - -// StartFunc represents a function that will be called with the initialized bootstrap.Opts -// and Extensions. This is invoked each time the setup_syncer is executed -// (which runs whenever the Setting CR is modified) -type StartFunc func(ctx context.Context, opts bootstrap.Opts, extensions Extensions) error - -// ExecuteAsynchronousStartFuncs accepts a collection of StartFunc inputs, and executes them within an Error Group -func ExecuteAsynchronousStartFuncs( - ctx context.Context, - opts bootstrap.Opts, - extensions Extensions, - startFuncs map[string]StartFunc, - errorGroup *errgroup.Group, -) { - for name, start := range startFuncs { - startFn := start // pike - namedCtx := contextutils.WithLogger(ctx, name) - - errorGroup.Go( - func() error { - contextutils.LoggerFrom(namedCtx).Infof("starting %s goroutine", name) - err := startFn(namedCtx, opts, extensions) - if err != nil { - contextutils.LoggerFrom(namedCtx).Errorf("%s goroutine failed: %v", name, err) - } - return err - }, - ) - } - - contextutils.LoggerFrom(ctx).Debug("main goroutines successfully started") -} - -// AdminServerStartFunc returns the setup.StartFunc for the Admin Server -// The Admin Server is the groundwork for an Administration Interface, similar to that of Envoy -// https://github.com/solo-io/gloo/issues/6494 -// The endpoints that are available on this server are split between two places: -// 1. The default endpoints are defined by our stats server: https://github.com/solo-io/go-utils/blob/8eda16b9878d71673e6a3a9756f6088160f75468/stats/stats.go#L79 -// 2. Custom endpoints are defined by our admin server handler in `gloo/pkg/servers/admin` -func AdminServerStartFunc(history iosnapshot.History, dbg *krt.DebugHandler) StartFunc { - return func(ctx context.Context, opts bootstrap.Opts, extensions Extensions) error { - // serverHandlers defines the custom handlers that the Admin Server will support - serverHandlers := admin.ServerHandlers(ctx, history, dbg) - - // The Stats Server is used as the running server for our admin endpoints - // - // NOTE: There is a slight difference in how we run this server -vs- how we used to run it - // In the past, we would start the server once, at the beginning of the running container - // Now, we start a new server each time we invoke a StartFunc. - if serverAdminHandlersWithStats() { - stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), serverHandlers) - } else { - stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) { - // let people know these moved - profiles[fmt.Sprintf("http://localhost:%d/snapshots/", admin.AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", admin.AdminPort) - }) - admin.StartHandlers(ctx, serverHandlers) - } - - return nil - } -} - -func serverAdminHandlersWithStats() bool { - return envutils.IsEnvTruthy("ADMIN_HANDLERS_WITH_STATS") -} diff --git a/projects/gloo/pkg/syncer/translator_syncer.go b/projects/gloo/pkg/syncer/translator_syncer.go index 68735b7191e..a445408affe 100644 --- a/projects/gloo/pkg/syncer/translator_syncer.go +++ b/projects/gloo/pkg/syncer/translator_syncer.go @@ -6,7 +6,6 @@ import ( "time" "github.com/solo-io/gloo/pkg/utils/statsutils/metrics" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/gloo/projects/gloo/pkg/utils" "github.com/hashicorp/go-multierror" @@ -38,15 +37,6 @@ type translatorSyncer struct { proxyClient v1.ProxyClient writeNamespace string - // used for debugging purposes only - // Deprecated: https://github.com/solo-io/gloo/issues/6494 - // Prefer to use the iosnapshot.History - latestSnap *v1snap.ApiSnapshot - - // snapshotHistory is used for debugging purposes - // The syncer updates the History each time it runs, and the History is then used by the Admin Server - snapshotHistory iosnapshot.History - statusSyncer *statusSyncer } @@ -76,7 +66,6 @@ func NewTranslatorSyncer( proxyClient v1.ProxyClient, writeNamespace string, identity leaderelector.Identity, - snapshotHistory iosnapshot.History, ) v1snap.ApiSyncer { s := &translatorSyncer{ translator: translator, @@ -96,14 +85,8 @@ func NewTranslatorSyncer( leaderStartupAction: leaderelector.NewLeaderStartupAction(identity), reportsLock: sync.RWMutex{}, }, - snapshotHistory: snapshotHistory, - } - if devMode { - // TODO(ilackarms): move this somewhere else? - go func() { - _ = s.ContextuallyServeXdsSnapshots(ctx) - }() } + go s.statusSyncer.syncStatusOnEmit(ctx) s.statusSyncer.leaderStartupAction.WatchElectionResults(ctx) return s diff --git a/projects/gloo/pkg/syncer/translator_syncer_test.go b/projects/gloo/pkg/syncer/translator_syncer_test.go index 550e38a0a59..39f0c31bc9d 100644 --- a/projects/gloo/pkg/syncer/translator_syncer_test.go +++ b/projects/gloo/pkg/syncer/translator_syncer_test.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/solo-io/gloo/pkg/utils/statsutils/metrics" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" "github.com/solo-io/gloo/pkg/bootstrap/leaderelector/singlereplica" @@ -78,11 +77,7 @@ var _ = Describe("Translate Proxy", func() { rep := reporter.NewReporter(ref, statusClient, proxyClient.BaseClient(), upstreamClient) - history := iosnapshot.GetHistoryFactory()(iosnapshot.HistoryFactoryParameters{ - Settings: settings, - Cache: xdsCache, - }) - syncer = NewTranslatorSyncer(ctx, &mockTranslator{true, false, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity(), history) + syncer = NewTranslatorSyncer(ctx, &mockTranslator{true, false, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity()) snap = &v1snap.ApiSnapshot{ Proxies: v1.ProxyList{ proxy, @@ -120,11 +115,7 @@ var _ = Describe("Translate Proxy", func() { snap = &snapClone snap.Proxies[0] = p1 - history = iosnapshot.GetHistoryFactory()(iosnapshot.HistoryFactoryParameters{ - Settings: settings, - Cache: xdsCache, - }) - syncer = NewTranslatorSyncer(ctx, &mockTranslator{false, false, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity(), history) + syncer = NewTranslatorSyncer(ctx, &mockTranslator{false, false, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity()) err = syncer.Sync(context.Background(), snap) Expect(err).NotTo(HaveOccurred()) @@ -258,11 +249,7 @@ var _ = Describe("Translate multiple proxies with errors", func() { rep := reporter.NewReporter(ref, statusClient, proxyClient.BaseClient(), usClient) - history := iosnapshot.GetHistoryFactory()(iosnapshot.HistoryFactoryParameters{ - Settings: settings, - Cache: xdsCache, - }) - syncer = NewTranslatorSyncer(ctx, &mockTranslator{true, true, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity(), history) + syncer = NewTranslatorSyncer(ctx, &mockTranslator{true, true, nil}, xdsCache, sanitizer, rep, false, nil, settings, statusMetrics, nil, proxyClient, "", singlereplica.Identity()) snap = &v1snap.ApiSnapshot{ Proxies: v1.ProxyList{ proxy1, diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index d68d04963e7..afd90fcb02b 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -20,7 +20,7 @@ import ( "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" + "github.com/solo-io/gloo/projects/gateway2/admin" "github.com/solo-io/go-utils/threadsafe" ) diff --git a/test/kubernetes/testutils/assertions/gloo.go b/test/kubernetes/testutils/assertions/gloo.go index be904c2db85..73fb7a5b43a 100644 --- a/test/kubernetes/testutils/assertions/gloo.go +++ b/test/kubernetes/testutils/assertions/gloo.go @@ -12,7 +12,7 @@ import ( "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" + "github.com/solo-io/gloo/projects/gateway2/admin" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" )