Admin server - part 1 (#10460)

docs/content/static/content/osa_provided.md

@@ -28,7 +28,6 @@ Name|Version|License
 [google/go-github](https://github.com/google/go-github)|v17.0.0+incompatible|BSD 3-clause "New" or "Revised" License
 [go-github/v32](https://github.com/google/go-github)|v32.0.0|BSD 3-clause "New" or "Revised" License
 [google/uuid](https://github.com/google/uuid)|v1.6.0|BSD 3-clause "New" or "Revised" License
-[gorilla/mux](https://github.com/gorilla/mux)|v1.8.1|BSD 3-clause "New" or "Revised" License
 [grpc-ecosystem/go-grpc-middleware](https://github.com/grpc-ecosystem/go-grpc-middleware)|v1.4.0|Apache License 2.0
 [hinshun/vt10x](https://github.com/hinshun/vt10x)|v0.0.0-20180809195222-d55458df857c|MIT License
 [imdario/mergo](https://github.com/imdario/mergo)|v0.3.16|BSD 3-clause "New" or "Revised" License

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/golang/protobuf v1.5.4
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-github/v32 v32.0.0
-	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/hashicorp/consul/api v1.28.2
 	github.com/hashicorp/go-multierror v1.1.1

pkg/utils/glooadminutils/admincli/client.go

@@ -8,7 +8,7 @@ import (
 	"github.com/rotisserie/eris"
 	"github.com/solo-io/gloo/pkg/utils/cmdutils"
 	"github.com/solo-io/gloo/pkg/utils/requestutils/curl"
-	"github.com/solo-io/gloo/projects/gloo/pkg/servers/admin"
+	"github.com/solo-io/gloo/projects/gateway2/admin"
 	"github.com/solo-io/go-utils/threadsafe"
 )
 

projects/gloo/pkg/servers/iosnapshot/iosnapshot_suite_test.go → projects/gateway2/admin/admin_suite_test.go

@@ -1,4 +1,4 @@
-package iosnapshot
+package admin
 
 import (
 	"testing"
@@ -7,7 +7,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestIoSnapshot(t *testing.T) {
+func TestAdmin(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "IOSnapshot Suite")
+	RunSpecs(t, "Admin Suite")
 }

projects/gloo/pkg/servers/iosnapshot/redact.go → projects/gateway2/admin/redact.go

@@ -1,7 +1,6 @@
-package iosnapshot
+package admin
 
 import (
-	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -34,24 +33,6 @@ func redactKubeSecretData(obj client.Object) {
 	redactAnnotations(secret.GetAnnotations())
 }
 
-// redactGlooSecretData modifies the secret to remove any sensitive information
-// The structure of a Secret in Gloo Gateway does not lend itself to easily redact data in different places.
-// As a result, we perform a primitive redaction method, where we maintain the metadata, and remove the entire spec
-func redactGlooSecretData(element *gloov1.Secret) {
-	element.Kind = nil
-
-	redactAnnotations(element.GetMetadata().GetAnnotations())
-}
-
-// redactGlooArtifactData modifies the artifact to remove any sensitive information
-func redactGlooArtifactData(element *gloov1.Artifact) {
-	for k := range element.GetData() {
-		element.GetData()[k] = redactedString
-	}
-
-	redactAnnotations(element.GetMetadata().GetAnnotations())
-}
-
 // redactGlooResourceMetadata modifies the metadata to remove any sensitive information
 // ref: https://github.com/solo-io/skv2/blob/1583cb716c04eb3f8d01ecb179b0deeabaa6e42b/contrib/pkg/snapshot/redact.go#L20-L26
 func redactAnnotations(annotations map[string]string) {

projects/gloo/pkg/servers/iosnapshot/resources.go → projects/gateway2/admin/resources.go

@@ -1,4 +1,4 @@
-package iosnapshot
+package admin
 
 import (
 	"slices"
@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
+// TODO: these need to be updated
 var (
 	KubernetesCoreGVKs = []schema.GroupVersionKind{
 		wellknownkube.SecretGVK,
@@ -33,14 +34,6 @@ var (
 		gatewayv1.RouteOptionGVK,
 	}
 
-	EdgeGatewayGVKs = []schema.GroupVersionKind{
-		gatewayv1.GatewayGVK,
-		gatewayv1.MatchableHttpGatewayGVK,
-		gatewayv1.MatchableTcpGatewayGVK,
-		gatewayv1.VirtualServiceGVK,
-		gatewayv1.RouteTableGVK,
-	}
-
 	KubernetesGatewayGVKs = []schema.GroupVersionKind{
 		wellknown.GatewayClassGVK,
 		wellknown.GatewayGVK,
@@ -56,16 +49,11 @@ var (
 		gatewayv1.HttpListenerOptionGVK,
 	}
 
-	EdgeOnlyInputSnapshotGVKs = slices.Concat(
+	// CompleteInputSnapshotGVKs is the list of GVKs that will be returned by the InputSnapshot API
+	CompleteInputSnapshotGVKs = slices.Concat(
 		KubernetesCoreGVKs,
 		GlooGVKs,
 		PolicyGVKs,
-		EdgeGatewayGVKs,
-	)
-
-	// CompleteInputSnapshotGVKs is the list of GVKs that will be returned by the InputSnapshot API
-	CompleteInputSnapshotGVKs = slices.Concat(
-		EdgeOnlyInputSnapshotGVKs,
 		KubernetesGatewayGVKs,
 		KubernetesGatewayIntegrationPolicyGVKs,
 	)

projects/gloo/pkg/servers/iosnapshot/response.go → projects/gateway2/admin/response.go

@@ -1,8 +1,12 @@
-package iosnapshot
+package admin
 
 import (
+	"cmp"
 	"encoding/json"
 	"fmt"
+	"slices"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // SnapshotResponseData is the data that is returned by Getter methods on the History object
@@ -76,6 +80,18 @@ func formatOutput(format OutputFormat, genericOutput interface{}) ([]byte, error
 	}
 }
 
+// sortResources sorts resources by gvk, namespace, and name
+func sortResources(resources []client.Object) {
+	slices.SortStableFunc(resources, func(a, b client.Object) int {
+		return cmp.Or(
+			cmp.Compare(a.GetObjectKind().GroupVersionKind().Version, b.GetObjectKind().GroupVersionKind().Version),
+			cmp.Compare(a.GetObjectKind().GroupVersionKind().Kind, b.GetObjectKind().GroupVersionKind().Kind),
+			cmp.Compare(a.GetNamespace(), b.GetNamespace()),
+			cmp.Compare(a.GetName(), b.GetName()),
+		)
+	})
+}
+
 func completeSnapshotResponse(data interface{}) SnapshotResponseData {
 	return SnapshotResponseData{
 		Data:  data,

projects/gloo/pkg/servers/iosnapshot/response_test.go → projects/gateway2/admin/response_test.go

@@ -1,34 +1,35 @@
-package iosnapshot
+package admin_test
 
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rotisserie/eris"
+	"github.com/solo-io/gloo/projects/gateway2/admin"
 	crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("SnapshotResponseData", func() {
 
 	DescribeTable("MarshalJSONString",
-		func(response SnapshotResponseData, expectedString string) {
+		func(response admin.SnapshotResponseData, expectedString string) {
 			responseStr := response.MarshalJSONString()
 			Expect(responseStr).To(Equal(expectedString))
 		},
 		Entry("successful response can be formatted as json",
-			SnapshotResponseData{
+			admin.SnapshotResponseData{
 				Data:  "my data",
 				Error: nil,
 			},
 			"{\"data\":\"my data\",\"error\":\"\"}"),
 		Entry("errored response can be formatted as json",
-			SnapshotResponseData{
+			admin.SnapshotResponseData{
 				Data:  "",
 				Error: eris.New("one error"),
 			},
 			"{\"data\":\"\",\"error\":\"one error\"}"),
 		Entry("CR list can be formatted as json",
-			SnapshotResponseData{
+			admin.SnapshotResponseData{
 				Data: []crdv1.Resource{
 					{
 						ObjectMeta: metav1.ObjectMeta{

projects/gloo/pkg/servers/admin/server.go → projects/gateway2/admin/server.go

@@ -8,50 +8,60 @@ import (
 	"net/http"
 	"sort"
 
-	"github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot"
+	"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
+	envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
+	"github.com/rotisserie/eris"
+	"github.com/solo-io/gloo/projects/gateway2/controller"
 	"github.com/solo-io/go-utils/contextutils"
+	"github.com/solo-io/go-utils/stats"
 	"istio.io/istio/pkg/kube/krt"
 )
 
 const (
 	AdminPort = 9095
 )
 
-// ServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux
+func RunAdminServer(ctx context.Context, setupOpts *controller.SetupOpts) error {
+	// serverHandlers defines the custom handlers that the Admin Server will support
+	serverHandlers := getServerHandlers(ctx, setupOpts.KrtDebugger, setupOpts.Cache)
+
+	stats.StartCancellableStatsServerWithPort(ctx, stats.DefaultStartupOptions(), func(mux *http.ServeMux, profiles map[string]string) {
+		// let people know these moved
+		profiles[fmt.Sprintf("http://localhost:%d/snapshots/", AdminPort)] = fmt.Sprintf("To see snapshots, port forward to port %d", AdminPort)
+	})
+	startHandlers(ctx, serverHandlers)
+
+	return nil
+}
+
+// getServerHandlers returns the custom handlers for the Admin Server, which will be bound to the http.ServeMux
 // These endpoints serve as the basis for an Admin Interface for the Control Plane (https://github.com/solo-io/gloo/issues/6494)
-func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.DebugHandler) func(mux *http.ServeMux, profiles map[string]string) {
+func getServerHandlers(ctx context.Context, dbg *krt.DebugHandler, cache envoycache.SnapshotCache) func(mux *http.ServeMux, profiles map[string]string) {
 	return func(m *http.ServeMux, profiles map[string]string) {
 
-		// The Input Snapshot is intended to return a list of resources that are persisted in the Kubernetes DB, etcD
-		m.HandleFunc("/snapshots/input", func(w http.ResponseWriter, request *http.Request) {
-			response := history.GetInputSnapshot(ctx)
-			respondJson(w, response)
-		})
-		profiles["/snapshots/input"] = "Input Snapshot"
-
-		// The Edge Snapshot is intended to return a representation of the ApiSnapshot object that the Control Plane
-		// manages internally. This is not intended to be consumed by users, but instead be a mechanism to feed this
-		// data into future unit tests
-		m.HandleFunc("/snapshots/edge", func(w http.ResponseWriter, request *http.Request) {
-			response := history.GetEdgeApiSnapshot(ctx)
-			respondJson(w, response)
-		})
-		profiles["/snapshots/edge"] = "Edge Snapshot"
-
-		// The Proxy Snapshot is intended to return a representation of the Proxies within the ApiSnapshot object.
-		// Proxies may either be persisted in etcD or in-memory, so this Api provides a single mechansim to access
-		// these resources.
-		m.HandleFunc("/snapshots/proxies", func(w http.ResponseWriter, r *http.Request) {
-			response := history.GetProxySnapshot(ctx)
-			respondJson(w, response)
-		})
-		profiles["/snapshots/proxies"] = "Proxy Snapshot"
+		/*
+			// The Input Snapshot is intended to return a list of resources that are persisted in the Kubernetes DB, etcD
+			m.HandleFunc("/snapshots/input", func(w http.ResponseWriter, request *http.Request) {
+				response := history.GetInputSnapshot(ctx)
+				respondJson(w, response)
+			})
+			profiles["/snapshots/input"] = "Input Snapshot"
+
+			// The Proxy Snapshot is intended to return a representation of the Proxies within the ApiSnapshot object.
+			// Proxies may either be persisted in etcD or in-memory, so this Api provides a single mechansim to access
+			// these resources.
+			m.HandleFunc("/snapshots/proxies", func(w http.ResponseWriter, r *http.Request) {
+				response := history.GetProxySnapshot(ctx)
+				respondJson(w, response)
+			})
+			profiles["/snapshots/proxies"] = "Proxy Snapshot"
+		*/
 
 		// The xDS Snapshot is intended to return the full in-memory xDS cache that the Control Plane manages
 		// and serves up to running proxies.
 		m.HandleFunc("/snapshots/xds", func(w http.ResponseWriter, r *http.Request) {
-			response := history.GetXdsSnapshot(ctx)
-			respondJson(w, response)
+			response := getXdsSnapshotDataFromCache(cache)
+			writeJSON(w, response, r)
 		})
 		profiles["/snapshots/xds"] = "XDS Snapshot"
 
@@ -62,7 +72,7 @@ func ServerHandlers(ctx context.Context, history iosnapshot.History, dbg *krt.De
 	}
 }
 
-func respondJson(w http.ResponseWriter, response iosnapshot.SnapshotResponseData) {
+func respondJson(w http.ResponseWriter, response SnapshotResponseData) {
 	w.Header().Set("Content-Type", getContentType("json"))
 
 	_, _ = fmt.Fprintf(w, "%+v", response.MarshalJSONString())
@@ -106,7 +116,7 @@ func writeJSON(w http.ResponseWriter, obj any, req *http.Request) {
 	}
 }
 
-func StartHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error {
+func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux, profiles map[string]string)) error {
 	mux := new(http.ServeMux)
 	profileDescriptions := map[string]string{}
 	for _, addHandler := range addHandlers {
@@ -171,3 +181,28 @@ func index(profileDescriptions map[string]string) func(w http.ResponseWriter, r
 		w.Write(buf.Bytes())
 	}
 }
+
+func getXdsSnapshotDataFromCache(xdsCache cache.SnapshotCache) SnapshotResponseData {
+	cacheKeys := xdsCache.GetStatusKeys()
+	cacheEntries := make(map[string]interface{}, len(cacheKeys))
+
+	for _, k := range cacheKeys {
+		xdsSnapshot, err := getXdsSnapshot(xdsCache, k)
+		if err != nil {
+			cacheEntries[k] = err.Error()
+		} else {
+			cacheEntries[k] = xdsSnapshot
+		}
+	}
+
+	return completeSnapshotResponse(cacheEntries)
+}
+
+func getXdsSnapshot(xdsCache cache.SnapshotCache, k string) (cache cache.ResourceSnapshot, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = eris.New(fmt.Sprintf("panic occurred while getting xds snapshot: %v", r))
+		}
+	}()
+	return xdsCache.GetSnapshot(k)
+}

projects/gateway2/setup/ggv2setup.go

@@ -12,6 +12,7 @@ import (
 	"github.com/solo-io/gloo/pkg/utils/kubeutils"
 	"github.com/solo-io/gloo/pkg/utils/namespaces"
 	"github.com/solo-io/gloo/pkg/utils/setuputils"
+	"github.com/solo-io/gloo/projects/gateway2/admin"
 	"github.com/solo-io/gloo/projects/gateway2/controller"
 	extensionsplug "github.com/solo-io/gloo/projects/gateway2/extensions2/plugin"
 	"github.com/solo-io/gloo/projects/gateway2/krtcollections"
@@ -171,6 +172,9 @@ func StartGGv2WithConfig(ctx context.Context, setupOpts *controller.SetupOpts,
 	kubeClient.RunAndWait(ctx.Done())
 	setting.Synced().WaitUntilSynced(ctx.Done())
 
+	logger.Info("starting admin server")
+	go admin.RunAdminServer(ctx, setupOpts)
+
 	logger.Info("starting controller")
 	return c.Start(ctx)
 }

projects/gateway2/setup/ggv2setup_test.go

@@ -288,10 +288,8 @@ func testScenario(t *testing.T, ctx context.Context, kdbg *krt.DebugHandler,
 
 	dump := dumper.Dump(t, ctx)
 	if len(dump.Listeners) == 0 {
-		//		xdsDump := iosnapshot.GetXdsSnapshotDataFromCache(snapCache).MarshalJSONString()
 		j, _ := kdbg.MarshalJSON()
 		t.Logf("timed out waiting - krt state for test: %s %s", t.Name(), string(j))
-		//		t.Logf("timed out waiting - xds state for test: %s %s", t.Name(), xdsDump)
 		t.Fatalf("timed out waiting for listeners")
 	}
 	if write {