From 741464de88532c290af0c6cb7a5baf16970bddb7 Mon Sep 17 00:00:00 2001 From: Mike Burgess Date: Mon, 3 Feb 2025 16:01:51 +0000 Subject: [PATCH 1/9] Dashboard UI sending `changed_input` field at wrong level in `input_changed` event. Fixes #708. --- ui/dashboard/src/hooks/useDashboardExecution.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/dashboard/src/hooks/useDashboardExecution.tsx b/ui/dashboard/src/hooks/useDashboardExecution.tsx index bdb10c1c..605e814e 100644 --- a/ui/dashboard/src/hooks/useDashboardExecution.tsx +++ b/ui/dashboard/src/hooks/useDashboardExecution.tsx @@ -258,7 +258,7 @@ export const DashboardExecutionProvider = ({ selectedDashboardRef.current === dashboardFullName ) { dashboardMessage.action = SocketActions.INPUT_CHANGED; - dashboardMessage.changed_input = lastChangedInput; + dashboardMessage.payload.changed_input = lastChangedInput; sendMessage(dashboardMessage); } else { selectedDashboardRef.current = dashboard.full_name; From 99d92b7359c0055904489e19120869bfe0c80a1a Mon Sep 17 00:00:00 2001 From: Mike Burgess Date: Mon, 3 Feb 2025 19:28:12 +0000 Subject: [PATCH 2/9] Improve search path button config popover to handle narrower screens. Fixes #711. --- .../SearchPath/ManageSearchPathButton.tsx | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/ui/dashboard/src/components/dashboards/SearchPath/ManageSearchPathButton.tsx b/ui/dashboard/src/components/dashboards/SearchPath/ManageSearchPathButton.tsx index 1ec86d3c..b7876f86 100644 --- a/ui/dashboard/src/components/dashboards/SearchPath/ManageSearchPathButton.tsx +++ b/ui/dashboard/src/components/dashboards/SearchPath/ManageSearchPathButton.tsx @@ -2,14 +2,17 @@ import Badge from "@powerpipe/components/Badge"; import Icon from "@powerpipe/components/Icon"; import NeutralButton from "@powerpipe/components/forms/NeutralButton"; import SearchPathConfig from "@powerpipe/components/dashboards/SearchPath/SearchPathConfig"; +import { createPortal } from "react-dom"; import { DashboardDataModeCLISnapshot, DashboardDataModeCloudSnapshot, } from "@powerpipe/types"; import { forwardRef, useEffect, useState } from "react"; import { Popover } from "@headlessui/react"; +import { ThemeProvider, ThemeWrapper } from "@powerpipe/hooks/useTheme"; import { useDashboardSearchPath } from "@powerpipe/hooks/useDashboardSearchPath"; import { useDashboardState } from "@powerpipe/hooks/useDashboardState"; +import { usePopper } from "react-popper"; const PopoverButton = forwardRef((props, ref) => { const { metadata, dashboardsMetadata, selectedDashboard } = @@ -67,6 +70,19 @@ const PopoverButton = forwardRef((props, ref) => { }); const ManageSearchPathButton = () => { + const [popperElement, setPopperElement] = useState(null); + const [referenceElement, setReferenceElement] = useState(null); + const { styles, attributes } = usePopper(referenceElement, popperElement, { + placement: "bottom-start", + modifiers: [ + { + name: "flip", + options: { + fallbackPlacements: ["bottom-end"], + }, + }, + ], + }); const { dashboard, dashboardsMetadata, dataMode, metadata } = useDashboardState(); const { searchPathPrefix } = useDashboardSearchPath(); @@ -90,28 +106,41 @@ const ManageSearchPathButton = () => { } else { setShow(!!metadata?.supports_search_path); } - }, [ - metadata?.supports_search_path, - dashboard, - dashboardsMetadata, - dataMode, - searchPathPrefix, - ]); + }, [metadata, dashboard, dashboardsMetadata, dataMode, searchPathPrefix]); return show ? ( - + {({ close }) => ( -
- -
+ <> + {createPortal( + + +
e.stopPropagation()} + > +
+ +
+
+ + , + // @ts-ignore as this element definitely exists + document.getElementById("portals"), + )} + )} From dccef88067b01c7708046ddc1e6b01ff3ee032e9 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 4 Feb 2025 10:54:02 +0000 Subject: [PATCH 3/9] Fix backend support if the database is specified by a connection string. Closes #713 * Ensure GetDefaultDatabaseConfig is only called once * Update GetDefaultDatabaseConfig to accept mod * remove DefaultDatabase global * dashboardServer and dashboardExecutor store defaultDatabase and defaultSearchPathConfig * Update backendSupport.setFromDb to support ConnectionString * log errors from dashboard execution --- go.mod | 2 +- go.sum | 4 +- internal/cmd/mod.go | 6 +- internal/cmd/server.go | 2 +- .../dashboard_execution_tree.go | 11 +- internal/dashboardexecute/executor.go | 15 ++- internal/dashboardserver/backend_support.go | 110 ++++++++++++++++++ internal/dashboardserver/payload.go | 74 ++---------- internal/dashboardserver/server.go | 50 +++++--- internal/db_client/database_config.go | 33 ++++-- internal/db_client/default_database.go | 6 - internal/initialisation/init_data.go | 24 ++-- internal/version/version.json | 2 +- 13 files changed, 203 insertions(+), 136 deletions(-) create mode 100644 internal/dashboardserver/backend_support.go delete mode 100644 internal/db_client/default_database.go diff --git a/go.mod b/go.mod index 3305d2c2..94abd105 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stevenle/topsort v0.2.0 github.com/turbot/go-kit v0.10.0-rc.0 - github.com/turbot/pipe-fittings/v2 v2.0.0-rc.2 + github.com/turbot/pipe-fittings/v2 v2.0.1-rc.0 github.com/turbot/steampipe-plugin-sdk/v5 v5.11.0 github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 // indirect github.com/xlab/treeprint v1.2.0 // indirect diff --git a/go.sum b/go.sum index 4b729556..94aa89cb 100644 --- a/go.sum +++ b/go.sum @@ -826,8 +826,8 @@ github.com/turbot/go-kit v0.10.0-rc.0 h1:kd+jp2ibbIV33Hc8SsMAN410Dl9Pz6SJ40axbKU github.com/turbot/go-kit v0.10.0-rc.0/go.mod h1:fFQqR59I5z5JeeBLfK1PjSifn4Oprs3NiQx0CxeSJxs= github.com/turbot/pipe-fittings v1.6.0 h1:sn4OJPQQR0fggKMNUuyvShJipZG/Ze5jR48Hl12Eokg= github.com/turbot/pipe-fittings v1.6.0/go.mod h1:1nlRVh18QkYy9eq5pW9gpnoE2VgnQW0Y2zKzrH8Q4kI= -github.com/turbot/pipe-fittings/v2 v2.0.0-rc.2 h1:0nUVueTEbgPSqhff/6DCCgUaAjkZHvHdKOf/PtoeyRE= -github.com/turbot/pipe-fittings/v2 v2.0.0-rc.2/go.mod h1:4srBITqYs/VWpgpqjQmJejJ3akkPVRueCHQa9CiETr0= +github.com/turbot/pipe-fittings/v2 v2.0.1-rc.0 h1:z11uypxjxxoFfltbNPJFC3Ub5HwoVtViPFB28T1ZS0g= +github.com/turbot/pipe-fittings/v2 v2.0.1-rc.0/go.mod h1:4srBITqYs/VWpgpqjQmJejJ3akkPVRueCHQa9CiETr0= github.com/turbot/pipes-sdk-go v0.9.1 h1:2yRojY2wymvJn6NQyE6A0EDFV267MNe+yDLxPVvsBwM= github.com/turbot/pipes-sdk-go v0.9.1/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/steampipe-plugin-code v0.7.0 h1:SROYIo/TI/Q/YNfXK+sAIS71umypUFm1Uz851TmoJkM= diff --git a/internal/cmd/mod.go b/internal/cmd/mod.go index 9241b92a..51027387 100644 --- a/internal/cmd/mod.go +++ b/internal/cmd/mod.go @@ -144,7 +144,7 @@ func runModInstallCmd(cmd *cobra.Command, args []string) { // if any mod names were passed as args, convert into formed mod names installOpts := modinstaller.NewInstallOpts(workspaceMod, args...) - installOpts.PluginVersions = getPluginVersions(ctx) + installOpts.PluginVersions = getPluginVersions(ctx, workspaceMod) installData, err := modinstaller.InstallWorkspaceDependencies(ctx, installOpts) if err != nil { @@ -162,8 +162,8 @@ func validateModArgs() error { return localcmdconfig.ValidateDatabaseArg() } -func getPluginVersions(ctx context.Context) *plugin.PluginVersionMap { - defaultDatabase, _, err := db_client.GetDefaultDatabaseConfig() +func getPluginVersions(ctx context.Context, workspaceMod *modconfig.Mod) *plugin.PluginVersionMap { + defaultDatabase, _, err := db_client.GetDefaultDatabaseConfig(workspaceMod) if err != nil { if !viper.GetBool(constants.ArgForce) { error_helpers.ShowWarning("Could not connect to database - plugin validation will not be performed") diff --git a/internal/cmd/server.go b/internal/cmd/server.go index c08e8df2..bdb088c8 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -85,7 +85,7 @@ func runServerCmd(cmd *cobra.Command, _ []string) { // setup a new webSocket service webSocket := melody.New() // create the dashboardServer - dashboardServer, err := dashboardserver.NewServer(ctx, modInitData.Workspace, webSocket) + dashboardServer, err := dashboardserver.NewServer(ctx, modInitData, webSocket) error_helpers.FailOnError(err) // send it over to the powerpipe API Server diff --git a/internal/dashboardexecute/dashboard_execution_tree.go b/internal/dashboardexecute/dashboard_execution_tree.go index 27ce3c5e..9e02bf50 100644 --- a/internal/dashboardexecute/dashboard_execution_tree.go +++ b/internal/dashboardexecute/dashboard_execution_tree.go @@ -49,12 +49,12 @@ type DashboardExecutionTree struct { DateTimeRange utils.TimeRange } -func newDashboardExecutionTree(rootResource modconfig.ModTreeItem, sessionId string, workspace *workspace.PowerpipeWorkspace, inputs *InputValues, defaultClientMap *db_client.ClientMap, opts ...backend.BackendOption) (*DashboardExecutionTree, error) { +func (e *DashboardExecutor) newDashboardExecutionTree(rootResource modconfig.ModTreeItem, sessionId string, workspace *workspace.PowerpipeWorkspace, inputs *InputValues, opts ...backend.BackendOption) (*DashboardExecutionTree, error) { // now populate the DashboardExecutionTree executionTree := &DashboardExecutionTree{ dashboardName: rootResource.Name(), sessionId: sessionId, - defaultClientMap: defaultClientMap, + defaultClientMap: e.defaultClient, clientMap: db_client.NewClientMap(), runs: make(map[string]dashboardtypes.DashboardTreeRun), workspace: workspace, @@ -63,12 +63,7 @@ func newDashboardExecutionTree(rootResource modconfig.ModTreeItem, sessionId str } executionTree.id = fmt.Sprintf("%p", executionTree) - // set the dashboard database and search patch config - defaultDatabase, defaultSearchPathConfig, err := db_client.GetDefaultDatabaseConfig(opts...) - if err != nil { - return nil, err - } - database, searchPathConfig, err := db_client.GetDatabaseConfigForResource(rootResource, workspace.Mod, defaultDatabase, defaultSearchPathConfig) + database, searchPathConfig, err := db_client.GetDatabaseConfigForResource(rootResource, workspace.Mod, e.defaultDatabase, e.defaultSearchPathConfig) if err != nil { return nil, err } diff --git a/internal/dashboardexecute/executor.go b/internal/dashboardexecute/executor.go index d7fd379c..f2ac0a24 100644 --- a/internal/dashboardexecute/executor.go +++ b/internal/dashboardexecute/executor.go @@ -11,6 +11,7 @@ import ( filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/connection" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/powerpipe/internal/dashboardevents" @@ -29,15 +30,19 @@ type DashboardExecutor struct { interactive bool // store the default client which is created during initData creation // - this is to avoid creating a new client for each dashboard execution if the database/search path is NOT overridden - defaultClient *db_client.ClientMap + defaultClient *db_client.ClientMap + defaultDatabase connection.ConnectionStringProvider + defaultSearchPathConfig backend.SearchPathConfig } -func NewDashboardExecutor(defaultClient *db_client.ClientMap) *DashboardExecutor { +func NewDashboardExecutor(defaultClient *db_client.ClientMap, defaultDatabase connection.ConnectionStringProvider, defaultSearchPathConfig backend.SearchPathConfig) *DashboardExecutor { return &DashboardExecutor{ executions: make(map[string]*DashboardExecutionTree), // default to interactive execution - interactive: true, - defaultClient: defaultClient, + interactive: true, + defaultClient: defaultClient, + defaultDatabase: defaultDatabase, + defaultSearchPathConfig: defaultSearchPathConfig, } } @@ -68,7 +73,7 @@ func (e *DashboardExecutor) ExecuteDashboard(ctx context.Context, sessionId stri e.CancelExecutionForSession(ctx, sessionId) // now create a new execution - executionTree, err = newDashboardExecutionTree(rootResource, sessionId, workspace, inputs, e.defaultClient, opts...) + executionTree, err = e.newDashboardExecutionTree(rootResource, sessionId, workspace, inputs, opts...) if err != nil { return err } diff --git a/internal/dashboardserver/backend_support.go b/internal/dashboardserver/backend_support.go new file mode 100644 index 00000000..f8422fc9 --- /dev/null +++ b/internal/dashboardserver/backend_support.go @@ -0,0 +1,110 @@ +package dashboardserver + +import ( + "context" + "log/slog" + + "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/connection" + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/modconfig" +) + +type backendSupport struct { + supportsSearchPath bool + supportsTimeRange bool +} + +// setFromDb sets the backend support based on the database type +func (bs *backendSupport) setFromDb(db connection.ConnectionStringProvider) { + if db != nil { + switch db.(type) { + case *connection.SteampipePgConnection, *connection.PostgresConnection: + bs.supportsSearchPath = true + case *connection.TailpipeConnection: + bs.supportsTimeRange = true + case *connection.ConnectionString: + // create a backend and get its name + // if it is a steampipe or postgres backend, set supportsSearchPath + // NOTE: a tailpipe connection cannot be specified by a connection string + // (as the connection string is dynamic and provided by the Tailpipe CLI), + // so we will not set the supportsTimeRange flag here + + // we do not expect an error from ConnectionString.GetConnectionString + connectionString, _ := db.GetConnectionString() + // get the backend name from the connection string + // NOTE: this does not create the backend and will therefore return postgres for a steampipe backend + // as we cannot tell the difference purely from the connection string + // This is fine as we just want to determine whether a search path is supported + backendName, _ := backend.NameFromConnectionString(context.Background(), connectionString) + // set supportsSearchPath if the backend is a steampipe or postgres backend + bs.supportsSearchPath = backendName == constants.PostgresBackendName + } + } +} + +func newBackendSupport(database connection.ConnectionStringProvider) *backendSupport { + bs := &backendSupport{} + bs.setFromDb(database) + return bs +} + +// determineBackendSupport determines the backend support for a dashboard +// if no resource has a specified database, use the default database to set the backend support +func determineBackendSupport(dashboard modconfig.ModTreeItem, defaultDatabase connection.ConnectionStringProvider) backendSupport { + bs := determineBackendSupportForResource(dashboard) + if bs == nil { + slog.Info("determineBackendSupport - no resource in the tree specifies a database, using default database") + bs = newBackendSupport(defaultDatabase) + } + + slog.Info("determineBackendSupport", "supportsSearchPath", bs.supportsSearchPath, "supportsTimeRange", bs.supportsTimeRange) + return *bs +} + +func determineBackendSupportForResource(item modconfig.ModTreeItem) *backendSupport { + var bs *backendSupport + + // NOT: just check the database on this resource - GetDatabase also checks the parents + // - there is no need to do that here as we are traversing down the tree + if db := item.GetModTreeItemImpl().Database; db != nil { + bs = &backendSupport{} + bs.setFromDb(db) + slog.Info("determineBackendSupportForResource - resource has database", "resource", item.Name(), "backendSupport", bs) + } + + // if we have now set both flags, we can stop - no need to traverse further + if backendSupportsAll(bs) { + return bs + } + + for _, child := range item.GetChildren() { + childBs := determineBackendSupportForResource(child) + // merge this with out ba + bs = mergeBackendSupport(bs, childBs) + + // if we have now set both flags, we can stop - no need to traverse further + if backendSupportsAll(bs) { + return bs + } + } + return bs +} + +func backendSupportsAll(bs *backendSupport) bool { + return bs != nil && bs.supportsSearchPath && bs.supportsTimeRange +} + +// merge 2 backend support objects +func mergeBackendSupport(bs1, bs2 *backendSupport) *backendSupport { + if bs1 == nil { + return bs2 + } + if bs2 == nil { + return bs1 + } + return &backendSupport{ + supportsSearchPath: bs1.supportsSearchPath || bs2.supportsSearchPath, + supportsTimeRange: bs1.supportsTimeRange || bs2.supportsTimeRange, + } +} diff --git a/internal/dashboardserver/payload.go b/internal/dashboardserver/payload.go index 9b04f0c7..7058e3cc 100644 --- a/internal/dashboardserver/payload.go +++ b/internal/dashboardserver/payload.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/spf13/viper" "log/slog" + "github.com/spf13/viper" typeHelpers "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/app_specific" "github.com/turbot/pipe-fittings/v2/backend" - "github.com/turbot/pipe-fittings/v2/connection" "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/steampipeconfig" @@ -23,7 +22,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/sperr" ) -func buildServerMetadataPayload(rm modconfig.ModResources, pipesMetadata *steampipeconfig.PipesMetadata) ([]byte, error) { +func (s *Server) buildServerMetadataPayload(rm modconfig.ModResources, pipesMetadata *steampipeconfig.PipesMetadata) ([]byte, error) { workspaceResources := rm.(*resources.PowerpipeModResources) installedMods := make(map[string]*ModMetadata) for _, mod := range workspaceResources.Mods { @@ -52,14 +51,8 @@ func buildServerMetadataPayload(rm modconfig.ModResources, pipesMetadata *steamp cliVersion = versionFile.Version } - defaultDatabase, defaultSearchPathConfig, err := db_client.GetDefaultDatabaseConfig() - if err != nil { - return nil, err - } - // populate the backend support flags (supportsSearchPath, supportsTimeRange) from the default database - var bs backendSupport - bs.setFromDb(defaultDatabase) + bs := newBackendSupport(s.defaultDatabase) payload := ServerMetadataPayload{ Action: "server_metadata", @@ -74,11 +67,11 @@ func buildServerMetadataPayload(rm modconfig.ModResources, pipesMetadata *steamp }, } - connectionString, err := defaultDatabase.GetConnectionString() + connectionString, err := s.defaultDatabase.GetConnectionString() if err != nil { return nil, err } - searchPath, err := getSearchPathMetadata(context.Background(), connectionString, defaultSearchPathConfig) + searchPath, err := getSearchPathMetadata(context.Background(), connectionString, s.defaultSearchPathConfig) if err != nil { return nil, err } @@ -99,18 +92,12 @@ func buildServerMetadataPayload(rm modconfig.ModResources, pipesMetadata *steamp return json.Marshal(payload) } -func buildDashboardMetadataPayload(dashboard modconfig.ModTreeItem) ([]byte, error) { +func (s *Server) buildDashboardMetadataPayload(dashboard modconfig.ModTreeItem) ([]byte, error) { slog.Debug("calling buildDashboardMetadataPayload") - defaultDatabase, _, err := db_client.GetDefaultDatabaseConfig() - if err != nil { - slog.Warn("error getting database config for resource", "error", err) - return nil, err - } - // walk the tree of resources and determine whether any of them are using a tailpipe/steampipe/postrgres // and set the SupportsSearchPath and SupportsTimeRange flags accordingly - backendSupport := determineBackendSupport(dashboard, defaultDatabase) + backendSupport := determineBackendSupport(dashboard, s.defaultDatabase) payload := DashboardMetadataPayload{ Action: "dashboard_metadata", @@ -128,53 +115,6 @@ func buildDashboardMetadataPayload(dashboard modconfig.ModTreeItem) ([]byte, err return res, nil } -type backendSupport struct { - supportsSearchPath bool - supportsTimeRange bool -} - -func (bs *backendSupport) setFromDb(db connection.ConnectionStringProvider) { - if db != nil { - switch db.(type) { - case *connection.SteampipePgConnection, *connection.PostgresConnection: - bs.supportsSearchPath = true - case *connection.TailpipeConnection: - bs.supportsTimeRange = true - } - } -} -func determineBackendSupport(dashboard modconfig.ModTreeItem, defaultDatabase connection.ConnectionStringProvider) backendSupport { - var res backendSupport - - usingDefaultDb := determineBackendSupportForResource(dashboard, &res) - if usingDefaultDb { - res.setFromDb(defaultDatabase) - } - return res -} - -func determineBackendSupportForResource(item modconfig.ModTreeItem, bs *backendSupport) bool { - var usingDefaultDb bool - db := item.GetDatabase() - if db == nil { - usingDefaultDb = true - } else { - bs.setFromDb(db) - } - - // if we have now found both, we can stop - if bs.supportsSearchPath && bs.supportsTimeRange { - return false - } - for _, child := range item.GetChildren() { - childUsingDefaultDb := determineBackendSupportForResource(child, bs) - if childUsingDefaultDb { - usingDefaultDb = true - } - } - return usingDefaultDb -} - func getSearchPathMetadata(ctx context.Context, database string, searchPathConfig backend.SearchPathConfig) (*SearchPathMetadata, error) { // if backend supports search path, get it client, err := db_client.NewClientMap().GetOrCreate(ctx, database, searchPathConfig) diff --git a/internal/dashboardserver/server.go b/internal/dashboardserver/server.go index 723035d1..8e7bade5 100644 --- a/internal/dashboardserver/server.go +++ b/internal/dashboardserver/server.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/turbot/steampipe-plugin-sdk/v5/sperr" "log/slog" "os" "reflect" @@ -14,35 +13,44 @@ import ( "github.com/turbot/go-kit/helpers" typeHelpers "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/connection" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/schema" "github.com/turbot/pipe-fittings/v2/steampipeconfig" "github.com/turbot/powerpipe/internal/dashboardevents" "github.com/turbot/powerpipe/internal/dashboardexecute" + "github.com/turbot/powerpipe/internal/initialisation" "github.com/turbot/powerpipe/internal/workspace" + "github.com/turbot/steampipe-plugin-sdk/v5/sperr" "gopkg.in/olahol/melody.v1" ) type Server struct { - mutex *sync.Mutex - dashboardClients map[string]*DashboardClientInfo - webSocket *melody.Melody - workspace *workspace.PowerpipeWorkspace + mutex *sync.Mutex + dashboardClients map[string]*DashboardClientInfo + webSocket *melody.Melody + workspace *workspace.PowerpipeWorkspace + defaultDatabase connection.ConnectionStringProvider + defaultSearchPathConfig backend.SearchPathConfig } -func NewServer(ctx context.Context, w *workspace.PowerpipeWorkspace, webSocket *melody.Melody) (*Server, error) { +func NewServer(ctx context.Context, initData *initialisation.InitData, webSocket *melody.Melody) (*Server, error) { OutputWait(ctx, "Starting WorkspaceEvents Server") var dashboardClients = make(map[string]*DashboardClientInfo) var mutex = &sync.Mutex{} + w := initData.Workspace + server := &Server{ - mutex: mutex, - dashboardClients: dashboardClients, - webSocket: webSocket, - workspace: w, + mutex: mutex, + dashboardClients: dashboardClients, + webSocket: webSocket, + workspace: w, + defaultDatabase: initData.DefaultDatabase, + defaultSearchPathConfig: initData.DefaultSearchPathConfig, } w.RegisterDashboardEventHandler(ctx, server.HandleDashboardEvent) @@ -207,7 +215,7 @@ func (s *Server) HandleDashboardEvent(ctx context.Context, event dashboardevents OutputMessage(ctx, "Available Dashboards updated") // Emit dashboard metadata event in case there is a new mod - else the UI won't know about this mod - payload, payloadError = buildServerMetadataPayload(s.workspace.GetModResources(), &steampipeconfig.PipesMetadata{}) + payload, payloadError = s.buildServerMetadataPayload(s.workspace.GetModResources(), &steampipeconfig.PipesMetadata{}) if payloadError != nil { return } @@ -270,7 +278,10 @@ func (s *Server) HandleDashboardEvent(ctx context.Context, event dashboardevents if typeHelpers.SafeString(dashboardClientInfo.Dashboard) == changedDashboardName { if changedResource := s.getResource(changedDashboardName); changedResource != nil { - _ = dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, changedResource, dashboardClientInfo.DashboardInputs, s.workspace) + err := dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, changedResource, dashboardClientInfo.DashboardInputs, s.workspace) + if err != nil { + OutputError(ctx, sperr.WrapWithMessage(err, "error executing dashboard")) + } } } } @@ -292,7 +303,10 @@ func (s *Server) HandleDashboardEvent(ctx context.Context, event dashboardevents for sessionId, dashboardClientInfo := range sessionMap { if typeHelpers.SafeString(dashboardClientInfo.Dashboard) == newDashboardName { if newDashboard := s.getResource(newDashboardName); newDashboard != nil { - _ = dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, newDashboard, dashboardClientInfo.DashboardInputs, s.workspace) + err := dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, newDashboard, dashboardClientInfo.DashboardInputs, s.workspace) + if err != nil { + OutputError(ctx, sperr.WrapWithMessage(err, "error executing dashboard")) + } } } } @@ -351,7 +365,7 @@ func (s *Server) handleMessageFunc(ctx context.Context) func(session *melody.Ses switch request.Action { case "get_server_metadata": - payload, err := buildServerMetadataPayload(s.workspace.GetModResources(), &steampipeconfig.PipesMetadata{}) + payload, err := s.buildServerMetadataPayload(s.workspace.GetModResources(), &steampipeconfig.PipesMetadata{}) if err != nil { OutputError(ctx, sperr.WrapWithMessage(err, "error building payload for get_metadata")) } @@ -379,10 +393,12 @@ func (s *Server) handleMessageFunc(ctx context.Context) func(session *melody.Ses SearchPathPrefix: request.Payload.SearchPathPrefix, })) } - _ = dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, dashboard, inputValues, s.workspace, opts...) - + err := dashboardexecute.Executor.ExecuteDashboard(ctx, sessionId, dashboard, inputValues, s.workspace, opts...) + if err != nil { + OutputError(ctx, sperr.WrapWithMessage(err, "error executing dashboard")) + } slog.Debug("get_dashboard_metadata", "dashboard", request.Payload.Dashboard.FullName) - payload, err := buildDashboardMetadataPayload(dashboard) + payload, err := s.buildDashboardMetadataPayload(dashboard) if err != nil { OutputError(ctx, sperr.WrapWithMessage(err, "error building payload for get_metadata_details")) } diff --git a/internal/db_client/database_config.go b/internal/db_client/database_config.go index a228ff2a..b0dbffb5 100644 --- a/internal/db_client/database_config.go +++ b/internal/db_client/database_config.go @@ -1,6 +1,8 @@ package db_client import ( + "log/slog" + "github.com/spf13/viper" "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/pipe-fittings/v2/connection" @@ -11,6 +13,8 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/sperr" ) +// GetDatabaseConfigForResource returns the ConnectionStringProvider and searchPathConfig for a resource +// if no database is set, use the default database, likewise for search path func GetDatabaseConfigForResource(resource modconfig.ModTreeItem, workspaceMod *modconfig.Mod, defaultDatabase connection.ConnectionStringProvider, defaultSearchPathConfig backend.SearchPathConfig) (connection.ConnectionStringProvider, backend.SearchPathConfig, error) { csp := defaultDatabase searchPathConfig := defaultSearchPathConfig @@ -35,8 +39,7 @@ func GetDatabaseConfigForResource(resource modconfig.ModTreeItem, workspaceMod * // if the mod requirement has a search path, prefix or database, set it in viper, if modRequirement.Database != nil { - // TODO K test/fix setting database in mod require - // if database is overriden, also use overriden search path and search path prefix (even if empty) + // if database is overridden, also use overriden search path and search path prefix (even if empty) csp = connection.NewConnectionString(*modRequirement.Database) searchPathConfig.SearchPath = modRequirement.SearchPath searchPathConfig.SearchPathPrefix = modRequirement.SearchPathPrefix @@ -81,9 +84,9 @@ func GetDatabaseConfigForResource(resource modconfig.ModTreeItem, workspaceMod * return csp, searchPathConfig, nil } -// GetDefaultDatabaseConfig builds the default database and searchPathConfig +// GetDefaultDatabaseConfig returns the default ConnectionStringProvider and searchPathConfig // NOTE: if the dashboardUI has overridden the search path, opts wil be passed in to set the overridden value -func GetDefaultDatabaseConfig(opts ...backend.BackendOption) (connection.ConnectionStringProvider, backend.SearchPathConfig, error) { +func GetDefaultDatabaseConfig(mod *modconfig.Mod, opts ...backend.BackendOption) (connection.ConnectionStringProvider, backend.SearchPathConfig, error) { var cfg backend.BackendConfig for _, opt := range opts { opt(&cfg) @@ -100,11 +103,23 @@ func GetDefaultDatabaseConfig(opts ...backend.BackendOption) (connection.Connect defaultSearchPathConfig = cfg.SearchPathConfig } - // TODO K hack - csp := DefaultDatabase //viper.GetString(constants.ArgDatabase) - + var csp connection.ConnectionStringProvider + // has a database arg been set in viper? + databaseArgs := viper.GetString(constants.ArgDatabase) + modDatabase := mod.GetDatabase() + + switch { + // if database command line was passed, set default + case databaseArgs != "": + slog.Info("GetDefaultDatabaseConfig: Using database connection string from command line", "database arg", databaseArgs) + csp = connection.NewConnectionString(databaseArgs) + case modDatabase != nil: + slog.Info("GetDefaultDatabaseConfig: Using database connection string from mod") + csp = modDatabase // if no database is set, use the default connection - if csp == nil { + default: + slog.Info("GetDefaultDatabaseConfig: Using default connection") + defaultConnection := powerpipeconfig.GlobalConfig.GetDefaultConnection() csp = defaultConnection // if no search path has been set, use the default connection @@ -120,6 +135,8 @@ func GetDefaultDatabaseConfig(opts ...backend.BackendOption) (connection.Connect // if the database is a cloud workspace, resolve the connection string if steampipeconfig.IsPipesWorkspaceConnectionString(csp) { + slog.Info("GetDefaultDatabaseConfig: Resolving Pipes workspace connection string") + cs, err := csp.GetConnectionString() if err != nil { return nil, backend.SearchPathConfig{}, err diff --git a/internal/db_client/default_database.go b/internal/db_client/default_database.go deleted file mode 100644 index cb2a1c15..00000000 --- a/internal/db_client/default_database.go +++ /dev/null @@ -1,6 +0,0 @@ -package db_client - -import "github.com/turbot/pipe-fittings/v2/connection" - -// TODO K HACK -var DefaultDatabase connection.ConnectionStringProvider diff --git a/internal/initialisation/init_data.go b/internal/initialisation/init_data.go index 3d4669f7..19410447 100644 --- a/internal/initialisation/init_data.go +++ b/internal/initialisation/init_data.go @@ -37,6 +37,9 @@ type InitData struct { ExportManager *export.Manager Targets []modconfig.ModTreeItem DefaultClient *db_client.DbClient + + DefaultDatabase connection.ConnectionStringProvider + DefaultSearchPathConfig backend.SearchPathConfig } func NewErrorInitData(err error) *InitData { @@ -80,21 +83,6 @@ func NewInitData[T modconfig.ModTreeItem](ctx context.Context, cmd *cobra.Comman } i.Targets = targets - // TODO K breaking hack do not use viper for database - // this is because DefaultDatabase is now a ConnectionStringProvider - if db_client.DefaultDatabase == nil { - db_client.DefaultDatabase = w.Mod.GetDatabase() - } - // if database command line was passed, set default - // TODO K will we only pass connection string in db arg or could we pass connection name? - if db := viper.GetString(constants.ArgDatabase); db != "" { - db_client.DefaultDatabase = connection.NewConnectionString(db) - } - // if the database is NOT set in viper, and the mod has a connection string, set it - //if !viper.IsSet(constants.ArgDatabase) && w.Mod.GetDatabase() != nil { - // viper.Set(constants.ArgDatabase, *w.Mod.GetDatabase().) - //} - // now do the actual initialisation i.Init(ctx, cmdArgs...) @@ -173,11 +161,13 @@ func (i *InitData) Init(ctx context.Context, args ...string) { // create default client // set the database and search patch config - csp, searchPathConfig, err := db_client.GetDefaultDatabaseConfig() + csp, searchPathConfig, err := db_client.GetDefaultDatabaseConfig(i.Workspace.Mod) if err != nil { i.Result.Error = err return } + i.DefaultDatabase = csp + i.DefaultSearchPathConfig = searchPathConfig // create client var opts []backend.BackendOption @@ -202,7 +192,7 @@ func (i *InitData) Init(ctx context.Context, args ...string) { // create the dashboard executor, passing the default client inside a client map clientMap := db_client.NewClientMap().Add(client, searchPathConfig) - dashboardexecute.Executor = dashboardexecute.NewDashboardExecutor(clientMap) + dashboardexecute.Executor = dashboardexecute.NewDashboardExecutor(clientMap, i.DefaultDatabase, i.DefaultSearchPathConfig) } func validateModRequirementsRecursively(mod *modconfig.Mod, client *db_client.DbClient) []string { diff --git a/internal/version/version.json b/internal/version/version.json index 42c53902..5c2d8d9b 100644 --- a/internal/version/version.json +++ b/internal/version/version.json @@ -1,5 +1,5 @@ { "major": 1, "minor": 2, - "patch": 0 + "patch": 1 } \ No newline at end of file From c8cb727cb99d01bfe7ef612c2bea9a496abfb68f Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 4 Feb 2025 11:11:33 +0000 Subject: [PATCH 4/9] Update CHANGELOG for 1.2.1 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0654de16..64ee27bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v1.2.1 [2025-02-04] + +Fix backend support if the database is specified by a connection string. ([#713](https://github.com/turbot/powerpipe/issues/713)) +Improve search path button config popover to handle narrower screens. ([#711](https://github.com/turbot/powerpipe/issues/711)) +Dashboard UI sending `changed_input` field at wrong level in `input_changed` event. ([#708](https://github.com/turbot/powerpipe/issues/708)) + ## v1.2.0 [2025-01-30] _Whats new_ - Add support for `tailpipe` detections and detection benchmarks. From 7dcdc2348cff14167f92f6925d1704ff6def7c00 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 4 Feb 2025 11:59:37 +0000 Subject: [PATCH 5/9] Update CHANGELOG for 1.2.1 --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64ee27bf..003ccb05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ ## v1.2.1 [2025-02-04] -Fix backend support if the database is specified by a connection string. ([#713](https://github.com/turbot/powerpipe/issues/713)) -Improve search path button config popover to handle narrower screens. ([#711](https://github.com/turbot/powerpipe/issues/711)) -Dashboard UI sending `changed_input` field at wrong level in `input_changed` event. ([#708](https://github.com/turbot/powerpipe/issues/708)) +_Bug Fixes_ +- Fix backend support if the database is specified by a connection string. ([#713](https://github.com/turbot/powerpipe/issues/713)) +- Improve search path button config popover to handle narrower screens. ([#711](https://github.com/turbot/powerpipe/issues/711)) +- Dashboard UI sending `changed_input` field at wrong level in `input_changed` event. ([#708](https://github.com/turbot/powerpipe/issues/708)) ## v1.2.0 [2025-01-30] _Whats new_ From 7ab32e87c218de4fb3df298fd3b57173ff161077 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Tue, 4 Feb 2025 17:51:48 +0530 Subject: [PATCH 6/9] Fix workflow to get the correct PR title --- .github/workflows/02-powerpipe-release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/02-powerpipe-release.yaml b/.github/workflows/02-powerpipe-release.yaml index 3c77fc5f..7b684b88 100644 --- a/.github/workflows/02-powerpipe-release.yaml +++ b/.github/workflows/02-powerpipe-release.yaml @@ -700,11 +700,11 @@ jobs: - name: Fail if PR title does not match with version run: | - if ${{ (steps.pr_title.outputs.PR_TITLE == env.VERSION) }} == 'true';then + if [[ "${{ steps.pr_title.outputs.PR_TITLE }}" == "Powerpipe ${{ env.VERSION }}" ]]; then echo "Correct version" else - echo "Incorrect version" - exit 1 + echo "Incorrect version" + exit 1 fi - name: Merge pull request to update brew formula From 3ff9fa941e5c726e8e8165dc2be48935d9fd0451 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 4 Feb 2025 17:18:12 +0000 Subject: [PATCH 7/9] When DashboardServer executes a dashboard, ensure search path prefix is respected. Closes #717 --- .../dashboardexecute/dashboard_execution_tree.go | 14 +++++++++++++- internal/db_client/database_config.go | 12 +----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/dashboardexecute/dashboard_execution_tree.go b/internal/dashboardexecute/dashboard_execution_tree.go index 9e02bf50..f8183eef 100644 --- a/internal/dashboardexecute/dashboard_execution_tree.go +++ b/internal/dashboardexecute/dashboard_execution_tree.go @@ -63,7 +63,19 @@ func (e *DashboardExecutor) newDashboardExecutionTree(rootResource modconfig.Mod } executionTree.id = fmt.Sprintf("%p", executionTree) - database, searchPathConfig, err := db_client.GetDatabaseConfigForResource(rootResource, workspace.Mod, e.defaultDatabase, e.defaultSearchPathConfig) + // apply options and use to override the default search path + var cfg backend.BackendConfig + for _, opt := range opts { + opt(&cfg) + } + + // has the search path been overridden? + defaultSearchPathConfig := e.defaultSearchPathConfig + if !cfg.SearchPathConfig.Empty() { + defaultSearchPathConfig = cfg.SearchPathConfig + } + + database, searchPathConfig, err := db_client.GetDatabaseConfigForResource(rootResource, workspace.Mod, e.defaultDatabase, defaultSearchPathConfig) if err != nil { return nil, err } diff --git a/internal/db_client/database_config.go b/internal/db_client/database_config.go index b0dbffb5..6d851832 100644 --- a/internal/db_client/database_config.go +++ b/internal/db_client/database_config.go @@ -86,23 +86,13 @@ func GetDatabaseConfigForResource(resource modconfig.ModTreeItem, workspaceMod * // GetDefaultDatabaseConfig returns the default ConnectionStringProvider and searchPathConfig // NOTE: if the dashboardUI has overridden the search path, opts wil be passed in to set the overridden value -func GetDefaultDatabaseConfig(mod *modconfig.Mod, opts ...backend.BackendOption) (connection.ConnectionStringProvider, backend.SearchPathConfig, error) { - var cfg backend.BackendConfig - for _, opt := range opts { - opt(&cfg) - } - +func GetDefaultDatabaseConfig(mod *modconfig.Mod) (connection.ConnectionStringProvider, backend.SearchPathConfig, error) { // resolve the active database and search search path config for the dashboard defaultSearchPathConfig := backend.SearchPathConfig{ SearchPath: viper.GetStringSlice(constants.ArgSearchPath), SearchPathPrefix: viper.GetStringSlice(constants.ArgSearchPathPrefix), } - // has the search path been overridden? - if !cfg.SearchPathConfig.Empty() { - defaultSearchPathConfig = cfg.SearchPathConfig - } - var csp connection.ConnectionStringProvider // has a database arg been set in viper? databaseArgs := viper.GetString(constants.ArgDatabase) From 5725e6ebceaaeea94f4a54893c4c2f165fe06164 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 4 Feb 2025 17:22:01 +0000 Subject: [PATCH 8/9] v1.2.2 --- CHANGELOG.md | 5 +++++ internal/version/version.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 003ccb05..1cac39e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v1.2.2 [tbd] + +_Bug Fixes_ +- When DashboardServer executes a dashboard, ensure search path prefix is respected. ([#717](https://github.com/turbot/powerpipe/issues/717)) + ## v1.2.1 [2025-02-04] _Bug Fixes_ diff --git a/internal/version/version.json b/internal/version/version.json index 5c2d8d9b..af6bd085 100644 --- a/internal/version/version.json +++ b/internal/version/version.json @@ -1,5 +1,5 @@ { "major": 1, "minor": 2, - "patch": 1 + "patch": 2 } \ No newline at end of file From 1b339c3cb54fc98512a00d57255b973d3d6aa473 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Wed, 5 Feb 2025 10:27:42 +0530 Subject: [PATCH 9/9] Update changelog --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cac39e9..76c7f847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,8 @@ -## v1.2.2 [tbd] - +## v1.2.2 [2025-02-05] _Bug Fixes_ - When DashboardServer executes a dashboard, ensure search path prefix is respected. ([#717](https://github.com/turbot/powerpipe/issues/717)) ## v1.2.1 [2025-02-04] - _Bug Fixes_ - Fix backend support if the database is specified by a connection string. ([#713](https://github.com/turbot/powerpipe/issues/713)) - Improve search path button config popover to handle narrower screens. ([#711](https://github.com/turbot/powerpipe/issues/711))