Allow default timezone to be specified on commandline (Velocidex#2388)

This allows users to encode in timezones which are not UTC (whichi is the default) when running the offline collector.
Genio40921 · Jan 21, 2023 · 78affca · 78affca
1 parent 13499cf
commit 78affca
Show file tree

Hide file tree

Showing 72 changed files with 1,245 additions and 1,069 deletions.
diff --git a/actions/events_test.go b/actions/events_test.go
@@ -64,8 +64,8 @@ func (self *EventsTestSuite) SetupTest() {
 	self.ConfigObj.Client.WritebackLinux = self.writeback
 	self.ConfigObj.Client.WritebackWindows = self.writeback
 	self.ConfigObj.Client.WritebackDarwin = self.writeback
-	self.ConfigObj.Frontend.ServerServices.ClientMonitoring = true
-	self.ConfigObj.Frontend.ServerServices.IndexServer = true
+	self.ConfigObj.Services.ClientMonitoring = true
+	self.ConfigObj.Services.IndexServer = true
 	self.TestSuite.SetupTest()
 
 	self.client_id = "C.2232"

diff --git a/api/builder.go b/api/builder.go
@@ -117,7 +117,7 @@ func (self *Builder) withAutoCertFrontendSelfSignedGUI(
 	logger := logging.GetLogger(config_obj, &logging.GUIComponent)
 	logger.Info("Autocert is enabled but GUI port is not 443, starting Frontend with autocert and GUI with self signed.")
 
-	if config_obj.Frontend.ServerServices.GuiServer && config_obj.GUI != nil {
+	if config_obj.Services.GuiServer && config_obj.GUI != nil {
 		mux := http.NewServeMux()
 
 		router, err := PrepareGUIMux(ctx, config_obj, mux)
@@ -136,7 +136,7 @@ func (self *Builder) withAutoCertFrontendSelfSignedGUI(
 		}
 	}
 
-	if !config_obj.Frontend.ServerServices.FrontendServer {
+	if !config_obj.Services.FrontendServer {
 		return nil
 	}
 
@@ -166,7 +166,7 @@ func (self *Builder) WithAutocertGUI(
 
 	mux := http.NewServeMux()
 
-	if self.config_obj.Frontend.ServerServices.FrontendServer {
+	if self.config_obj.Services.FrontendServer {
 		err := server.PrepareFrontendMux(self.config_obj, self.server_obj, mux)
 		if err != nil {
 			return err
@@ -196,7 +196,7 @@ func startSharedSelfSignedFrontend(
 		return errors.New("Frontend not configured")
 	}
 
-	if config_obj.Frontend.ServerServices.FrontendServer {
+	if config_obj.Services.FrontendServer {
 		err := server.PrepareFrontendMux(config_obj, server_obj, mux)
 		if err != nil {
 			return err
@@ -226,13 +226,12 @@ func startSelfSignedFrontend(
 	config_obj *config_proto.Config,
 	server_obj *server.Server) error {
 
-	if config_obj.Frontend == nil ||
-		config_obj.Frontend.ServerServices == nil {
+	if config_obj.Services == nil {
 		return errors.New("Frontend not configured")
 	}
 
 	// Launch a new server for the GUI.
-	if config_obj.Frontend.ServerServices.GuiServer {
+	if config_obj.Services.GuiServer {
 		mux := http.NewServeMux()
 
 		router, err := PrepareGUIMux(ctx, config_obj, mux)
@@ -251,7 +250,7 @@ func startSelfSignedFrontend(
 		}
 	}
 
-	if !config_obj.Frontend.ServerServices.FrontendServer {
+	if !config_obj.Services.FrontendServer {
 		return nil
 	}
 

diff --git a/api/datastore_test.go b/api/datastore_test.go
@@ -39,7 +39,7 @@ func (self *DatastoreAPITest) SetupTest() {
 	assert.NoError(self.T(), err)
 
 	// Now bring up an API server.
-	self.ConfigObj.Frontend.ServerServices = &config_proto.ServerServicesConfig{}
+	self.ConfigObj.Services = &config_proto.ServerServicesConfig{}
 
 	// Wait for the server to come up.
 	vtesting.WaitUntil(2*time.Second, self.T(), func() bool {

diff --git a/api/hunts.go b/api/hunts.go
@@ -72,12 +72,12 @@ func (self *ApiServer) GetHuntFlows(
 			flow.Context.ClientId,
 			services.GetHostname(ctx, org_config_obj, flow.Context.ClientId),
 			flow.Context.SessionId,
-			json.AnyToString(flow.Context.StartTime/1000, vjson.NoEncOpts),
+			json.AnyToString(flow.Context.StartTime/1000, vjson.DefaultEncOpts()),
 			flow.Context.State.String(),
 			json.AnyToString(flow.Context.ExecutionDuration/1000000000,
-				vjson.NoEncOpts),
-			json.AnyToString(flow.Context.TotalUploadedBytes, vjson.NoEncOpts),
-			json.AnyToString(flow.Context.TotalCollectedRows, vjson.NoEncOpts)}
+				vjson.DefaultEncOpts()),
+			json.AnyToString(flow.Context.TotalUploadedBytes, vjson.DefaultEncOpts()),
+			json.AnyToString(flow.Context.TotalCollectedRows, vjson.DefaultEncOpts())}
 
 		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
 

diff --git a/api/upload.go b/api/upload.go
@@ -46,7 +46,7 @@ func toolUploadHandler() http.Handler {
 
 		// Parse our multipart form, 10 << 20 specifies a maximum
 		// upload of 10 MB files.
-		err = r.ParseMultipartForm(10 << 20)
+		err = r.ParseMultipartForm(10 << 25)
 		if err != nil {
 			returnError(w, http.StatusBadRequest, "Unsupported params")
 			return

diff --git a/api/vql.go b/api/vql.go
@@ -70,7 +70,7 @@ func RunVQL(
 				value = ""
 			}
 			new_row.Cell = append(new_row.Cell,
-				json.AnyToString(value, json.NoEncOpts))
+				json.AnyToString(value, json.DefaultEncOpts()))
 		}
 
 		result.Rows = append(result.Rows, new_row)

diff --git a/bin/binary_test.go b/bin/binary_test.go
@@ -451,13 +451,16 @@ func TestShowConfigWithMergePatch(t *testing.T) {
 	// replaces the Nonce With Foo, then adds another server to the
 	// urls: Merges are done first, then patches.
 	cmd := exec.Command(
-		binary, "config", "show", "--config", config_file.Name(),
+		binary, "config", "show", "--config", config_file.Name(), "-v",
 		"--merge",
 		`{"Client": {"nonce": "Foo", "server_urls": ["https://192.168.1.11:8000/"]}}`,
 		"--patch",
 		`[{"op": "add", "path": "/Client/server_urls/0", "value": "https://SomeServer/"}]`,
 	)
 	out, err := cmd.Output()
+	if err != nil {
+		fmt.Println(string(err.(*exec.ExitError).Stderr))
+	}
 	require.NoError(t, err, string(out))
 
 	// Try to load it now.

diff --git a/bin/config.go b/bin/config.go
@@ -132,10 +132,7 @@ func doShowConfig() error {
 		return err
 	}
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()
@@ -161,7 +158,7 @@ func doShowConfig() error {
 	}
 
 	// Hide these fields which are autogenerated
-	config_obj.Frontend.ServerServices = nil
+	config_obj.Services = nil
 
 	if *config_show_command_json {
 		serialized, err := json.Marshal(config_obj)
@@ -391,10 +388,7 @@ func doDumpApiClientConfig() error {
 			"api keys with this name.")
 	}
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()

diff --git a/bin/frontend.go b/bin/frontend.go
@@ -68,11 +68,11 @@ func doFrontend() error {
 
 	// Come up with a suitable services plan depending on the frontend
 	// role.
-	if config_obj.Frontend.ServerServices == nil {
+	if config_obj.Services == nil {
 		if *frontend_cmd_minion {
-			config_obj.Frontend.ServerServices = services.MinionServicesSpec()
+			config_obj.Services = services.MinionServicesSpec()
 		} else {
-			config_obj.Frontend.ServerServices = services.AllServerServicesSpec()
+			config_obj.Services = services.AllServerServicesSpec()
 		}
 	}
 

diff --git a/bin/golden.go b/bin/golden.go
@@ -279,10 +279,7 @@ func doGolden() error {
 
 	failures := []string{}
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	config_obj.Frontend.ServerServices = services.GoldenServicesSpec()
+	config_obj.Services = services.GoldenServicesSpec()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()

diff --git a/bin/grant.go b/bin/grant.go
@@ -8,7 +8,6 @@ import (
 	jsonpatch "github.com/evanphx/json-patch/v5"
 	"www.velocidex.com/golang/velociraptor/acls"
 	acl_proto "www.velocidex.com/golang/velociraptor/acls/proto"
-	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/startup"
@@ -64,10 +63,7 @@ func doGrant() error {
 	ctx, cancel := install_sig_handler()
 	defer cancel()
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	sm, err := startup.StartToolServices(ctx, config_obj)
 	defer sm.Close()

diff --git a/bin/gui.go b/bin/gui.go
@@ -187,11 +187,8 @@ func doGUI() error {
 		fd.Close()
 	}
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	if config_obj.Frontend.ServerServices == nil {
-		config_obj.Frontend.ServerServices = services.AllServerServicesSpec()
+	if config_obj.Services == nil {
+		config_obj.Services = services.AllServerServicesSpec()
 	}
 
 	// Now start the frontend

diff --git a/bin/main.go b/bin/main.go
@@ -126,13 +126,16 @@ func main() {
 
 	// If no args are given check if there is an embedded config
 	// with autoexec.
-	if len(args) == 0 {
+	pre, post := splitArgs(args)
+	if len(pre) == 0 {
 		config_obj, err := new(config.Loader).WithVerbose(*verbose_flag).
 			WithEmbedded().LoadAndValidate()
 		if err == nil && config_obj.Autoexec != nil && config_obj.Autoexec.Argv != nil {
+			args = nil
 			for _, arg := range config_obj.Autoexec.Argv {
 				args = append(args, os.ExpandEnv(arg))
 			}
+			args = append(args, post...)
 			logging.Prelog("Autoexec with parameters: %v", args)
 		}
 	}
@@ -169,6 +172,7 @@ func main() {
 		WithCustomValidator("validator: initFilestoreAccessor",
 			initFilestoreAccessor).
 		WithCustomValidator("validator: initDebugServer", initDebugServer).
+		WithCustomValidator("validator: timezone", initTimezone).
 		WithConfigMutator("Mutator: applyMinionRole", applyMinionRole).
 		WithCustomValidator("validator: applyAnalysisTarget",
 			applyAnalysisTarget).
@@ -214,10 +218,30 @@ func makeDefaultConfigLoader() *config.Loader {
 		WithCustomValidator("validator: initFilestoreAccessor",
 			initFilestoreAccessor).
 		WithCustomValidator("validator: initDebugServer", initDebugServer).
+		WithCustomValidator("validator: timezone", initTimezone).
 		WithLogFile(*logging_flag).
 		WithOverride(*override_flag).
 		WithConfigMutator("Mutator applyMinionRole", applyMinionRole).
 		WithCustomValidator("validator: ensureProxy", ensureProxy).
 		WithConfigMutator("Mutator applyAnalysisTarget", applyAnalysisTarget).
 		WithConfigMutator("Mutator maybeAddDefinitionsDirectory", maybeAddDefinitionsDirectory)
 }
+
+// Split the command line into args before the -- and after the --
+func splitArgs(args []string) (pre, post []string) {
+	seen := false
+	for _, arg := range args {
+		if arg == "--" {
+			seen = true
+			continue
+		}
+
+		if seen {
+			post = append(post, arg)
+		} else {
+			pre = append(pre, arg)
+		}
+	}
+
+	return pre, post
+}
diff --git a/bin/orgs.go b/bin/orgs.go
@@ -37,7 +37,7 @@ func doOrgLs() error {
 	if err != nil {
 		return fmt.Errorf("loading config file: %w", err)
 	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()
@@ -69,7 +69,7 @@ func doOrgUserAdd() error {
 	if err != nil {
 		return fmt.Errorf("loading config file: %w", err)
 	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()
@@ -115,7 +115,7 @@ func doOrgCreate() error {
 		return fmt.Errorf("loading config file: %w", err)
 	}
 
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()
@@ -151,7 +151,7 @@ func doOrgDelete() error {
 		return fmt.Errorf("loading config file: %w", err)
 	}
 
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 
 	ctx, cancel := install_sig_handler()
 	defer cancel()

diff --git a/bin/query.go b/bin/query.go
@@ -127,7 +127,7 @@ func outputCSV(ctx context.Context,
 		10, *max_wait)
 
 	csv_writer := csv.GetCSVAppender(config_obj,
-		scope, &StdoutWrapper{out}, csv.WriteHeaders, json.NoEncOpts)
+		scope, &StdoutWrapper{out}, csv.WriteHeaders, json.DefaultEncOpts())
 	defer csv_writer.Close()
 
 	for result := range result_chan {
@@ -242,7 +242,7 @@ func doRemoteQuery(
 
 			csv_writer := csv.GetCSVAppender(config_obj,
 				scope, &StdoutWrapper{os.Stdout},
-				csv.WriteHeaders, json.NoEncOpts)
+				csv.WriteHeaders, json.DefaultEncOpts())
 			defer csv_writer.Close()
 
 			for _, row := range rows {
@@ -259,14 +259,11 @@ func doQuery() error {
 		return err
 	}
 
-	if config_obj.Frontend == nil {
-		config_obj.Frontend = &config_proto.FrontendConfig{}
-	}
-	config_obj.Frontend.ServerServices = services.GenericToolServices()
+	config_obj.Services = services.GenericToolServices()
 	if config_obj.Datastore != nil && config_obj.Datastore.Location != "" {
-		config_obj.Frontend.ServerServices.IndexServer = true
-		config_obj.Frontend.ServerServices.ClientInfo = true
-		config_obj.Frontend.ServerServices.Label = true
+		config_obj.Services.IndexServer = true
+		config_obj.Services.ClientInfo = true
+		config_obj.Services.Label = true
 	}
 
 	ctx, cancel := install_sig_handler()

diff --git a/bin/server_service_windows.go b/bin/server_service_windows.go
@@ -464,8 +464,8 @@ func NewVelociraptorServerService(name string) (
 				continue
 			}
 
-			if config_obj.Frontend.ServerServices == nil {
-				config_obj.Frontend.ServerServices = services.AllServerServicesSpec()
+			if config_obj.Services == nil {
+				config_obj.Services = services.AllServerServicesSpec()
 			}
 
 			ctx, cancel := install_sig_handler()

diff --git a/bin/timezone.go b/bin/timezone.go
@@ -0,0 +1,18 @@
+package main
+
+import (
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/utils"
+)
+
+var (
+	timezone_flag = app.Flag(
+		"timezone", "Default encoding timezone (e.g. Australia/Brisbane). If not set we use UTC").String()
+)
+
+func initTimezone(config_obj *config_proto.Config) error {
+	if *timezone_flag != "" {
+		return utils.SetGlobalTimezone(*timezone_flag)
+	}
+	return nil
+}