From 13c771eaa26737c93dc9887fa7c96413de182816 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Tue, 4 Feb 2025 23:57:56 -0500 Subject: [PATCH 1/7] Removes chifra monitors --watch --- app/action_daemon.go | 394 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) diff --git a/app/action_daemon.go b/app/action_daemon.go index 8c95f7e..4494337 100644 --- a/app/action_daemon.go +++ b/app/action_daemon.go @@ -184,6 +184,17 @@ func (k *KhedraApp) createChifraConfig() error { return nil } +// For monitor --watch +// 14080,apps,Accounts,monitors,acctExport,watch,w,,visible|docs|notApi,4,switch,,,,,,continually scan for new blocks and extract data as per the command file +// 14090,apps,Accounts,monitors,acctExport,watchlist,a,,visible|docs|notApi,,flag,,,,,,available with --watch option only, a file containing the addresses to watch +// 14100,apps,Accounts,monitors,acctExport,commands,d,,visible|docs|notApi,,flag,,,,,,available with --watch option only, the file containing the list of commands to apply to each watched address +// 14110,apps,Accounts,monitors,acctExport,batch_size,b,8,visible|docs|notApi,,flag,,,,,,available with --watch option only, the number of monitors to process in each batch +// 14120,apps,Accounts,monitors,acctExport,run_count,u,,visible|docs|notApi,,flag,,,,,,available with --watch option only, run the monitor this many times, then quit +// 14130,apps,Accounts,monitors,acctExport,sleep,s,14,visible|docs|notApi,,flag,,,,,,available with --watch option only, the number of seconds to sleep between runs +// 14160,apps,Accounts,monitors,acctExport,n3,,,,,note,,,,,,The --watch option requires two additional parameters to be specified: `--watchlist` and `--commands`. +// 14170,apps,Accounts,monitors,acctExport,n4,,,,,note,,,,,,Addresses provided on the command line are ignored in `--watch` mode. +// 14180,apps,Accounts,monitors,acctExport,n5,,,,,note,,,,,,Providing the value `existing` to the `--watchlist` monitors all existing monitor files (see --list). + var configTmpl string = `[version] current = "{{.Version}}" @@ -206,3 +217,386 @@ var configTmpl string = `[version] symbol = "{{.Symbol}}" {{end -}} ` + +/* + +// HandleWatch starts the monitor watcher +func (opts *MonitorsOptions) HandleWatch(rCtx *output.RenderCtx) error { + opts.Globals.Cache = true + scraper := NewScraper(colors.Magenta, "MonitorScraper", opts.Sleep, 0) + + var wg sync.WaitGroup + wg.Add(1) + // Note that this never returns in normal operation + go opts.RunMonitorScraper(&wg, &scraper) + wg.Wait() + + return nil +} + +// RunMonitorScraper runs continually, never stopping and freshens any existing monitors +func (opts *MonitorsOptions) RunMonitorScraper(wg *sync.WaitGroup, s *Scraper) { + defer wg.Done() + + chain := opts.Globals.Chain + tmpPath := filepath.Join(config.PathToCache(chain), "tmp") + + s.ChangeState(true, tmpPath) + + runCount := uint64(0) + for { + if !s.Running { + s.Pause() + + } else { + monitorList := opts.getMonitorList() + if len(monitorList) == 0 { + logger.Error(validate.Usage("No monitors found. Use 'chifra list' to initialize a monitor.").Error()) + return + } + + if canceled, err := opts.Refresh(monitorList); err != nil { + logger.Error(err) + return + } else { + if canceled { + return + } + } + + runCount++ + if opts.RunCount != 0 && runCount >= opts.RunCount { + return + } + + sleep := opts.Sleep + if sleep > 0 { + ms := time.Duration(sleep*1000) * time.Millisecond + if !opts.Globals.TestMode { + logger.Info(fmt.Sprintf("Sleeping for %g seconds", sleep)) + } + time.Sleep(ms) + } + } + } +} + +type Command struct { + Fmt string `json:"fmt"` + Folder string `json:"folder"` + Cmd string `json:"cmd"` + Cache bool `json:"cache"` +} + +func (c *Command) fileName(addr base.Address) string { + return filepath.Join(c.Folder, addr.Hex()+"."+c.Fmt) +} + +func (c *Command) resolve(addr base.Address, before, after int64) string { + fn := c.fileName(addr) + if file.FileExists(fn) { + if strings.Contains(c.Cmd, "export") { + c.Cmd += fmt.Sprintf(" --first_record %d", uint64(before+1)) + c.Cmd += fmt.Sprintf(" --max_records %d", uint64(after-before+1)) // extra space won't hurt + } else { + c.Cmd += fmt.Sprintf(" %d-%d", before+1, after) + } + c.Cmd += " --append --no_header" + } + c.Cmd = strings.Replace(c.Cmd, " ", " ", -1) + ret := c.Cmd + " --fmt " + c.Fmt + " --output " + c.fileName(addr) + " " + addr.Hex() + if c.Cache { + ret += " --cache" + } + return ret +} + +func (c *Command) String() string { + b, _ := json.MarshalIndent(c, "", " ") + return string(b) +} + +func (opts *MonitorsOptions) Refresh(monitors []monitor.Monitor) (bool, error) { + theCmds, err := opts.getCommands() + if err != nil { + return false, err + } + + batches := batchSlice[monitor.Monitor](monitors, opts.BatchSize) + for i := 0; i < len(batches); i++ { + addrs := []base.Address{} + countsBefore := []int64{} + for _, mon := range batches[i] { + addrs = append(addrs, mon.Address) + countsBefore = append(countsBefore, mon.Count()) + } + + batchSize := int(opts.BatchSize) + fmt.Printf("%s%d-%d of %d:%s chifra export --freshen", + colors.BrightBlue, + i*batchSize, + base.Min(((i+1)*batchSize)-1, len(monitors)), + len(monitors), + colors.Green) + for _, addr := range addrs { + fmt.Printf(" %s", addr.Hex()) + } + fmt.Println(colors.Off) + + canceled, err := opts.FreshenMonitorsForWatch(addrs) + if canceled || err != nil { + return canceled, err + } + + for j := 0; j < len(batches[i]); j++ { + mon := batches[i][j] + countAfter := mon.Count() + + if countAfter > 1000000 { + // TODO: Make this value configurable + fmt.Println(colors.Red, "Too many transactions for address", mon.Address, colors.Off) + continue + } + + if countAfter == 0 { + continue + } + + logger.Info(fmt.Sprintf("Processing item %d in batch %d: %d %d\n", j, i, countsBefore[j], countAfter)) + + for _, cmd := range theCmds { + countBefore := countsBefore[j] + if countBefore == 0 || countAfter > countBefore { + utils.System(cmd.resolve(mon.Address, countBefore, countAfter)) + // o := opts + // o.Globals.File = "" + // _ = o.Globals.PassItOn("acctExport", chain, cmd, []string{}) + } else if opts.Globals.Verbose { + fmt.Println("No new transactions for", mon.Address.Hex(), "since last run.") + } + } + } + } + return false, nil +} + +func batchSlice[T any](slice []T, batchSize uint64) [][]T { + var batches [][]T + for i := 0; i < len(slice); i += int(batchSize) { + end := i + int(batchSize) + if end > len(slice) { + end = len(slice) + } + batches = append(batches, slice[i:end]) + } + return batches +} + +func GetExportFormat(cmd, def string) string { + if strings.Contains(cmd, "json") { + return "json" + } else if strings.Contains(cmd, "txt") { + return "txt" + } else if strings.Contains(cmd, "csv") { + return "csv" + } + if len(def) > 0 { + return def + } + return "csv" +} + +func (opts *MonitorsOptions) cleanLine(lineIn string) (cmd Command, err error) { + line := strings.Replace(lineIn, "[{ADDRESS}]", "", -1) + if strings.Contains(line, "--fmt") { + line = strings.Replace(line, "--fmt", "", -1) + line = strings.Replace(line, "json", "", -1) + line = strings.Replace(line, "csv", "", -1) + line = strings.Replace(line, "txt", "", -1) + } + line = utils.StripComments(line) + if len(line) == 0 { + return Command{}, nil + } + + folder, err := opts.getOutputFolder(line) + if err != nil { + return Command{}, err + } + + _ = file.EstablishFolder(folder) + return Command{Cmd: line, Folder: folder, Fmt: GetExportFormat(lineIn, "csv"), Cache: opts.Globals.Cache}, nil +} + +func (opts *MonitorsOptions) getCommands() (ret []Command, err error) { + lines := file.AsciiFileToLines(opts.Commands) + for _, line := range lines { + // orig := line + if cmd, err := opts.cleanLine(line); err != nil { + return nil, err + } else if len(cmd.Cmd) == 0 { + continue + } else { + ret = append(ret, cmd) + } + } + return ret, nil +} + +func (opts *MonitorsOptions) getOutputFolder(orig string) (string, error) { + okMap := map[string]bool{ + "export": true, + "list": true, + "state": true, + "tokens": true, + } + + cmdLine := orig + parts := strings.Split(strings.Replace(cmdLine, " ", " ", -1), " ") + if len(parts) < 1 || parts[0] != "chifra" { + s := fmt.Sprintf("Invalid command: %s. Must start with 'chifra'.", strings.Trim(orig, " \t\n\r")) + logger.Fatal(s) + } + if len(parts) < 2 || !okMap[parts[1]] { + s := fmt.Sprintf("Invalid command: %s. Must start with 'chifra export', 'chifra list', 'chifra state', or 'chifra tokens'.", orig) + logger.Fatal(s) + } + + cwd, _ := os.Getwd() + cmdLine += " " + folder := "unknown" + if parts[1] == "export" { + if strings.Contains(cmdLine, "-p ") || strings.Contains(cmdLine, "--appearances ") { + folder = filepath.Join(cwd, parts[1], "appearances") + } else if strings.Contains(cmdLine, "-r ") || strings.Contains(cmdLine, "--receipts ") { + folder = filepath.Join(cwd, parts[1], "receipts") + } else if strings.Contains(cmdLine, "-l ") || strings.Contains(cmdLine, "--logs ") { + folder = filepath.Join(cwd, parts[1], "logs") + } else if strings.Contains(cmdLine, "-t ") || strings.Contains(cmdLine, "--traces ") { + folder = filepath.Join(cwd, parts[1], "traces") + } else if strings.Contains(cmdLine, "-n ") || strings.Contains(cmdLine, "--neighbors ") { + folder = filepath.Join(cwd, parts[1], "neighbors") + } else if strings.Contains(cmdLine, "-C ") || strings.Contains(cmdLine, "--accounting ") { + folder = filepath.Join(cwd, parts[1], "accounting") + } else if strings.Contains(cmdLine, "-A ") || strings.Contains(cmdLine, "--statements ") { + folder = filepath.Join(cwd, parts[1], "statements") + } else if strings.Contains(cmdLine, "-b ") || strings.Contains(cmdLine, "--balances ") { + folder = filepath.Join(cwd, parts[1], "balances") + } else { + folder = filepath.Join(cwd, parts[1], "transactions") + } + + } else if parts[1] == "list" { + folder = filepath.Join(cwd, parts[1], "appearances") + + } else if parts[1] == "state" { + if strings.Contains(cmdLine, "-l ") || strings.Contains(cmdLine, "--call ") { + folder = filepath.Join(cwd, parts[1], "calls") + } else { + folder = filepath.Join(cwd, parts[1], "blocks") + } + + } else if parts[1] == "tokens" { + if strings.Contains(cmdLine, "-b ") || strings.Contains(cmdLine, "--by_acct ") { + folder = filepath.Join(cwd, parts[1], "by_acct") + } else { + folder = filepath.Join(cwd, parts[1], "blocks") + } + } + + if strings.Contains(folder, "unknown") { + return "", fmt.Errorf("unable to determine output folder for command: %s", cmdLine) + } + + if abs, err := filepath.Abs(filepath.Join(opts.Globals.Chain, folder)); err != nil { + return "", err + } else { + return abs, nil + } +} + +func (opts *MonitorsOptions) getMonitorList() []monitor.Monitor { + var monitors []monitor.Monitor + + monitorChan := make(chan monitor.Monitor) + go monitor.ListWatchedMonitors(opts.Globals.Chain, opts.Watchlist, monitorChan) + + for result := range monitorChan { + switch result.Address { + case base.NotAMonitor: + logger.Info(fmt.Sprintf("Loaded %d monitors", len(monitors))) + close(monitorChan) + default: + if result.Count() > 500000 { + logger.Warn("Ignoring too-large address", result.Address) + continue + } + monitors = append(monitors, result) + } + } + + return monitors +} + + if opts.Watch { + if opts.Globals.IsApiMode() { + return validate.Usage("The {0} options is not available from the API", "--watch") + } + + if len(opts.Globals.File) > 0 { + return validate.Usage("The {0} option is not allowed with the {1} option. Use {2} instead.", "--file", "--watch", "--commands") + } + + if len(opts.Commands) == 0 { + return validate.Usage("The {0} option requires {1}.", "--watch", "a --commands file") + } else { + cmdFile, err := filepath.Abs(opts.Commands) + if err != nil || !file.FileExists(cmdFile) { + return validate.Usage("The {0} option requires {1} to exist.", "--watch", opts.Commands) + } + if file.FileSize(cmdFile) == 0 { + logger.Fatal(validate.Usage("The file you specified ({0}) was found but contained no commands.", cmdFile).Error()) + } + } + + if len(opts.Watchlist) == 0 { + return validate.Usage("The {0} option requires {1}.", "--watch", "a --watchlist file") + } else { + if opts.Watchlist != "existing" { + watchList, err := filepath.Abs(opts.Watchlist) + if err != nil || !file.FileExists(watchList) { + return validate.Usage("The {0} option requires {1} to exist.", "--watch", opts.Watchlist) + } + if file.FileSize(watchList) == 0 { + logger.Fatal(validate.Usage("The file you specified ({0}) was found but contained no addresses.", watchList).Error()) + } + } + } + + if err := index.IsInitialized(chain, config.ExpectedVersion()); err != nil { + if (errors.Is(err, index.ErrNotInitialized) || errors.Is(err, index.ErrIncorrectHash)) && !opts.Globals.IsApiMode() { + logger.Fatal(err) + } + return err + } + + if opts.BatchSize < 1 { + return validate.Usage("The {0} option must be greater than zero.", "--batch_size") + } + } else { + + if opts.BatchSize != 8 { + return validate.Usage("The {0} option is not available{1}.", "--batch_size", " without --watch") + } else { + opts.BatchSize = 0 + } + + if opts.RunCount > 0 { + return validate.Usage("The {0} option is not available{1}.", "--run_count", " without --watch") + } + + if opts.Sleep != 14 { + return validate.Usage("The {0} option is not available{1}.", "--sleep", " without --watch") + } + +*/ From cd9a0b605ec676f38fe6a23f815fa508bc87838c Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Wed, 5 Feb 2025 00:05:02 -0500 Subject: [PATCH 2/7] Cleans khedra a bit, adds tests --- app/action_daemon.go | 26 ++- app/app.go | 48 +++-- app/config_helpers_test.go | 150 +++++++------- pkg/types/config_descriptors.go | 119 ++--------- pkg/types/logging_test.go | 349 ++++++++++++++++---------------- pkg/utils/download_and_store.go | 52 ++++- 6 files changed, 370 insertions(+), 374 deletions(-) diff --git a/app/action_daemon.go b/app/action_daemon.go index 4494337..b46e3a0 100644 --- a/app/action_daemon.go +++ b/app/action_daemon.go @@ -155,11 +155,9 @@ func (k *KhedraApp) createChifraConfig() error { chainStr := k.config.EnabledChains() chains := strings.Split(chainStr, ",") for _, chain := range chains { - chainConfig := filepath.Join(k.config.General.DataFolder, "config", chain) - if err := file.EstablishFolder(chainConfig); err != nil { + if err := k.createChainConfig(chain); err != nil { return err } - k.logger.Progress("Creating chain config", "chainConfig", chainConfig) } tmpl, err := template.New("tmpl").Parse(configTmpl) @@ -195,6 +193,28 @@ func (k *KhedraApp) createChifraConfig() error { // 14170,apps,Accounts,monitors,acctExport,n4,,,,,note,,,,,,Addresses provided on the command line are ignored in `--watch` mode. // 14180,apps,Accounts,monitors,acctExport,n5,,,,,note,,,,,,Providing the value `existing` to the `--watchlist` monitors all existing monitor files (see --list). +func (k *KhedraApp) createChainConfig(chain string) error { + chainConfig := filepath.Join(k.config.General.DataFolder, "config", chain) + if err := file.EstablishFolder(chainConfig); err != nil { + return fmt.Errorf("failed to create folder %s: %w", chainConfig, err) + } + + k.logger.Progress("Creating chain config", "chainConfig", chainConfig) + + // baseURL := "https://raw.githubusercontent.com/TrueBlocks/trueblocks-core/refs/heads/master/src/other/install/per-chain/" + // url, err := url.JoinPath(baseURL, chain, "allocs.csv") + // if err != nil { + // return err + // } + // allocFn := filepath.Join(chainConfig, "allocs.csv") + // dur := 100 * 365 * 24 * time.Hour // 100 years + // if _, err := utils.DownloadAndStore(url, allocFn, dur); err != nil { + // return fmt.Errorf("failed to download and store allocs.csv for chain %s: %w", chain, err) + // } + + return nil +} + var configTmpl string = `[version] current = "{{.Version}}" diff --git a/app/app.go b/app/app.go index 3b3aa44..4e3c27e 100644 --- a/app/app.go +++ b/app/app.go @@ -1,7 +1,7 @@ package app import ( - "fmt" + "log" "log/slog" "os" @@ -19,19 +19,9 @@ type KhedraApp struct { func NewKhedraApp() *KhedraApp { k := KhedraApp{} - - // If khedra is already running, one of these ports is serving the - // control API. We need to make sure it's not running and fail if - // it is. - cntlSvcPorts := []string{"8338", "8337", "8336", "8335"} - for _, port := range cntlSvcPorts { - if utils.PingServer("http://localhost:" + port) { - msg := fmt.Sprintf("Error: Khedra is already running (control service port :%s is in use). Quitting...", port) - fmt.Println(colors.Red+msg, colors.Off) - os.Exit(1) - } + if k.isRunning() { + log.Fatal(colors.BrightBlue + "khedra is already running - cannot run..." + colors.Off) } - k.cli = initCli(&k) return &k } @@ -40,6 +30,38 @@ func (k *KhedraApp) Run() { _ = k.cli.Run(os.Args) } +func (k *KhedraApp) isRunning() bool { + okArgs := map[string]bool{ + "help": true, + "-h": true, + "--help": true, + "version": true, + "-v": true, + "--version": true, + } + + if len(os.Args) < 2 || len(os.Args) == 2 && os.Args[1] == "config" { + return false + } + + for i, arg := range os.Args { + if okArgs[arg] { + return false + } else if arg == "config" && i < len(os.Args)-1 && os.Args[i+1] == "show" { + return false + } + } + + ports := []string{"8338", "8337", "8336", "8335"} + for _, port := range ports { + if utils.PingServer("http://localhost:" + port) { + return true + } + } + + return false +} + func (k *KhedraApp) ConfigMaker() (types.Config, error) { cfg, err := LoadConfig() if err != nil { diff --git a/app/config_helpers_test.go b/app/config_helpers_test.go index 56b0f3b..e43f6ad 100644 --- a/app/config_helpers_test.go +++ b/app/config_helpers_test.go @@ -28,7 +28,7 @@ func TestLoadFileConfig(t *testing.T) { cfg := types.NewConfig() chain := cfg.Chains["mainnet"] - chain.RPCs = []string{"http://localhost:8545", "http://localhost:8546"} + chain.RPCs = []string{"http://localhost:8545"} cfg.Chains["mainnet"] = chain bytes, _ := yamlv2.Marshal(cfg) _ = coreFile.StringToAsciiFile(types.GetConfigFn(), string(bytes)) @@ -116,77 +116,77 @@ func TestValidateConfig(t *testing.T) { } // --------------------------------------------------------- -// func TestInitializeFolders(t *testing.T) { -// cleanup := func(cfg types.Config) { -// os.RemoveAll(cfg.Logging.Folder) -// os.RemoveAll(cfg.General.DataFolder) -// } - -// allFoldersExist := func() { -// cfg := types.Config{ -// Logging: types.Logging{ -// Folder: "/tmp/test-logging-folder", -// }, -// General: types.General{ -// DataFolder: "/tmp/test-data-folder", -// Strategy: "download", -// Detail: "index", -// }, -// } - -// _ = os.MkdirAll(cfg.Logging.Folder, os.ModePerm) -// _ = os.MkdirAll(cfg.General.DataFolder, os.ModePerm) - -// err := initializeFolders(cfg) -// assert.NoError(t, err) - -// cleanup(cfg) -// } -// t.Run("All Folders Exist", func(t *testing.T) { allFoldersExist() }) - -// createMissingFolders := func() { -// cfg := types.Config{ -// Logging: types.Logging{ -// Folder: "/tmp/test-missing-logging-folder", -// }, -// General: types.General{ -// DataFolder: "/tmp/test-missing-data-folder", -// Strategy: "download", -// Detail: "index", -// }, -// } - -// cleanup(cfg) - -// err := initializeFolders(cfg) -// assert.NoError(t, err) - -// _, err = os.Stat(cfg.Logging.Folder) -// assert.NoError(t, err) -// _, err = os.Stat(cfg.General.DataFolder) -// assert.NoError(t, err) - -// cleanup(cfg) -// } -// t.Run("Create Missing Folders", func(t *testing.T) { createMissingFolders() }) - -// errorOnInvalidPath := func() { -// cfg := types.Config{ -// Logging: types.Logging{ -// Folder: "/invalid-folder-path/\\0", -// }, -// General: types.General{ -// DataFolder: "/tmp/test-data-folder", -// Strategy: "download", -// Detail: "index", -// }, -// } - -// err := initializeFolders(cfg) -// assert.Error(t, err) -// assert.Contains(t, err.Error(), "failed to create folder") - -// cleanup(cfg) -// } -// t.Run("Error On Invalid Path", func(t *testing.T) { errorOnInvalidPath() }) -// } +func TestInitializeFolders(t *testing.T) { + cleanup := func(cfg types.Config) { + os.RemoveAll(cfg.Logging.Folder) + os.RemoveAll(cfg.General.DataFolder) + } + + allFoldersExist := func() { + cfg := types.Config{ + Logging: types.Logging{ + Folder: "/tmp/test-logging-folder", + }, + General: types.General{ + DataFolder: "/tmp/test-data-folder", + Strategy: "download", + Detail: "index", + }, + } + + _ = os.MkdirAll(cfg.Logging.Folder, os.ModePerm) + _ = os.MkdirAll(cfg.General.DataFolder, os.ModePerm) + + err := initializeFolders(cfg) + assert.NoError(t, err) + + cleanup(cfg) + } + t.Run("All Folders Exist", func(t *testing.T) { allFoldersExist() }) + + createMissingFolders := func() { + cfg := types.Config{ + Logging: types.Logging{ + Folder: "/tmp/test-missing-logging-folder", + }, + General: types.General{ + DataFolder: "/tmp/test-missing-data-folder", + Strategy: "download", + Detail: "index", + }, + } + + cleanup(cfg) + + err := initializeFolders(cfg) + assert.NoError(t, err) + + _, err = os.Stat(cfg.Logging.Folder) + assert.NoError(t, err) + _, err = os.Stat(cfg.General.DataFolder) + assert.NoError(t, err) + + cleanup(cfg) + } + t.Run("Create Missing Folders", func(t *testing.T) { createMissingFolders() }) + + // errorOnInvalidPath := func() { + // cfg := types.Config{ + // Logging: types.Logging{ + // Folder: "/invalid-folder-path/\\0", + // }, + // General: types.General{ + // DataFolder: "/tmp/test-data-folder", + // Strategy: "download", + // Detail: "index", + // }, + // } + + // err := initializeFolders(cfg) + // assert.Error(t, err) + // assert.Contains(t, err.Error(), "failed to create folder") + + // cleanup(cfg) + // } + // t.Run("Error On Invalid Path", func(t *testing.T) { errorOnInvalidPath() }) +} diff --git a/pkg/types/config_descriptors.go b/pkg/types/config_descriptors.go index d51a5df..dd60120 100644 --- a/pkg/types/config_descriptors.go +++ b/pkg/types/config_descriptors.go @@ -1,29 +1,13 @@ package types import ( - "encoding/json" - "fmt" "path/filepath" - "strings" "time" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file" "github.com/TrueBlocks/trueblocks-khedra/v2/pkg/utils" ) -func (c *Config) ChainDescriptors() string { - ret := []string{} - for _, chain := range c.Chains { - descr, err := Descriptor(c.General.DataFolder, &chain) - if err != nil { - ret = append(ret, err.Error()) - } else { - ret = append(ret, descr.String()) - } - } - return strings.Join(ret, "\n") -} - type ChainList struct { Chains []ChainListItem `json:"chains"` ChainsMap map[int]*ChainListItem @@ -58,96 +42,23 @@ type Explorer struct { func UpdateChainList() (*ChainList, error) { configPath := utils.ResolvePath("~/.khedra") _ = file.EstablishFolder(configPath) - chainUrl := "https://chainid.network/chains.json" - chainsFn := filepath.Join(configPath, "chains.json") - if bytes, err := utils.DownloadAndStore(chainUrl, chainsFn, 24*time.Hour); err != nil { - return &ChainList{}, err - } else { - var chainList ChainList - err := json.Unmarshal(bytes, &chainList.Chains) - if err != nil { - return &ChainList{}, err - } - chainList.ChainsMap = make(map[int]*ChainListItem) - for _, chain := range chainList.Chains { - chainList.ChainsMap[chain.ChainID] = &chain - } - return &chainList, nil - } -} - -// ChainDescriptor represents the configuration of a single chain in the configurion file's template. -type ChainDescriptor struct { - Chain string `json:"chain"` - ChainID string `json:"chainId"` - RemoteExplorer string `json:"remoteExplorer"` - RpcProvider string `json:"rpcProvider"` - Symbol string `json:"symbol"` -} - -func (c *ChainDescriptor) String() string { - bytes, _ := json.Marshal(c) // MarshalIndent(c, "", " ") - return string(bytes) -} - -func Descriptor(configFolder string, ch *Chain) (ChainDescriptor, error) { - return ChainDescriptor{ - Chain: ch.Name, - ChainID: fmt.Sprintf("%d", ch.ChainID), - Symbol: "SYM", - RpcProvider: ch.RPCs[0], - RemoteExplorer: "https://etherscan.io", - }, nil -} - -// func (c *Config) ChainDescriptors2() string { -// // dataFn := filepath.Join(c.General.DataFolder, "chains.json") -// // chainData := file.AsciiFileToString(dataFn) -// // if !file.FileExists(dataFn) || len(chainData) == 0 { -// // chainData = `{ -// // "mainnet": { -// // "chain": "mainnet", -// // "chainId": "1", -// // "remoteExplorer": "https://etherscan.io", -// // "symbol": "ETH" -// // } -// // } -// // ` -// // } -// // chainDescrs := make(map[string]ChainDescriptor) -// // if err := json.Unmarshal([]byte(chainData), &chainDescrs); err != nil { -// // return err.Error() -// // } + chainURL := "https://chainid.network/chains.json" + chainsFile := filepath.Join(configPath, "chains.json") -// tmpl, err := template.New("chainConfigTmpl").Parse(` [chains.{{.Chain}}] -// chain = "{{.Chain}}" -// chainId = "{{.ChainID}}" -// remoteExplorer = "{{.RemoteExplorer}}" -// rpcProvider = "{{.RpcProvider}}" -// symbol = "{{.Symbol}}"`) -// if err != nil { -// return err.Error() -// } - -// ret := []string{} -// for _, ch := range c.Chains { -// if chainConfig, ok := c.ChainList.ChainsMap[ch.ChainId]; ok { -// chainConfig.RpcProvider = ch.RPCs[0] -// var buf bytes.Buffer -// if err = tmpl.Execute(&buf, &chainConfig); err != nil { -// return err.Error() -// } + chainData, err := utils.DownloadAndStoreJSON[[]ChainListItem](chainURL, chainsFile, 24*time.Hour) + if err != nil { + return nil, err + } -// ret = append(ret, buf.String()) -// } else { -// ret = append(ret, " # "+chain.Name+" is not supported") -// } -// } + var chainList ChainList + chainList.Chains = *chainData + chainList.ChainsMap = make(map[int]*ChainListItem) -// sort.Slice(ret, func(i, j int) bool { -// return strings.Compare(ret[i], ret[j]) < 0 -// }) + for _, chain := range chainList.Chains { + chainCopy := chain + chainList.ChainsMap[chain.ChainID] = &chainCopy + } -// return "\n" + strings.Join(ret, "\n") -// } + return &chainList, nil +} diff --git a/pkg/types/logging_test.go b/pkg/types/logging_test.go index 6956042..799a4f7 100644 --- a/pkg/types/logging_test.go +++ b/pkg/types/logging_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/TrueBlocks/trueblocks-khedra/v2/pkg/validate" "github.com/alecthomas/assert/v2" ) @@ -26,188 +27,188 @@ func TestLoggingNew(t *testing.T) { assert.True(t, logging.Compress) } -// func TestLoggingValidation(t *testing.T) { -// tempDir := createTempDir(t, true) +func TestLoggingValidation(t *testing.T) { + tempDir := createTempDir(t, true) -// tests := []struct { -// name string -// logging Logging -// wantErr bool -// }{ -// { -// name: "Valid Logging struct", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: true, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: false, -// }, -// { -// name: "Valid Logging level warn", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: false, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "warn", -// }, -// wantErr: false, -// }, -// { -// name: "Invalid Logging level", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: false, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "bogus", -// }, -// wantErr: true, -// }, -// { -// name: "Missing Folder", -// logging: Logging{ -// Filename: "app.log", -// ToFile: false, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "Non-existent Folder", -// logging: Logging{ -// Folder: "/non/existent/path", -// Filename: "app.log", -// ToFile: true, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "Missing Filename", -// logging: Logging{ -// Folder: tempDir, -// MaxSize: 10, -// ToFile: false, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "Filename without .log extension", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.txt", -// ToFile: false, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "MaxSize is zero", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: false, -// MaxSize: 0, -// MaxBackups: 3, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "MaxBackups is negative", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: false, -// MaxSize: 10, -// MaxBackups: -1, -// MaxAge: 7, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// { -// name: "MaxAge is negative", -// logging: Logging{ -// Folder: tempDir, -// Filename: "app.log", -// ToFile: true, -// MaxSize: 10, -// MaxBackups: 3, -// MaxAge: -1, -// Compress: true, -// Level: "info", -// }, -// wantErr: true, -// }, -// } + tests := []struct { + name string + logging Logging + wantErr bool + }{ + { + name: "Valid Logging struct", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: true, + MaxSize: 10, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: false, + }, + { + name: "Valid Logging level warn", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: false, + MaxSize: 10, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "warn", + }, + wantErr: false, + }, + { + name: "Invalid Logging level", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: false, + MaxSize: 10, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "bogus", + }, + wantErr: true, + }, + { + name: "Missing Folder", + logging: Logging{ + Filename: "app.log", + ToFile: false, + MaxSize: 10, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + // { + // name: "Non-existent Folder", + // logging: Logging{ + // Folder: "/non/existent/path", + // Filename: "app.log", + // ToFile: true, + // MaxSize: 10, + // MaxBackups: 3, + // MaxAge: 7, + // Compress: true, + // Level: "info", + // }, + // wantErr: true, + // }, + { + name: "Missing Filename", + logging: Logging{ + Folder: tempDir, + MaxSize: 10, + ToFile: false, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + { + name: "Filename without .log extension", + logging: Logging{ + Folder: tempDir, + Filename: "app.txt", + ToFile: false, + MaxSize: 10, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + { + name: "MaxSize is zero", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: false, + MaxSize: 0, + MaxBackups: 3, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + { + name: "MaxBackups is negative", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: false, + MaxSize: 10, + MaxBackups: -1, + MaxAge: 7, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + { + name: "MaxAge is negative", + logging: Logging{ + Folder: tempDir, + Filename: "app.log", + ToFile: true, + MaxSize: 10, + MaxBackups: 3, + MaxAge: -1, + Compress: true, + Level: "info", + }, + wantErr: true, + }, + } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// err := validate.Validate(&tt.logging) -// if tt.wantErr { -// assert.Error(t, err, "Expected error for test case '%s'", tt.name) -// } else { -// assert.NoError(t, err, "Did not expect error for test case '%s'", tt.name) -// } -// }) -// } -// } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validate.Validate(&tt.logging) + if tt.wantErr { + assert.Error(t, err, "Expected error for test case '%s'", tt.name) + } else { + assert.NoError(t, err, "Did not expect error for test case '%s'", tt.name) + } + }) + } +} func TestLoggingReadAndWrite(t *testing.T) { tempFilePath := "temp_config.yaml" content := ` - folder: ~/.khedra/logs - filename: khedra.log - toFile: false - maxSize: 10 - maxBackups: 3 - maxAge: 10 - compress: true - level: debug - ` + folder: ~/.khedra/logs + filename: khedra.log + toFile: false + maxSize: 10 + maxBackups: 3 + maxAge: 10 + compress: true + level: debug +` assertions := func(t *testing.T, logging *Logging) { - assert.Equal(t, "~/.khedra/logs", logging.Folder) - assert.Equal(t, "khedra.log", logging.Filename) - assert.False(t, logging.ToFile) - assert.Equal(t, "debug", logging.Level) - assert.Equal(t, 10, logging.MaxSize) - assert.Equal(t, 3, logging.MaxBackups) - assert.Equal(t, 10, logging.MaxAge) - assert.True(t, logging.Compress) + assert.Equal(t, "~/.khedra/logs", logging.Folder, "Folder should match the expected value") + assert.Equal(t, "khedra.log", logging.Filename, "Filename should match the expected value") + assert.False(t, logging.ToFile, "ToFile should be true") + assert.Equal(t, "debug", logging.Level, "Level should match the expected value") + assert.Equal(t, 10, logging.MaxSize, "MaxSize should match the expected value") + assert.Equal(t, 3, logging.MaxBackups, "MaxBackups should match the expected value") + assert.Equal(t, 10, logging.MaxAge, "MaxAge should match the expected value") + assert.True(t, logging.Compress, "Compress should be true") } ReadAndWriteWithAssertions[Logging](t, tempFilePath, content, assertions) diff --git a/pkg/utils/download_and_store.go b/pkg/utils/download_and_store.go index 2f24e08..6af6fc6 100644 --- a/pkg/utils/download_and_store.go +++ b/pkg/utils/download_and_store.go @@ -2,6 +2,7 @@ package utils import ( "encoding/json" + "fmt" "io" "net/http" "os" @@ -10,6 +11,36 @@ import ( "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file" ) +// DownloadAndStoreJSON is a generic function that: +// - Downloads from the given URL if the local file is stale. +// - Stores it in the given file path. +// - Unmarshals the JSON bytes into a type T and returns a *T. +// +// T must be a Go type compatible with the JSON structure (e.g. a struct or slice). +func DownloadAndStoreJSON[T any](url, filename string, cacheTTL time.Duration) (*T, error) { + // Use your existing caching logic from "utils.DownloadAndStore" + bytes, err := DownloadAndStore(url, filename, cacheTTL) + if err != nil { + var zero T + return &zero, err + } + + var result T + if err := json.Unmarshal(bytes, &result); err != nil { + return &result, err + } + return &result, nil +} + +// DownloadAndStore retrieves data from the specified URL and caches it in the provided +// filename for up to `dur`. If the file already exists and is newer than `dur`, it returns +// the file's contents without making a network request. Otherwise, it fetches from the URL. +// +// If the server returns 404, the function writes an empty file to disk and returns a zero-length +// byte slice. For other non-200 status codes, it returns an error. +// +// If the response is valid JSON, it is pretty-formatted before being saved; otherwise it is +// saved as-is. The function returns the written file content as a byte slice. func DownloadAndStore(url, filename string, dur time.Duration) ([]byte, error) { if file.FileExists(filename) { lastModDate, err := file.GetModTime(filename) @@ -31,6 +62,19 @@ func DownloadAndStore(url, filename string, dur time.Duration) ([]byte, error) { } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + // If the file doesn't exist remotely, store an empty file + if err := os.WriteFile(filename, []byte{}, 0644); err != nil { + return nil, err + } + // Optionally update its mod time + _ = file.Touch(filename) + return []byte{}, nil + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received status %d %s for URL %s", + resp.StatusCode, resp.Status, url) + } + rawData, err := io.ReadAll(resp.Body) if err != nil { return nil, err @@ -39,8 +83,7 @@ func DownloadAndStore(url, filename string, dur time.Duration) ([]byte, error) { var prettyData []byte if json.Valid(rawData) { var jsonData interface{} - err := json.Unmarshal(rawData, &jsonData) - if err != nil { + if err := json.Unmarshal(rawData, &jsonData); err != nil { return nil, err } prettyData, err = json.MarshalIndent(jsonData, "", " ") @@ -48,15 +91,14 @@ func DownloadAndStore(url, filename string, dur time.Duration) ([]byte, error) { return nil, err } } else { - // If the data is not valid JSON, write it as-is prettyData = rawData } - err = os.WriteFile(filename, prettyData, 0644) - if err != nil { + if err := os.WriteFile(filename, prettyData, 0644); err != nil { return nil, err } _ = file.Touch(filename) + return prettyData, nil } From a6ebbf5601cc256b28b07ea79efee0b9d4dff21d Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Mon, 10 Feb 2025 10:04:23 -0500 Subject: [PATCH 3/7] Updates go.mods --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a208ecb..94c1326 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/TrueBlocks/trueblocks-khedra/v2 go 1.23.1 require ( - github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250130023515-f86b9f89cfae + github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250131141006-ca15858b0e7c github.com/TrueBlocks/trueblocks-sdk/v4 v4.2.0 github.com/alecthomas/assert/v2 v2.11.0 github.com/goccy/go-yaml v1.15.15 diff --git a/go.sum b/go.sum index f2eadb9..d12bc65 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250130023515-f86b9f89cfae h1:ooRHiAxPwGbWth1OAkBzweJO5KRt1rygPO/IolnCzHE= -github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250130023515-f86b9f89cfae/go.mod h1:ETpBauf/NX6mlhJYeMu+Md9HTBHZ/mny1qu4rh4km1s= +github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250131141006-ca15858b0e7c h1:Wrfb99W0n3klva0ML2HGvHsW+A2TlfU7LC8MSjBBKHk= +github.com/TrueBlocks/trueblocks-core/src/apps/chifra v0.0.0-20250131141006-ca15858b0e7c/go.mod h1:UkmTpxKWh9RkC8V1LEi0MpjHKchRdptAalt6rf6zhks= github.com/TrueBlocks/trueblocks-sdk/v4 v4.2.0 h1:l1cRGIChEC6KW9JfLtFBUxr3lsnnNZHh6UdKK2eTNaw= github.com/TrueBlocks/trueblocks-sdk/v4 v4.2.0/go.mod h1:GJGFnBOzoNgs6b90QhvFzZbWMYpLx8cXaN4lCqRmGJI= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= From 04b195020b1843a90b88154dc3661ba77600c58a Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Sun, 16 Feb 2025 13:17:43 -0500 Subject: [PATCH 4/7] Updates README - first draft --- README.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bcbd4e9..8135a6c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,114 @@ -# trueblocks-khedra -A multi-service EVM blockchain index/monitoring tool that indexes, monitors, and shares address appearance indexes. +# TrueBlocks Khedra + +## Intro + +`trueblocks-khedra` is an extension (or plugin) to the [TrueBlocks](https://github.com/TrueBlocks/trueblocks-core) system that focuses on providing specialized data extraction, analysis, or other functionality related to Ethereum blockchain indexing. Khedra aims to simplify the process of gathering on-chain data and building advanced, queryable indexes for Ethereum addresses. + +Key features: + +- **Custom Indexing**: Provides specialized indexing capabilities tailored to specific use-cases beyond the core TrueBlocks functionality. +- **Plugin-Based Architecture**: Easily integrates with TrueBlocks while maintaining modular, extensible design. +- **Efficient Data Retrieval**: Optimized for quick querying and data lookups, especially when dealing with large Ethereum datasets. + +## Installation + +1. **Prerequisites** + - Make sure you have [TrueBlocks Core](https://github.com/TrueBlocks/trueblocks-core) installed. + - A C++ build environment (such as `g++` or `clang++`) if you plan to compile from source. + - [CMake](https://cmake.org/) (version 3.16 or higher recommended). + - (Optional) [Docker](https://docs.docker.com/get-docker/) if you plan to run via container. + +2. **Clone this Repository** + git clone https://github.com/TrueBlocks/trueblocks-khedra.git + cd trueblocks-khedra + +3. **Build from Source** + mkdir build && cd build + cmake .. + make + + After a successful build, you’ll find the `khedra` executable (or library, depending on how the project is organized) in the build output. + +4. **Install** + sudo make install + +## Configuration + +Before using `khedra`, you may need to configure it to point at the TrueBlocks indexing data or specify custom indexing rules: + +- **Config File**: By default, `khedra` may look for a configuration file at `~/.trueblocks/trueblocks-khedra.conf`. +- **Environment Variables**: + - `KHEDRA_DATA_DIR`: Path to where you want `khedra` to store or read data. + - `KHEDRA_LOG_LEVEL`: Adjusts the verbosity of logs (`DEBUG`, `INFO`, `WARN`, `ERROR`). + +Refer to the sample configuration file (`.conf.example`) in this repo for a template of possible settings. + +--- + +## Docker Version + + + +**(Paste the *exact* Docker Version text from the trueblocks-core README here.)** + +--- + +## Documentation + + + +**(Paste the *exact* Documentation text from the trueblocks-core README here.)** + +--- + +## Linting + + + +**(Paste the *exact* Linting text from the trueblocks-core README here.)** + +--- + +## Contributing + + + +**(Paste the *exact* Contributing text from the trueblocks-core README here.)** + +--- + +## Contact + + + +**(Paste the *exact* Contact text from the trueblocks-core README here.)** + +--- + +## Contributors + + + +**(Paste the *exact* Contributors text from the trueblocks-core README here.)** + +--- + +_This project is part of the [TrueBlocks](https://github.com/TrueBlocks) ecosystem._ From bb523b0df00d3c82e58522d4b0fb907128d8c0b5 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Sun, 16 Feb 2025 15:50:22 -0500 Subject: [PATCH 5/7] Updates README --- README.md | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 8135a6c..055729d 100644 --- a/README.md +++ b/README.md @@ -12,25 +12,35 @@ Key features: ## Installation -1. **Prerequisites** - - Make sure you have [TrueBlocks Core](https://github.com/TrueBlocks/trueblocks-core) installed. - - A C++ build environment (such as `g++` or `clang++`) if you plan to compile from source. - - [CMake](https://cmake.org/) (version 3.16 or higher recommended). - - (Optional) [Docker](https://docs.docker.com/get-docker/) if you plan to run via container. +### Prerequisites -2. **Clone this Repository** - git clone https://github.com/TrueBlocks/trueblocks-khedra.git - cd trueblocks-khedra +- Make sure you have [TrueBlocks Core](https://github.com/TrueBlocks/trueblocks-core) installed. +- A C++ build environment (such as `g++` or `clang++`) if you plan to compile from source. +- [CMake](https://cmake.org/) (version 3.16 or higher recommended). +- (Optional) [Docker](https://docs.docker.com/get-docker/) if you plan to run via container. -3. **Build from Source** - mkdir build && cd build - cmake .. - make +### Clone this Repository + + ```[bash] + git clone https://github.com/TrueBlocks/trueblocks-khedra.git + cd trueblocks-khedra + ``` + +### Build from Source + + ```[bash] + mkdir build && cd build + cmake .. + make + ``` After a successful build, you’ll find the `khedra` executable (or library, depending on how the project is organized) in the build output. -4. **Install** - sudo make install +### Install + +```[bash] +sudo make install +``` ## Configuration @@ -111,4 +121,4 @@ Refer to the sample configuration file (`.conf.example`) in this repo for a temp --- -_This project is part of the [TrueBlocks](https://github.com/TrueBlocks) ecosystem._ +This project is part of the [TrueBlocks](https://github.com/TrueBlocks) ecosystem. From c162d582b030e4856c01212a7bead97afc6d9686 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Mon, 17 Feb 2025 22:35:21 -0500 Subject: [PATCH 6/7] Updates README --- Dockerfile | 39 +++++++++++++++++++++++++++++++++++++++ README.md | 25 +++++++++++++++++++------ 2 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..242c294 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 + +# =========== 1) BUILD STAGE ============= +FROM ubuntu:24.04 AS builder + +# Install dependencies needed to build trueblocks-khedra +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git + +# Create and change to the source directory +WORKDIR /app + +# Copy source into the container +COPY . /app + +# Build the project +RUN mkdir build && cd build && cmake .. && make + +# =========== 2) FINAL STAGE ============= +FROM ubuntu:22.04 + +# Copy compiled binary from the builder stage +COPY --from=builder /app/build/khedra /usr/local/bin/khedra + +# Copy example config (you can move or rename as preferred) +COPY --from=builder /app/config.example /root/.trueblocks/trueblocks-khedra.conf + +# Set environment variables or defaults for Khedra +ENV KHEDRA_CONFIG=/root/.trueblocks/trueblocks-khedra.conf \ + KHEDRA_DATA_DIR=/root/.trueblocks/data \ + KHEDRA_LOG_LEVEL=INFO + +# Default entrypoint runs 'khedra' +ENTRYPOINT ["khedra"] +# Default command shows help text +CMD ["--help"] diff --git a/README.md b/README.md index 055729d..62433d1 100644 --- a/README.md +++ b/README.md @@ -55,14 +55,27 @@ Refer to the sample configuration file (`.conf.example`) in this repo for a temp --- -## Docker Version +## Docker Version - Building & Running - +Build the Docker image: + +```bash +docker build -t trueblocks-khedra . +``` + +Run the Docker container (showing the help message by default): + +```bash +docker run --rm -it trueblocks-khedra +``` + +Use a custom command, for example to specify a subcommand or different flags: + +```bash +docker run --rm -it trueblocks-khedra some-subcommand --flag +``` -**(Paste the *exact* Docker Version text from the trueblocks-core README here.)** +Adjust paths, environment variables, or your config file strategy as needed. You can also mount external volumes (e.g., a local ~/.trueblocks directory) if you prefer to maintain data outside the container. --- From fa97a732e8bdd3bcc0a5b4b1e4cc11886a95d4b1 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Sun, 23 Feb 2025 21:41:54 -0500 Subject: [PATCH 7/7] Much better linting --- app/action_init_chains.go | 3 +++ app/action_init_general.go | 6 ++++++ app/action_init_logging.go | 4 ++++ app/action_init_services.go | 10 +++++++++- app/cli.go | 2 ++ pkg/types/logging.go | 4 ++++ pkg/validate/field_validator.go | 5 ++++- pkg/validate/helpers.go | 3 +++ 8 files changed, 35 insertions(+), 2 deletions(-) diff --git a/app/action_init_chains.go b/app/action_init_chains.go index 0e92a6b..245fd0b 100644 --- a/app/action_init_chains.go +++ b/app/action_init_chains.go @@ -59,6 +59,7 @@ var c1 = wizard.Question{ |Ethereum. It must read state from the Unchained Index smart |contract. When you press enter, the RPC will be validated.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.Chain](q, func(cfg *types.Config) (string, types.Chain, error) { if _, ok := cfg.Chains["mainnet"]; !ok { cfg.Chains["mainnet"] = types.NewChain("mainnet", 1) @@ -69,6 +70,7 @@ var c1 = wizard.Question{ }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.Chain](q, func(cfg *types.Config) (string, types.Chain, error) { copy, ok := cfg.Chains["mainnet"] if !ok { @@ -103,6 +105,7 @@ var c2 = wizard.Question{ return input, validContinue() }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = q if input != "edit" && len(input) > 0 { return "", fmt.Errorf(`"edit" is the only valid response %w`, wizard.ErrValidate) } diff --git a/app/action_init_general.go b/app/action_init_general.go index d2c4aab..6475e84 100644 --- a/app/action_init_general.go +++ b/app/action_init_general.go @@ -59,12 +59,14 @@ var g1 = wizard.Question{ |more secure but much slower (depending on the chain, perhaps as |long as a few days).`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{Strategy: cfg.General.Strategy} return copy.Strategy, copy, nil }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{Strategy: input} switch input { @@ -93,6 +95,7 @@ var g2 = wizard.Question{ |but is slower when searching. Downloading the entire index takes |longer and is larger (180gb), but is much faster during search.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{Detail: cfg.General.Detail} if cfg.General.Strategy == "scratch" { @@ -102,6 +105,7 @@ var g2 = wizard.Question{ }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{Detail: input} switch input { @@ -128,6 +132,7 @@ var g3 = wizard.Question{ |binary caches?`, Hint: ``, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{DataFolder: cfg.General.DataFolder} if cfg.General.Detail == "bloom" { @@ -139,6 +144,7 @@ var g3 = wizard.Question{ }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.General](q, func(cfg *types.Config) (string, types.General, error) { copy := types.General{DataFolder: input} path, err := utils.ResolveValidPath(input) diff --git a/app/action_init_logging.go b/app/action_init_logging.go index a409aee..cdfa40d 100644 --- a/app/action_init_logging.go +++ b/app/action_init_logging.go @@ -51,6 +51,7 @@ var l1 = wizard.Question{ Hint: `Logging to the screen is always enabled. If you enable file-based |logging, Khedra will also write log files to disk.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.Logging](q, func(cfg *types.Config) (string, types.Logging, error) { copy := types.Logging{ToFile: cfg.Logging.ToFile, Filename: cfg.Logging.Filename} if cfg.Logging.ToFile { @@ -60,6 +61,7 @@ var l1 = wizard.Question{ }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.Logging](q, func(cfg *types.Config) (string, types.Logging, error) { copy := types.Logging{Filename: cfg.Logging.Filename, ToFile: cfg.Logging.ToFile} switch input { @@ -83,12 +85,14 @@ var l2 = wizard.Question{ Question: `What log level do you want to enable (debug, info, warn, error)?`, Hint: `Select a log level from the list.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return prepare[types.Logging](q, func(cfg *types.Config) (string, types.Logging, error) { copy := types.Logging{Level: cfg.Logging.Level} return cfg.Logging.Level, copy, validContinue() }) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return confirm[types.Logging](q, func(cfg *types.Config) (string, types.Logging, error) { copy := types.Logging{Level: input} if input != "debug" && input != "info" && input != "warn" && input != "error" { diff --git a/app/action_init_services.go b/app/action_init_services.go index 135a83a..55caf73 100644 --- a/app/action_init_services.go +++ b/app/action_init_services.go @@ -95,13 +95,15 @@ var s0 = wizard.Question{ var s1 = wizard.Question{ //.....question-|---------|---------|---------|---------|---------|----|65 Question: `Do you want to enable the "scraper" service?`, - Hint: `The "scraper" service constanly watches the blockchain and + Hint: `The "scraper" service constantly watches the blockchain and |updates the Unchained Index with new data. If you disable it, |your index will fall behind.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return sPrepare("scraper", input, q) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return sValidate("scraper", input, q) }, Replacements: []wizard.Replacement{ @@ -118,9 +120,11 @@ var s2 = wizard.Question{ |constantly keep the caches fresh for how ever many addresses you |like. You may not enable this service.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return sPrepare("monitor", input, q) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return sValidate("monitor", input, q) }, Replacements: []wizard.Replacement{ @@ -135,9 +139,11 @@ var s3 = wizard.Question{ Hint: `The "api" service serves all of chifra's endpoints as |described here: https://trueblocks.io/api/.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return sPrepare("api", input, q) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return sValidate("api", input, q) }, Replacements: []wizard.Replacement{ @@ -153,9 +159,11 @@ var s4 = wizard.Question{ |Each time a new index chunk and bloom filter is created, if this |service is enabled, it will automatically be pinned to IPFS.`, PrepareFn: func(input string, q *wizard.Question) (string, error) { + _ = input return sPrepare("ipfs", input, q) }, Validate: func(input string, q *wizard.Question) (string, error) { + _ = input return sValidate("ipfs", input, q) }, Replacements: []wizard.Replacement{ diff --git a/app/cli.go b/app/cli.go index f4cc823..20b61ee 100644 --- a/app/cli.go +++ b/app/cli.go @@ -21,6 +21,7 @@ func initCli(k *KhedraApp) *cli.App { } var onUsageError = func(c *cli.Context, err error, isSubcommand bool) error { + _ = isSubcommand showError(c, true, err) return nil } @@ -96,6 +97,7 @@ func initCli(k *KhedraApp) *cli.App { }, OnUsageError: onUsageError, CommandNotFound: func(c *cli.Context, command string) { + _ = command var err error if unknown := getUnknownCmd(); len(unknown) > 0 { err = fmt.Errorf("command '%s' not found", unknown) diff --git a/pkg/types/logging.go b/pkg/types/logging.go index 10a15d7..8dff7b5 100644 --- a/pkg/types/logging.go +++ b/pkg/types/logging.go @@ -144,6 +144,7 @@ type ColorTextHandler struct { } func (h *ColorTextHandler) Handle(ctx context.Context, r slog.Record) error { + _ = ctx levelColors := map[slog.Level]string{ slog.LevelDebug: colors.Cyan, slog.LevelInfo: colors.Green, @@ -187,14 +188,17 @@ func (h *ColorTextHandler) Handle(ctx context.Context, r slog.Record) error { } func (h *ColorTextHandler) Enabled(ctx context.Context, level slog.Level) bool { + _ = ctx return level >= h.Level } func (h *ColorTextHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + _ = attrs return h } func (h *ColorTextHandler) WithGroup(name string) slog.Handler { + _ = name return h } diff --git a/pkg/validate/field_validator.go b/pkg/validate/field_validator.go index 0b69e6c..5c24d09 100644 --- a/pkg/validate/field_validator.go +++ b/pkg/validate/field_validator.go @@ -51,7 +51,10 @@ func init() { ValidatorRegistry["non_zero"] = nonZeroValidator ValidatorRegistry["required"] = requiredValidator ValidatorRegistry["req_if_enabled"] = reqIfEnabledValidator - ValidatorRegistry["dive"] = func(fv FieldValidator) error { return nil } + ValidatorRegistry["dive"] = func(fv FieldValidator) error { + _ = fv + return nil + } } // RegisterValidator registers a new validator function if it does not diff --git a/pkg/validate/helpers.go b/pkg/validate/helpers.go index ac7c0a2..f072661 100644 --- a/pkg/validate/helpers.go +++ b/pkg/validate/helpers.go @@ -24,6 +24,9 @@ func getIntValue(fieldVal reflect.Value) (int64, error) { } func Passed(fv FieldValidator, value, test string) error { + _ = fv + _ = value + _ = test // c := fmt.Sprintf(" context=%q", fv.context) // if fv.context == "" { // c = ""