Messing around with configs

app/app.go

@@ -10,9 +10,9 @@ import (
 
 type KhedraApp struct {
 	Cli        *cli.App
-	Config     *config.Config
-	FileLogger *slog.Logger
-	ProgLogger *slog.Logger
+	config     *config.Config
+	fileLogger *slog.Logger
+	progLogger *slog.Logger
 }
 
 func NewKhedraApp() *KhedraApp {
@@ -21,9 +21,9 @@ func NewKhedraApp() *KhedraApp {
 	cli := initializeCli()
 
 	k := &KhedraApp{
-		Config:     cfg,
-		FileLogger: fileLogger,
-		ProgLogger: progLogger,
+		config:     cfg,
+		fileLogger: fileLogger,
+		progLogger: progLogger,
 		Cli:        cli,
 	}
 

app/loggers.go

@@ -0,0 +1,37 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+func (k *KhedraApp) Debug(msg string, v ...any) {
+	k.fileLogger.Debug(msg, v...)
+}
+
+func (k *KhedraApp) Info(msg string, v ...any) {
+	k.fileLogger.Info(msg, v...)
+	k.progLogger.Info(msg, v...)
+}
+
+func (k *KhedraApp) Warn(msg string, v ...any) {
+	k.fileLogger.Warn(msg, v...)
+	k.progLogger.Warn(msg, v...)
+}
+
+func (k *KhedraApp) Error(msg string, v ...any) {
+	k.fileLogger.Error(msg, v...)
+	k.progLogger.Error(msg, v...)
+}
+
+func (k *KhedraApp) Prog(msg string, v ...any) {
+	if len(v) > 0 && fmt.Sprint(v[len(v)-1]) == "\n" {
+		k.progLogger.Info(msg, v...)
+	} else {
+		timestamp := time.Now().Format("2006-01-02 15:04:05")
+		message := fmt.Sprintf("PROG %s %s: %s", timestamp, msg, fmt.Sprint(v...))
+		fmt.Fprintf(os.Stdout, "%s\r", message)
+		os.Stdout.Sync()
+	}
+}

config.yaml.example

@@ -2,53 +2,72 @@
 # Version: 2.0
 
 general:
-  data_dir: "~/.khedra/data"
-  log_level: "info"           # Options: debug, info, warn, error
+  data_dir: "~/.khedra/data"     # See note 1
 
 chains:
-  - name: "mainnet"
-    rpcs:
-      - "<replace-with-mainnet-rpc>"
-    enabled: true
+  - name: "mainnet"              # Blockchain name (see notes 2, 3, and 4)
+    rpcs:                        # A list of RPC endpoints (at least one is required)
+      - "rpc_endpoint_for_mainnet"
+    enabled: true                # `true` if this chain is enabled
 
   - name: "sepolia"
     rpcs:
-      - "<replace-with-sepolia-rpc>"
+      - "rpc_endpoint_for_sepolia"
     enabled: true
 
-  - name: "gnosis"
+  - name: "gnosis"                # Add as many chains as your machine can handle
     rpcs:
-      - "<replace-with-gnosis-rpc>"
-    enabled: false
+      - "rpc_endpoint_for_gnosis" # must be a reachable, valid URL if the chain is enabled
+    enabled: false                # this chain is disabled
 
   - name: "optimism"
     rpcs:
-      - "<replace-with-optimism-rpc>"
+      - "rpc_endpoint_for_optimism"
     enabled: false
 
-services:
-  - name: "api"
-    enabled: true
-    port: 8080
+services:                        # See note 5
+  - name: "scraper"              # Required. (One of: api, scraper, monitor, ipfs, control)
+    enabled: true                # `true` if the service is enabled
+    sleep: 12                    # Seconds between scraping batches (see note 6)
+    batch_size: 500              # Number of blocks to process in a batch (range: 50-10000)
 
-  - name: "scraper"
+  - name: "monitor"
     enabled: true
-    sleep: 60 # In seconds
-    batch_size: 500
+    sleep: 12                    # Seconds between scraping batches (see note 6)
+    batch_size: 500              # Number of blocks processed in a batch (range: 50-10000)
 
-  - name: "monitor"
+  - name: "api"
     enabled: true
-    sleep: 60 # In seconds
-    batch_size: 500
+    port: 8080                   # Port number for API service (the port must be available)
 
   - name: "ipfs"
     enabled: true
-    port: 5001
+    port: 5001                   # Port number for IPFS service (the port must be available)
+
+  - name: "control"
+    enabled: true                # Always enabled - false values are invalid
+    port: 5001                   # Port number for IPFS service (the port must be available)
 
 logging:
-  folder: "~/.khedra/logs"
-  filename: "khedra.log"
-  max_size_mb: 10
-  max_backups: 5
-  max_age_days: 30
-  compress: true
+  folder: "~/.khedra/logs"       # Path to log directory (must exist and be writable)
+  filename: "khedra.log"         # Log file name (must end with .log)
+  log_level: "info"              # One of: debug, info, warn, error
+  max_size_mb: 10                # Max log file size in MB
+  max_backups: 5                 # Number of backup log files to keep
+  max_age_days: 30               # Number of days to retain old logs
+  compress: true                 # Whether to compress backup logs
+
+#
+# **Notes:**
+#
+# 1. The `data_dir` value must be a valid, existing directory that is writable. You may wish to change this value to a location with suitable disc scape. Depending on configuration, the Unchained Index and binary caches may approach 200GB.
+#
+# 2. The `chains` section is required. At least one chain must be enabled.
+#
+# 3. If chains other than Ethereum `mainnet` are configured, you must also configure Ethereum `mainnet`. The software reads `mainnet` smart contracts (such as the *Unchained Index* and *UniSwap*) during normal operation.
+#
+# 4. We've used [this repository](https://github.com/ethereum-lists/chains) to identify chain names. Using consistent chain names aides in sharing indexes. Use these values in your configuration if you wish to fully participate in sharing the *Unchained Index*.
+#
+# 5. The `services` section is required. At least one service must be enabled.
+#
+# 6. When a `scraper` or `monitor` is "catching up" to a chain, the `sleep` value is ignored.

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/go-playground/validator/v10 v10.23.0
 	github.com/knadh/koanf/parsers/yaml v0.1.0
+	github.com/knadh/koanf/providers/env v1.0.0
 	github.com/knadh/koanf/providers/file v1.1.2
 	github.com/knadh/koanf/v2 v2.1.2
 	github.com/stretchr/testify v1.10.0
@@ -48,7 +49,6 @@ require (
 	github.com/ipfs/go-ipfs-api v0.6.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
-	github.com/knadh/koanf/providers/env v1.0.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect

main.go

@@ -9,13 +9,12 @@ import (
 func main() {
 	// Create a new Khedra app...
 	k := app.NewKhedraApp()
-
-	k.FileLogger.Info("Khedra started.")
-	defer k.FileLogger.Info("Khedra stopped.")
+	k.Debug("Khedra started.")
+	defer k.Debug("Khedra stopped.")
 
 	// ...and run it
 	if err := k.Run(); err != nil {
-		k.ProgLogger.Error(err.Error())
+		k.Error(err.Error())
 		os.Exit(1)
 	}
 }

pkg/config/chain.go

@@ -6,9 +6,9 @@ type Chain struct {
 	Enabled bool     `koanf:"enabled"`                                                 // Defaults to false if not specified
 }
 
-func NewChain() Chain {
+func NewChain(chain string) Chain {
 	return Chain{
-		Name:    "mainnet",
+		Name:    chain,
 		RPCs:    []string{"http://localhost:8545"},
 		Enabled: true,
 	}

pkg/config/config.go

@@ -13,21 +13,28 @@ type Config struct {
 }
 
 func NewConfig() Config {
+	chains := []Chain{NewChain("mainnet"), NewChain("sepolia")}
+	services := []Service{
+		NewService("scraper"),
+		NewService("monitor"),
+		NewService("api"),
+		NewService("ipfs"),
+	}
+
 	return Config{
-		General: NewGeneral(),
-		Chains:  []Chain{NewChain()},
-		Services: []Service{
-			NewService("scraper"),
-			NewService("monitor"),
-			NewService("api"),
-			NewService("ipfs"),
-		},
-		Logging: NewLogging(),
+		General:  NewGeneral(),
+		Chains:   chains,
+		Services: services,
+		Logging:  NewLogging(),
 	}
 }
 
 func establishConfig(fn string) bool {
 	cfg := NewConfig()
+	return writeConfig(&cfg, fn)
+}
+
+func writeConfig(cfg *Config, fn string) bool {
 	bytes, _ := yaml.Marshal(cfg)
 	coreFile.StringToAsciiFile(fn, string(bytes))
 	return coreFile.FileExists(fn)