Skip to content

Commit

Permalink
Best version yet
Browse files Browse the repository at this point in the history
  • Loading branch information
@tjayrush
tjayrush committed Jan 28, 2025
1 parent a6cedda commit 50ec790
Show file tree
Hide file tree
Showing 13 changed files with 260 additions and 331 deletions.
174 changes: 54 additions & 120 deletions app/action_daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package app
import (
"fmt"
"log/slog"
"regexp"
"strings"
"unicode"

coreFile "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
_ "github.com/TrueBlocks/trueblocks-khedra/v2/pkg/env"
Expand Down Expand Up @@ -37,7 +35,7 @@ func (k *KhedraApp) daemonAction(c *cli.Context) error {
}

var activeServices []services.Servicer
chains := strings.Split(strings.ReplaceAll(k.config.ChainList(), " ", ""), ",")
chains := strings.Split(strings.ReplaceAll(k.config.ChainList(false /* enabledOnly */), " ", ""), ",")
scraperSvc := services.NewScrapeService(
k.logger,
"all",
Expand Down Expand Up @@ -73,24 +71,14 @@ func (k *KhedraApp) daemonAction(c *cli.Context) error {
}

/*
func init() {
if pwd, err := os.Getwd(); err == nil {
if file.FileExists(filepath.Join(pwd, ".env")) {
if err = godotenv.Load(filepath.Join(pwd, ".env")); err != nil {
fmt.Fprintf(os.Stderr, "Found .env, but could not read it\n")
}
}
}
}
SHOW ALL THE TB_KHEDRA_ VARIABLES FOUND
SHOW THE DATA FOLDER
THERE USED TO BE A DIFFERENCE BETWEEN THE INDEXED CHAINS AND THE CHAINS REQUIRING RPCS
TELL THE USER WHICH CHAINS ARE BEING PROCESSED
TELL THE USER WHICH SERVICES ARE BEING STARTED
// EstablishConfig either reads an existing configuration file or creates it if it doesn't exist.
func (a *App) EstablishConfig() error {
for _, arg := range os.Args {
if arg == "--help" || arg == "-h" || arg == "--version" {
return nil
}
}
var ok bool
var err error
if a.Config.ConfigPath, ok = os.LookupEnv("TB_NODE_DATADIR"); !ok {
Expand Down Expand Up @@ -201,47 +189,6 @@ func (a *App) EstablishConfig() error {
return nil
}
func (a *App) tryConnect(chain, providerUrl string, maxAttempts int) error {
for i := 1; i <= maxAttempts; i++ {
err := rpc.PingRpc(providerUrl)
if err == nil {
return nil
} else {
a.Logger.Warn("retrying RPC", "chain", chain, "provider", providerUrl)
if i < maxAttempts {
time.Sleep(1 * time.Second)
}
}
}
return fmt.Errorf("cannot connect to RPC (%s-%s) after %d attempts", chain, providerUrl, maxAttempts)
}
func isValidURL(str string) bool {
u, err := url.Parse(str)
return err == nil && u.Scheme != "" && u.Host != ""
}
// cleanDataPath cleans up the data path, replacing PWD, ~, and HOME with the appropriate values
func cleanDataPath(in string) (string, error) {
pwd, err := os.Getwd()
if err != nil {
return in, err
}
out := strings.ReplaceAll(in, "PWD", pwd)
home, err := os.UserHomeDir()
if err != nil {
return in, err
}
out = strings.ReplaceAll(out, "~", home)
out = strings.ReplaceAll(out, "HOME", home)
ret := filepath.Clean(out)
if strings.HasSuffix(ret, "/unchained") {
ret = strings.ReplaceAll(ret, "/unchained", "")
}
return ret, nil
}
var configTmpl string = `[version]
current = "v4.0.0"
Expand All @@ -258,9 +205,6 @@ var configTmpl string = `[version]
`
AT LEAST ONE SERVICE (OUT OF MONITOR, SCRAPER, API) MUST BE ENABLED
AT LEAST ONE VALID CHAIN WITH ACTIVE RPC MUST BE PROVIDED
A MAINNET RPC MUST BE PROVIDED
handleService := func(i int, feature Feature) (int, error) {
if hasValue(i) {
Expand Down Expand Up @@ -308,72 +252,62 @@ A MAINNET RPC MUST BE PROVIDED
activeServices = append([]services.Servicer{controlService}, activeServices...)
}
*/
// cleanChainString processes and ensures the correctness of a chain string.
// - Uses splitChainString to validate and clean the input.
// - Guarantees that "mainnet" appears at the front of the returned `chains` string,
// appending it if not already included.
// - The `chains` string includes all valid, deduplicated chains starting with "mainnet".
// - The `targets` string preserves the validated input order of chains.
func cleanChainString(input string) (string, string, error) {
targets, err := splitChainString(input)
if err != nil {
return "", "", fmt.Errorf("invalid chain string: %v", err)
}
chainStrs := []string{"mainnet"}
for _, chain := range targets {
if chain != "mainnet" {
chainStrs = append(chainStrs, chain)
}
FOR RUNNING CORE
os.Setenv("XDG_CONFIG_HOME", a.Config.ConfigPath)
os.Setenv("TB_ SETTINGS_DEFAULTCHAIN", "mainnet")
os.Setenv("TB_ SETTINGS_INDEXPATH", a.Config.IndexPath())
os.Setenv("TB_ SETTINGS_CACHEPATH", a.Config.CachePath())
for chain, providerUrl := range a.Config.ProviderMap {
envKey := "TB_CHAINS_" + strings.ToUpper(chain) + "_RPCPROVIDER"
os.Setenv(envKey, providerUrl)
}

return strings.Join(chainStrs, ","), strings.Join(targets, ","), nil
}

// splitChainString validates and processes a comma-separated string of chains.
// - Trims leading/trailing whitespace from each chain.
// - Ensures no internal whitespace within each chain.
// - Validates that each chain contains only alphanumeric characters, dashes, and underscores.
// - Removes duplicates while preserving order.
// Returns a slice of valid chains or an error if validation fails.
func splitChainString(input string) ([]string, error) {
validChainRegex := regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)

parts := strings.Split(input, ",")
var cleanedParts []string
haveMap := map[string]bool{}

for _, part := range parts {
trimmedPart := strings.TrimSpace(part)
if len(trimmedPart) == 0 {
continue
}
for _, r := range trimmedPart {
if unicode.IsSpace(r) {
return nil, fmt.Errorf("%w: '%s'", ErrInternalWhitespace, trimmedPart)
for _, env := range os.Environ() {
if (strings.HasPrefix(env, "TB_") || strings.HasPrefix(env, "XDG_")) && strings.Contains(env, "=") {
parts := strings.Split(env, "=")
if len(parts) > 1 {
a.Logger.Info("environment", parts[0], parts[1])
} else {
a.Logger.Info("environment", parts[0], "<empty>")
}
}
if !validChainRegex.MatchString(trimmedPart) {
return nil, fmt.Errorf("%w: '%s'", ErrInvalidCharacter, trimmedPart)
}
for _, chain := range chains {
providerUrl := a.Config.ProviderMap[chain]
if err := a.tryConnect(chain, providerUrl, 5); err != nil {
return err
} else {
a.Logger.Info("test connection", "result", "okay", "chain", chain, "providerUrl", providerUrl)
}
if !haveMap[trimmedPart] {
cleanedParts = append(cleanedParts, trimmedPart)
haveMap[trimmedPart] = true
}
USED TO CHECK THAT IF THE USER SPECIFIED A CHAIN IN THE ENV, THEN IT HAD TO EXISTING IN TRUEBLOCKS.TOML
configFn := filepath.Join(a.Config.ConfigPath, "trueBlocks.toml")
if file.FileExists(configFn) {
a.Logger.Info("config loaded", "configFile", configFn, "nChains", len(a.Config.ProviderMap))
// check to make sure the config file has all the chains
contents := file.AsciiFileToString(configFn)
for chain := range a.Config.ProviderMap {
search := "[chains." + chain + "]"
if !strings.Contains(contents, search) {
msg := fmt.Sprintf("config file {%s} does not contain {%s}", configFn, search)
msg = colors.ColoredWith(msg, colors.Red)
return errors.New(msg)
}
}
return nil
}

if len(cleanedParts) == 0 {
return nil, ErrEmptyResult
USED TO ESTABLISH FOLDERS
if err := file.Establish Folder(a.Config.ConfigPath); err != nil {
return err
}
for _, chain := range chains {
chainConfig := filepath.Join(a.Config.ConfigPath, "config", chain)
if err := file.Establish Folder(chainConfig); err != nil {
return err
}
}
WOULD CREATE A MINIMAL TRUEBLOCKS.TOML IF NOT FOUND
return cleanedParts, nil
}

var (
ErrInternalWhitespace = fmt.Errorf("invalid chain string: internal whitespace in part")
ErrInvalidCharacter = fmt.Errorf("invalid chain string: invalid character in part")
ErrEmptyResult = fmt.Errorf("invalid chain string: no valid chains found")
)
*/
111 changes: 41 additions & 70 deletions app/action_init_chains.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package app

import (
"encoding/json"
"fmt"
"log"
"strings"
Expand Down Expand Up @@ -47,54 +46,6 @@ If you are rate limited (likely), use the sleep option. See "help".
}
}

// --------------------------------------------------------
func cPrepare(key, input string, q *wizard.Question) (string, error) {
if cfg, ok := q.Screen.Wizard.Backing.(*types.Config); ok {
if key == "mainnet" {
if ch, ok := cfg.Chains[key]; !ok {
ch.Name = key
ch.Enabled = true
cfg.Chains[key] = ch
}

if ch, ok := cfg.Chains[key]; !ok {
log.Fatal("chain not found")
} else {
if !ch.HasValidRpc() {
bytes, _ := json.Marshal(&ch)
q.State = string(bytes)
msg := fmt.Sprintf("no rpcs for chain %s ", key)
return strings.Join(ch.RPCs, ","), fmt.Errorf(msg+"%w", wizard.ErrValidate)
}
}
}
}
return input, validOk("skip - have all rpcs", input)
}

// --------------------------------------------------------
func cValidate(key string, input string, q *wizard.Question) (string, error) {
if _, ok := q.Screen.Wizard.Backing.(*types.Config); ok {
if cfg, ok := q.Screen.Wizard.Backing.(*types.Config); ok {
if key == "mainnet" {
if ch, ok := cfg.Chains[key]; !ok {
log.Fatal("chain not found")
} else {
ch.RPCs = strings.Split(input, ",")
cfg.Chains[key] = ch
if !ch.HasValidRpc() {
bytes, _ := json.Marshal(&ch)
q.State = string(bytes)
msg := fmt.Sprintf("no rpcs for chain %s ", key)
return strings.Join(ch.RPCs, ","), fmt.Errorf(msg+"%w", wizard.ErrValidate)
}
}
}
}
}
return input, nil
}

// --------------------------------------------------------
var c0 = wizard.Question{
//.....question-|---------|---------|---------|---------|---------|----|65
Expand All @@ -103,17 +54,35 @@ var c0 = wizard.Question{
// --------------------------------------------------------
var c1 = wizard.Question{
//.....question-|---------|---------|---------|---------|---------|----|65
Question: `Please provide an RPC for Ethereum mainnet?`,
Hint: `Khedra requires an Ethereum mainnet RPC. It needs to read
|state from the Unchained Index smart contract. Type "help" for
|more information.`,
Question: `Please provide an RPC for Ethereum mainnet.`,
Hint: `Khedra requires a valid, reachable RPC for Mainnet
|Ethereum. It must read state from the Unchained Index smart
|contract. When you press enter, the RPC will be validated.`,
PrepareFn: func(input string, q *wizard.Question) (string, error) {
// qq := ChainQuestion{Question: *q}
return cPrepare("mainnet", input, q)
return prepare[types.Chain](q, func(cfg *types.Config) (string, types.Chain, error) {
if _, ok := cfg.Chains["mainnet"]; !ok {
cfg.Chains["mainnet"] = types.NewChain("mainnet")
}
copy := cfg.Chains["mainnet"]
copy.Name = ""
return strings.Join(copy.RPCs, ","), copy, validContinue()
})
},
Validate: func(input string, q *wizard.Question) (string, error) {
// qq := ChainQuestion{Question: *q}
return cValidate("mainnet", input, q)
return confirm[types.Chain](q, func(cfg *types.Config) (string, types.Chain, error) {
copy, ok := cfg.Chains["mainnet"]
if !ok {
log.Fatal("chain mainnet not found")
}
copy.RPCs = strings.Split(input, ",")
if !copy.HasValidRpc() {
copy.Name = ""
return strings.Join(copy.RPCs, ","), copy, fmt.Errorf(`no rpcs for chain mainnet %w`, wizard.ErrValidate)
}
cfg.Chains["mainnet"] = copy
copy.Name = ""
return input, copy, validOk("mainnet rpc set to %s", input)
})
},
Replacements: []wizard.Replacement{
{Color: colors.Green, Values: []string{"\"help\"", "Unchained Index"}},
Expand All @@ -123,22 +92,24 @@ var c1 = wizard.Question{
// --------------------------------------------------------
var c2 = wizard.Question{
//.....question-|---------|---------|---------|---------|---------|----|65
Question: `Which chains do you want to index?`,
Hint: `Enter a comma separated list of chains to index. The wizard will
|ask you next for RPCs. Enter "chains" to open a large list of
|EVM chains. Use the shortNames from that list to name your
|chains. When you publish your index, others' indexes will
|match (e.g., mainnet, gnosis, optimism, sepolia, etc.)`,
Question: `Do you want to index other chains?`,
Hint: `You may index as many chains as you wish. All you need
|is a separate, fast RPC endpoint for each chain. If
|you do want to index another chain, type "edit" to open
|the file in your editor. Adding your own chains should be
|obvious. Save your work to return to this screen.`,
PrepareFn: func(input string, q *wizard.Question) (string, error) {
// qq := ChainQuestion{Question: *q}
return cPrepare("chain", input, q)
q.Screen.Instructions = `Type "edit" to add another chain or press enter to continue.`
return input, validContinue()
},
Validate: func(input string, q *wizard.Question) (string, error) {
// qq := ChainQuestion{Question: *q}
return cValidate("chain", input, q)
if input != "edit" && len(input) > 0 {
return "", fmt.Errorf(`"edit" is the only valid response %w`, wizard.ErrValidate)
}
return input, validContinue()
},
}

// type ChainQuestion struct {
// wizard.Question
// }
type ChainQuestion struct {
wizard.Question
}
Loading

0 comments on commit 50ec790

Please sign in to comment.