Merge pull request #11 from bitwiseguy/ss/replayor-testing

Updates based on optimism benchmark testing

.gitignore

@@ -5,3 +5,4 @@ geth-data/
 secret/
 results/
 reth-data/
+*.crt

Dockerfile

@@ -12,4 +12,6 @@ WORKDIR /app
 
 FROM golang:1.21
 
-COPY --from=builder /app/bin/replayor /app/bin/replayor
+COPY --from=builder /app/bin/replayor /app/bin/replayor
+
+CMD ["/app/bin/replayor"]

cmd/replayor/main.go

@@ -45,18 +45,23 @@ func Main() cliapp.LifecycleAction {
 		if err != nil {
 			return nil, err
 		}
+		logger.Info("config setup complete")
 
 		c, err := clients.SetupClients(cfg, logger, ctx)
 		if err != nil {
 			return nil, err
 		}
+		logger.Info("client setup complete")
 
 		// Benchmark stats
 		s, err := stats.NewStorage(logger, cfg)
 		if err != nil {
 			return nil, err
 		}
-		statsRecorder := stats.NewStoredStats(s, logger)
+		logger.Info("storage setup complete")
+
+		statsRecorder := stats.NewStoredStats(s, logger, cfg.BenchmarkStartBlock)
+		logger.Info("stats recorder setup complete")
 
 		return replayor.NewService(c, statsRecorder, cfg, logger, close), nil
 	}

packages/clients/clients.go

@@ -21,21 +21,22 @@ type Clients struct {
 }
 
 func SetupClients(cfg config.ReplayorConfig, logger log.Logger, ctx context.Context) (Clients, error) {
+	logger.Info("dialing source node url", "url", cfg.SourceNodeUrl)
 	sourceNode, err := ethclient.Dial(cfg.SourceNodeUrl)
 	if err != nil {
 		return Clients{}, err
 	}
 
 	// Add some retries around connecting to the dest node as it starts up at the same time as the replayor
 	destNode, err := retry.Do(ctx, 120, retry.Fixed(time.Second), func() (*ethclient.Client, error) {
+		logger.Info("dialing destination node url", "url", cfg.ExecutionUrl)
 		d, e := ethclient.Dial(cfg.ExecutionUrl)
 		if e != nil {
 			logger.Info("waiting for geth (exec) to start")
 		}
 
 		return d, e
 	})
-
 	if err != nil {
 		return Clients{}, err
 	}
@@ -49,6 +50,7 @@ func SetupClients(cfg config.ReplayorConfig, logger log.Logger, ctx context.Cont
 
 	// Add some retries around connecting to the dest node as it starts up at the same time as the replayor
 	engineApi, err := retry.Do(ctx, 120, retry.Fixed(time.Second), func() (*sources.EngineAPIClient, error) {
+		logger.Info("dialing destination engine api url", "url", cfg.EngineApiUrl)
 		l2Node, err := client.NewRPC(ctx, logger, cfg.EngineApiUrl, opts...)
 		if err != nil {
 			logger.Info("waiting for geth (rpc) to start")
@@ -59,7 +61,6 @@ func SetupClients(cfg config.ReplayorConfig, logger log.Logger, ctx context.Cont
 
 		return engineApi, nil
 	})
-
 	if err != nil {
 		return Clients{}, err
 	}

packages/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"math/big"
 	"os"
 	"strings"
 
@@ -15,14 +16,14 @@ import (
 type ReplayorConfig struct {
 	EngineApiSecret     common.Hash
 	SourceNodeUrl       string
-	ChainId             string
+	ChainId             *big.Int
 	RollupConfig        *rollup.Config
 	EngineApiUrl        string
 	ExecutionUrl        string
 	Strategy            string
 	BlockCount          int
-	GasTarget           int
-	GasLimit            int
+	GasTarget           uint64
+	GasLimit            uint64
 	BenchmarkStartBlock uint64
 	BenchmarkOpcodes    bool
 	ComputeStorageDiffs bool
@@ -44,13 +45,12 @@ func valueOrNil(i *uint64) string {
 }
 
 func LoadReplayorConfig(cliCtx *cli.Context, l log.Logger) (ReplayorConfig, error) {
-	jwtFile := cliCtx.String(EngineApiSecret.Name)
-	jwtBytes, err := os.ReadFile(jwtFile)
-	if err != nil {
-		return ReplayorConfig{}, err
+	secret := cliCtx.String(EngineApiSecret.Name)
+	if secret == "" {
+		return ReplayorConfig{}, fmt.Errorf("must provide REPLAYOR_ENGINE_API_SECRET env var")
 	}
 
-	secret := common.HexToHash(strings.TrimSpace(string(jwtBytes)))
+	secretHash := common.HexToHash(strings.TrimSpace(secret))
 
 	chainId := cliCtx.String(ChainId.Name)
 	rollupCfgPath := cliCtx.String(RollupConfigPath.Name)
@@ -66,26 +66,35 @@ func LoadReplayorConfig(cliCtx *cli.Context, l log.Logger) (ReplayorConfig, erro
 
 	l.Info("activation", "canyon", valueOrNil(rollupCfg.CanyonTime), "delta", valueOrNil(rollupCfg.DeltaTime), "ecotone", valueOrNil(rollupCfg.EcotoneTime), "fjord", valueOrNil(rollupCfg.FjordTime))
 
-	hostname, err := os.Hostname()
-	if err != nil {
-		return ReplayorConfig{}, err
+	testName := cliCtx.String(TestName.Name)
+	if testName == "" {
+		testName, err = os.Hostname()
+		if err != nil {
+			return ReplayorConfig{}, err
+		}
+	}
+
+	gasLimit := cliCtx.Uint64(GasLimit.Name)
+	gasTarget := cliCtx.Uint64(GasTarget.Name)
+	if gasTarget > gasLimit {
+		return ReplayorConfig{}, fmt.Errorf("cannot set gasTarget greater than gasLimit")
 	}
 
 	return ReplayorConfig{
-		EngineApiSecret:     secret,
+		EngineApiSecret:     secretHash,
 		SourceNodeUrl:       cliCtx.String(SourceNodeUrl.Name),
-		ChainId:             chainId,
+		ChainId:             rollupCfg.L2ChainID,
 		RollupConfig:        rollupCfg,
 		EngineApiUrl:        cliCtx.String(EngineApiUrl.Name),
 		ExecutionUrl:        cliCtx.String(ExecutionUrl.Name),
 		Strategy:            cliCtx.String(Strategy.Name),
 		BlockCount:          cliCtx.Int(BlockCount.Name),
-		GasTarget:           cliCtx.Int(GasTarget.Name),
-		GasLimit:            cliCtx.Int(GasLimit.Name),
+		GasTarget:           gasTarget,
+		GasLimit:            gasLimit,
 		BenchmarkStartBlock: cliCtx.Uint64(BenchmarkStartBlock.Name),
 		BenchmarkOpcodes:    cliCtx.Bool(BenchmarkOpcodes.Name),
 		ComputeStorageDiffs: cliCtx.Bool(ComputeStorageDiffs.Name),
-		TestName:            hostname,
+		TestName:            testName + "_",
 		Bucket:              cliCtx.String(S3Bucket.Name),
 		StorageType:         cliCtx.String(StorageType.Name),
 		DiskPath:            cliCtx.String(DiskPath.Name),

packages/config/flags.go

@@ -11,7 +11,7 @@ const EnvVarPrefix = "REPLAYOR"
 var (
 	EngineApiSecret = &cli.StringFlag{
 		Name:     "engine-api-secret",
-		Usage:    "The path to the engine api secret",
+		Usage:    "Engine api secret",
 		Required: true,
 		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "ENGINE_API_SECRET"),
 	}
@@ -82,6 +82,12 @@ var (
 		Usage:   "whether to include storage diff metrics in the benchmark results",
 		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "COMPUTE_STORAGE_DIFFS"),
 	}
+	TestName = &cli.StringFlag{
+		Name:     "test-name",
+		Usage:    "test name used as prefix for output file",
+		Required: false,
+		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "TEST_NAME"),
+	}
 	S3Bucket = &cli.StringFlag{
 		Name:     "s3-bucket",
 		Usage:    "The S3 bucket to store results in",
@@ -104,7 +110,7 @@ var (
 
 func init() {
 	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
-	Flags = append(Flags, EngineApiSecret, SourceNodeUrl, ChainId, EngineApiUrl, ExecutionUrl, Strategy, BlockCount, GasTarget, GasLimit, S3Bucket, StorageType, DiskPath, BenchmarkStartBlock, BenchmarkOpcodes, ComputeStorageDiffs, RollupConfigPath)
+	Flags = append(Flags, EngineApiSecret, SourceNodeUrl, ChainId, EngineApiUrl, ExecutionUrl, Strategy, BlockCount, GasTarget, GasLimit, S3Bucket, StorageType, DiskPath, BenchmarkStartBlock, BenchmarkOpcodes, ComputeStorageDiffs, TestName, RollupConfigPath)
 }
 
 // Flags contains the list of configuration options available to the binary.