From 8a97233e90109db42b9aad0b5c48bd08f1e18602 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 21 Feb 2025 03:39:05 +0530 Subject: [PATCH 01/11] eth call support --- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 255 +++++++++++++++------- 1 file changed, 182 insertions(+), 73 deletions(-) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 5d0aac513..17ed4c61a 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -13,7 +13,12 @@ import chronicles, json_rpc/[rpcserver, rpcclient, rpcproxy], eth/common/accounts, - web3/eth_api, + web3/[primitives, eth_api_types, eth_api], + ../../nimbus/beacon/web3_eth_conv, + ../../nimbus/common/common, + ../../nimbus/db/ledger, + ../../nimbus/transaction/call_evm, + ../../nimbus/[evm/types, evm/state], ../validate_proof, ../block_cache @@ -73,12 +78,132 @@ proc getBlockByTag( of BlockNumber: proxy.blockCache.getByNumber(tag.blockNumber) +proc getBlockByHash( + proxy: VerifiedRpcProxy, blockHash: Hash32 +): results.Opt[BlockObject] {.raises: [ValueError].} = + checkPreconditions(proxy) + proxy.blockCache.getPayloadByHash(blockHash) + proc getBlockByTagOrThrow( proxy: VerifiedRpcProxy, quantityTag: BlockTag ): BlockObject {.raises: [ValueError].} = getBlockByTag(proxy, quantityTag).valueOr: raise newException(ValueError, "No block stored for given tag " & $quantityTag) +proc getBlockByHashOrThrow( + proxy: VerifiedRpcProxy, blockHash: Hash32 +): BlockObject {.raises: [ValueError].} = + getBlockByHash(proxy, blockHash).valueOr: + raise newException(ValueError, "No block stored for given hash " & $blockHash) + +proc getBlockHeaderByTagOrThrow( + proxy: VerifiedRpcProxy, quantityTag: BlockTag +): Header {.raises: [ValueError].} = + let blk = getBlockByTag(proxy, quantityTag).valueOr: + raise newException(ValueError, "No block stored for given tag " & $quantityTag) + + return Header( + parentHash: blk.parentHash, + ommersHash: blk.sha3Uncles, + coinbase: blk.miner, + stateRoot: blk.stateRoot, + transactionsRoot: blk.transactionsRoot, + receiptsRoot: blk.receiptsRoot, + logsBloom: blk.logsBloom, + difficulty: blk.difficulty, + number: distinctBase(blk.number), + gasLimit: distinctBase(blk.gasLimit), + gasUsed: distinctBase(blk.gasUsed), + timestamp: blk.timestamp.ethTime, + extraData: distinctBase(blk.extraData), + mixHash: Bytes32(distinctBase(blk.mixHash)), + nonce: blk.nonce.get, + baseFeePerGas: blk.baseFeePerGas, + withdrawalsRoot: blk.withdrawalsRoot, + blobGasUsed: blk.blobGasUsed.u64, + excessBlobGas: blk.excessBlobGas.u64, + parentBeaconBlockRoot: blk.parentBeaconBlockRoot, + requestsHash: blk.requestsHash + ) + +proc getBlockHeaderByHashOrThrow( + proxy: VerifiedRpcProxy, blockHash: Hash32 +): Header {.raises: [ValueError].} = + let blk = getBlockByHash(proxy, blockHash).valueOr: + raise newException(ValueError, "No block stored for given hash " & $blockHash) + + return Header( + parentHash: blk.parentHash, + ommersHash: blk.sha3Uncles, + coinbase: blk.miner, + stateRoot: blk.stateRoot, + transactionsRoot: blk.transactionsRoot, + receiptsRoot: blk.receiptsRoot, + logsBloom: blk.logsBloom, + difficulty: blk.difficulty, + number: distinctBase(blk.number), + gasLimit: distinctBase(blk.gasLimit), + gasUsed: distinctBase(blk.gasUsed), + timestamp: blk.timestamp.ethTime, + extraData: distinctBase(blk.extraData), + mixHash: Bytes32(distinctBase(blk.mixHash)), + nonce: blk.nonce.get, + baseFeePerGas: blk.baseFeePerGas, + withdrawalsRoot: blk.withdrawalsRoot, + blobGasUsed: blk.blobGasUsed.u64, + excessBlobGas: blk.excessBlobGas.u64, + parentBeaconBlockRoot: blk.parentBeaconBlockRoot, + requestsHash: blk.requestsHash + ) + +proc getAccount(lcProxy: VerifiedRpcProxy, address: Address, quantityTag: BlockTag): Future[Account] {.async: (raises: [ValueError, CatchableError]).} = + let + blk = lcProxy.getBlockByTagOrThrow(quantityTag) + blockNumber = blk.number.uint64 + + let + proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) + account = getAccountFromProof( + blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, + proof.storageHash, proof.accountProof, + ).valueOr: + raise newException(ValueError, error) + + return account + +proc getCode(lcProxy: VerifiedRpcProxy, address: Address, quantityTag: BlockTag): Future[seq[byte]] {.async: (raises: [ValueError, CatchableError]).} = + let + blk = lcProxy.getBlockByTagOrThrow(quantityTag) + blockNumber = blk.number.uint64 + account = await lcProxy.getAccount(address, quantityTag) + + info "Forwarding eth_getCode", blockNumber + + if account.codeHash == EMPTY_CODE_HASH: + # account does not have any code, return empty hex data + return @[] + + let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber)) + + if isValidCode(account, code): + return code + else: + raise newException(ValueError, "received code doesn't match the account code hash") + +proc getStorageAt(lcProxy: VerifiedRpcProxy, address: Address, slot: UInt256, quantityTag: BlockTag): Future[UInt256] {.async: (raises: [ValueError, CatchableError]).} = + let + blk = lcProxy.getBlockByTagOrThrow(quantityTag) + blockNumber = blk.number.uint64 + + info "Forwarding eth_getStorageAt", blockNumber + + let + proof = await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber)) + slotValue = getStorageData(blk.stateRoot, slot, proof).valueOr: + raise newException(ValueError, error) + + slotValue + proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = lcProxy.proxy.rpc("eth_chainId") do() -> UInt256: lcProxy.chainId @@ -93,95 +218,79 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = lcProxy.proxy.rpc("eth_getBalance") do( address: Address, quantityTag: BlockTag ) -> UInt256: - # When requesting state for `latest` block number, we need to translate - # `latest` to actual block number as `latest` on proxy and on data provider - # can mean different blocks and ultimatly piece received piece of state - # must by validated against correct state root - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 - - info "Forwarding eth_getBalance call", blockNumber - - let - proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) - account = getAccountFromProof( - blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, - proof.storageHash, proof.accountProof, - ).valueOr: - raise newException(ValueError, error) - + let account = await lcProxy.getAccount(address, quantityTag) account.balance lcProxy.proxy.rpc("eth_getStorageAt") do( address: Address, slot: UInt256, quantityTag: BlockTag ) -> UInt256: - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 - - info "Forwarding eth_getStorageAt", blockNumber - - let proof = - await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber)) - - getStorageData(blk.stateRoot, slot, proof).valueOr: - raise newException(ValueError, error) + await lcProxy.getStorageAt(address, slot, quantityTag) lcProxy.proxy.rpc("eth_getTransactionCount") do( address: Address, quantityTag: BlockTag - ) -> uint64: - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 - - info "Forwarding eth_getTransactionCount", blockNumber - - let - proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) - - account = getAccountFromProof( - blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, - proof.storageHash, proof.accountProof, - ).valueOr: - raise newException(ValueError, error) - - account.nonce + ) -> Quantity: + let account = await lcProxy.getAccount(address, quantityTag) + Quantity(account.nonce) lcProxy.proxy.rpc("eth_getCode") do( address: Address, quantityTag: BlockTag ) -> seq[byte]: - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 - - info "Forwarding eth_getCode", blockNumber - let - proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) - account = getAccountFromProof( - blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, - proof.storageHash, proof.accountProof, - ).valueOr: - raise newException(ValueError, error) - - if account.codeHash == EMPTY_CODE_HASH: - # account does not have any code, return empty hex data - return @[] - - let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber)) - - if isValidCode(account, code): - return code - else: - raise newException( - ValueError, "Received code which does not match the account code hash" - ) + await lcProxy.getCode(address, quantityTag) + + lcProxy.proxy.rpc("eth_call") do( + args: TransactionArgs, quantityTag: BlockTag + ) -> seq[byte]: + + # eth_call + # 1. get the code with proof + let to = if args.to.isSome(): args.to.get() + else: raise newException(ValueError, "contract address missing in transaction args") + + # 2. get all storage locations that are accessed + let + code = await lcProxy.getCode(to, quantityTag) + header = lcProxy.getBlockHeaderByTagOrThrow(quantityTag) + blkNumber = header.number.uint64 + parent = lcProxy.getBlockHeaderByHashOrThrow(header.parentHash) + accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(blkNumber)) + + let accessList = if not accessListResult.error.isSome(): accessListResult.accessList + else: raise newException(ValueError, "couldn't get an access list for eth call") + + # 3. pull the storage values that are access along with their accounts and initialize db + let + com = CommonRef.new(newCoreDbRef DefaultDbMemory, nil) + fork = com.toEVMFork(header) + vmState = BaseVMState() + + vmState.init(parent, header, com, com.db.baseTxFrame()) + vmState.mutateLedger: + for accessPair in accessList: + let + accountAddr = accessPair.address + acc = await lcProxy.getAccount(accountAddr, quantityTag) + accCode = await lcProxy.getCode(accountAddr, quantityTag) + + db.setNonce(accountAddr, acc.nonce) + db.setBalance(accountAddr, acc.balance) + db.setCode(accountAddr, accCode) + + for slot in accessPair.storageKeys: + let slotInt = UInt256.fromHex(toHex(slot)) + let slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, quantityTag) + db.setStorage(accountAddr, slotInt, slotValue) + db.persist(clearEmptyAccount = false) # settle accounts storage + + # 4. run the evm with the initialized storage + let evmResult = rpcCallEvm(args, header, vmState).valueOr: + raise newException(ValueError, "rpcCallEvm error: " & $error.code) + + evmResult.output # TODO: # Following methods are forwarded directly to the web3 provider and therefore # are not validated in any way. lcProxy.proxy.registerProxyMethod("net_version") - lcProxy.proxy.registerProxyMethod("eth_call") lcProxy.proxy.registerProxyMethod("eth_sendRawTransaction") lcProxy.proxy.registerProxyMethod("eth_getTransactionReceipt") From f3d66e10a4dbfca6bc321b1fb0ac1f8f400fc0e5 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 25 Feb 2025 10:20:51 +0530 Subject: [PATCH 02/11] rebase fix --- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 17ed4c61a..067ddd4a8 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -14,11 +14,11 @@ import json_rpc/[rpcserver, rpcclient, rpcproxy], eth/common/accounts, web3/[primitives, eth_api_types, eth_api], - ../../nimbus/beacon/web3_eth_conv, - ../../nimbus/common/common, - ../../nimbus/db/ledger, - ../../nimbus/transaction/call_evm, - ../../nimbus/[evm/types, evm/state], + ../../execution_chain/beacon/web3_eth_conv, + ../../execution_chain/common/common, + ../../execution_chain/db/ledger, + ../../execution_chain/transaction/call_evm, + ../../execution_chain/[evm/types, evm/state], ../validate_proof, ../block_cache From 3c8784d14ea677e56e8c9dd24398037b1c5cc6a2 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 28 Feb 2025 11:06:31 +0530 Subject: [PATCH 03/11] use cache of headers instead of blocks --- nimbus_verified_proxy/block_cache.nim | 48 --- nimbus_verified_proxy/header_store.nim | 120 +++++++ .../nimbus_verified_proxy.nim | 150 ++++----- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 304 ++++++++---------- 4 files changed, 304 insertions(+), 318 deletions(-) delete mode 100644 nimbus_verified_proxy/block_cache.nim create mode 100644 nimbus_verified_proxy/header_store.nim diff --git a/nimbus_verified_proxy/block_cache.nim b/nimbus_verified_proxy/block_cache.nim deleted file mode 100644 index bc3430c6e..000000000 --- a/nimbus_verified_proxy/block_cache.nim +++ /dev/null @@ -1,48 +0,0 @@ -# nimbus_verified_proxy -# Copyright (c) 2022-2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [].} - -import eth/common/hashes, web3/eth_api_types, minilru, results - -## Cache for payloads received through block gossip and validated by the -## consensus light client. -## The payloads are stored in order of arrival. When the cache is full, the -## oldest payload is deleted first. -type BlockCache* = ref object - blocks: LruCache[Hash32, BlockObject] - -proc new*(T: type BlockCache, max: uint32): T = - let maxAsInt = int(max) - BlockCache(blocks: LruCache[Hash32, BlockObject].init(maxAsInt)) - -func len*(self: BlockCache): int = - len(self.blocks) - -func isEmpty*(self: BlockCache): bool = - len(self.blocks) == 0 - -proc add*(self: BlockCache, payload: BlockObject) = - # Only add if it didn't exist before - the implementation of `latest` relies - # on this.. - if payload.hash notin self.blocks: - self.blocks.put(payload.hash, payload) - -proc latest*(self: BlockCache): Opt[BlockObject] = - for b in self.blocks.values: - return Opt.some(b) - Opt.none(BlockObject) - -proc getByNumber*(self: BlockCache, number: Quantity): Opt[BlockObject] = - for b in self.blocks.values: - if b.number == number: - return Opt.some(b) - - Opt.none(BlockObject) - -proc getPayloadByHash*(self: BlockCache, hash: Hash32): Opt[BlockObject] = - self.blocks.get(hash) diff --git a/nimbus_verified_proxy/header_store.nim b/nimbus_verified_proxy/header_store.nim new file mode 100644 index 000000000..7cf8d976c --- /dev/null +++ b/nimbus_verified_proxy/header_store.nim @@ -0,0 +1,120 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + eth/common/hashes, + eth/common/headers, + web3/eth_api_types, + std/tables, + beacon_chain/spec/beaconstate, + beacon_chain/spec/datatypes/[phase0, altair, bellatrix], + beacon_chain/[light_client, nimbus_binary_common, version], + beacon_chain/el/engine_api_conversions, + minilru, + results + +type + HeaderStore* = ref object + headers: LruCache[Hash32, Header] + hashes: Table[base.BlockNumber, Hash32] + +func convHeader(lcHeader: ForkedLightClientHeader): Header = + withForkyHeader(lcHeader): + template p(): auto = forkyHeader.execution + + when lcDataFork >= LightClientDataFork.Capella: + let withdrawalsRoot = Opt.some(p.withdrawals_root.asBlockHash) + else: + let withdrawalsRoot = Opt.none(Hash32) + + when lcDataFork >= LightClientDataFork.Deneb: + let + blobGasUsed = Opt.some(p.blob_gas_used) + excessBlobGas = Opt.some(p.excess_blob_gas) + parentBeaconBlockRoot = Opt.some(forkyHeader.beacon.parent_root.asBlockHash) + else: + let + blobGasUsed = Opt.none(uint64) + excessBlobGas = Opt.none(uint64) + parentBeaconBlockRoot = Opt.none(Hash32) + + when lcDataFork >= LightClientDataFork.Electra: + # TODO: there is no visibility of the execution requests hash in light client header + let requestsHash = Opt.none(Hash32) + else: + let requestsHash = Opt.none(Hash32) + + when lcDataFork > LightClientDataFork.Altair: + let h = Header( + parentHash: p.parent_hash.asBlockHash, + ommersHash: EMPTY_UNCLE_HASH, + coinbase: addresses.Address(p.fee_recipient.data), + stateRoot: p.state_root.asBlockHash, + transactionsRoot: p.transactions_root.asBlockHash, + receiptsRoot: p.receipts_root.asBlockHash, + logsBloom: FixedBytes[BYTES_PER_LOGS_BLOOM](p.logs_bloom.data), + difficulty: DifficultyInt(0.u256), + number: base.BlockNumber(p.block_number), + gasLimit: GasInt(p.gas_limit), + gasUsed: GasInt(p.gas_used), + timestamp: EthTime(p.timestamp), + extraData: seq[byte](p.extra_data), + mixHash: p.prev_randao.data.to(Bytes32), + nonce: default(Bytes8), + baseFeePerGas: Opt.some(p.base_fee_per_gas), + withdrawalsRoot: withdrawalsRoot, + blobGasUsed: blobGasUsed, + excessBlobGas: excessBlobGas, + parentBeaconBlockRoot: parentBeaconBlockRoot, + requestsHash: requestsHash + ) + else: + # INFO: should never reach this point because running verified + # proxy for altair doesn't make sense + let h = Header() + return h + +proc new*(T: type HeaderStore, max: int): T = + HeaderStore(headers: LruCache[Hash32, Header].init(max)) + +func len*(self: HeaderStore): int = + len(self.headers) + +func isEmpty*(self: HeaderStore): bool = + len(self.headers) == 0 + +proc add*(self: HeaderStore, header: ForkedLightClientHeader) = + # Only add if it didn't exist before - the implementation of `latest` relies + # on this.. + let execHeader = convHeader(header) + withForkyHeader(header): + when lcDataFork > LightClientDataFork.Altair: + let execHash = forkyHeader.execution.block_hash.asBlockHash + + if execHash notin self.headers: + self.headers.put(execHash, execHeader) + self.hashes[execHeader.number] = execHash + +proc latest*(self: HeaderStore): Opt[Header] = + for h in self.headers.values: + return Opt.some(h) + + Opt.none(Header) + +proc get*(self: HeaderStore, number: base.BlockNumber): Opt[Header] = + let hash = + try: + self.hashes[number] + except: + return Opt.none(Header) + + return self.headers.peek(hash) + +proc get*(self: HeaderStore, hash: Hash32): Opt[Header] = + self.headers.peek(hash) diff --git a/nimbus_verified_proxy/nimbus_verified_proxy.nim b/nimbus_verified_proxy/nimbus_verified_proxy.nim index 9527ec25d..8a4347b0a 100644 --- a/nimbus_verified_proxy/nimbus_verified_proxy.nim +++ b/nimbus_verified_proxy/nimbus_verified_proxy.nim @@ -14,18 +14,17 @@ import confutils, eth/common/[keys, eth_types_rlp], json_rpc/rpcproxy, - beacon_chain/el/[el_manager, engine_api_conversions], + beacon_chain/el/[el_manager], beacon_chain/gossip_processing/optimistic_processor, beacon_chain/networking/network_metadata, beacon_chain/networking/topic_params, beacon_chain/spec/beaconstate, beacon_chain/spec/datatypes/[phase0, altair, bellatrix], beacon_chain/[light_client, nimbus_binary_common, version], - ../execution_chain/rpc/[cors, rpc_utils], - ../execution_chain/beacon/payload_conv, + ../execution_chain/rpc/cors, ./rpc/rpc_eth_api, ./nimbus_verified_proxy_conf, - ./block_cache + ./header_store from beacon_chain/gossip_processing/eth2_processor import toValidationResult @@ -56,12 +55,7 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): UInt256 = proc run*( config: VerifiedProxyConf, ctx: ptr Context ) {.raises: [CatchableError], gcsafe.} = - var headerCallback: OnHeaderCallback - if ctx != nil: - headerCallback = ctx.onHeader - - # Required as both Eth2Node and LightClient requires correct config type - var lcConfig = config.asLightClientConf() + # Required as both Eth2Node and LightClient requires correct config type {.gcsafe.}: setupLogging(config.logLevel, config.logStdout, none(OutFile)) @@ -72,16 +66,30 @@ proc run*( except Exception: notice "commandLineParams() exception" + # load constants and metadata for the selected chain + let metadata = loadEth2Network(config.eth2Network) + + # initialze verified proxy let - metadata = loadEth2Network(config.eth2Network) chainId = getConfiguredChainId(metadata) + authHooks = @[httpCors(@[])] # TODO: for now we serve all cross origin requests + # TODO: write a comment + clientConfig = config.web3url.asClientConfig() - for node in metadata.bootstrapNodes: - lcConfig.bootstrapNodes.add node + rpcProxy = RpcProxy.new( + [initTAddress(config.rpcAddress, config.rpcPort)], clientConfig, authHooks + ) - template cfg(): auto = - metadata.cfg + headerStore = HeaderStore.new(64) # block cache contains blocks downloaded from p2p + verifiedProxy = VerifiedRpcProxy.new(rpcProxy, headerStore, chainId) + # add handlers that verify RPC calls /rpc/rpc_eth_api.nim + verifiedProxy.installEthApiHandlers() + + # just for short hand convenience + template cfg(): auto = metadata.cfg + + # initiialize beacon node genesis data, beacon clock and forkDigests let genesisState = try: @@ -96,11 +104,14 @@ proc run*( except CatchableError as err: raiseAssert "Invalid baked-in state: " & err.msg + # getStateField reads seeks info directly from a byte array + # get genesis time and instantiate the beacon clock genesisTime = getStateField(genesisState[], genesis_time) beaconClock = BeaconClock.init(genesisTime).valueOr: error "Invalid genesis time in state", genesisTime quit QuitFailure + # get the function that itself get the current beacon time getBeaconTime = beaconClock.getBeaconTimeFn() genesis_validators_root = getStateField(genesisState[], genesis_validators_root) @@ -108,6 +119,13 @@ proc run*( genesisBlockRoot = get_initial_beacon_block(genesisState[]).root + # transform the config to fit as a light client config and as a p2p node(Eth2Node) config + var lcConfig = config.asLightClientConf() + for node in metadata.bootstrapNodes: + lcConfig.bootstrapNodes.add node + + # create new network keys, create a p2p node(Eth2Node) and create a light client + let rng = keys.newRng() netKeys = getRandomNetKeys(rng[]) @@ -116,96 +134,29 @@ proc run*( rng, lcConfig, netKeys, cfg, forkDigests, getBeaconTime, genesis_validators_root ) - blockCache = BlockCache.new(uint32(64)) - - # TODO: for now we serve all cross origin requests - authHooks = @[httpCors(@[])] - - clientConfig = config.web3url.asClientConfig() - - rpcProxy = RpcProxy.new( - [initTAddress(config.rpcAddress, config.rpcPort)], clientConfig, authHooks - ) - - verifiedProxy = VerifiedRpcProxy.new(rpcProxy, blockCache, chainId) - - optimisticHandler = proc( - signedBlock: ForkedSignedBeaconBlock - ) {.async: (raises: [CancelledError]).} = - notice "New LC optimistic block", - opt = signedBlock.toBlockId(), wallSlot = getBeaconTime().slotOrZero - withBlck(signedBlock): - when consensusFork >= ConsensusFork.Bellatrix: - if forkyBlck.message.is_execution_block: - template payload(): auto = - forkyBlck.message.body - - try: - # TODO parentBeaconBlockRoot / requestsHash - let blk = ethBlock( - executionPayload(payload.asEngineExecutionPayload()), - parentBeaconBlockRoot = Opt.none(Hash32), - requestsHash = Opt.none(Hash32), - ) - blockCache.add(populateBlockObject(blk.header.rlpHash, blk, 0.u256, true)) - except RlpError as exc: - debug "Invalid block received", err = exc.msg - - optimisticProcessor = initOptimisticProcessor(getBeaconTime, optimisticHandler) - + # light client is set to optimistic finalization mode lightClient = createLightClient( network, rng, lcConfig, cfg, forkDigests, getBeaconTime, genesis_validators_root, LightClientFinalizationMode.Optimistic, ) - verifiedProxy.installEthApiHandlers() - - info "Listening to incoming network requests" - network.registerProtocol( - PeerSync, - PeerSync.NetworkState.init(cfg, forkDigests, genesisBlockRoot, getBeaconTime), - ) - network.addValidator( - getBeaconBlocksTopic(forkDigests.phase0), - proc(signedBlock: phase0.SignedBeaconBlock): ValidationResult = - toValidationResult(optimisticProcessor.processSignedBeaconBlock(signedBlock)), - ) - network.addValidator( - getBeaconBlocksTopic(forkDigests.altair), - proc(signedBlock: altair.SignedBeaconBlock): ValidationResult = - toValidationResult(optimisticProcessor.processSignedBeaconBlock(signedBlock)), - ) - network.addValidator( - getBeaconBlocksTopic(forkDigests.bellatrix), - proc(signedBlock: bellatrix.SignedBeaconBlock): ValidationResult = - toValidationResult(optimisticProcessor.processSignedBeaconBlock(signedBlock)), - ) - network.addValidator( - getBeaconBlocksTopic(forkDigests.capella), - proc(signedBlock: capella.SignedBeaconBlock): ValidationResult = - toValidationResult(optimisticProcessor.processSignedBeaconBlock(signedBlock)), - ) - network.addValidator( - getBeaconBlocksTopic(forkDigests.deneb), - proc(signedBlock: deneb.SignedBeaconBlock): ValidationResult = - toValidationResult(optimisticProcessor.processSignedBeaconBlock(signedBlock)), - ) - lightClient.installMessageValidators() - + # start the p2p network and rpcProxy waitFor network.startListening() waitFor network.start() waitFor rpcProxy.start() + + # verify chain id that the proxy is connected to waitFor verifiedProxy.verifyChaindId() proc onFinalizedHeader( lightClient: LightClient, finalizedHeader: ForkedLightClientHeader ) = withForkyHeader(finalizedHeader): - when lcDataFork > LightClientDataFork.None: + when lcDataFork > LightClientDataFork.Altair: info "New LC finalized header", finalized_header = shortLog(forkyHeader) - if headerCallback != nil: + if ctx != nil: try: - headerCallback(cstring(Json.encode(forkyHeader)), 0) + ctx.onHeader(cstring(Json.encode(forkyHeader)), 0) except SerializationError as e: error "finalizedHeaderCallback exception", error = e.msg @@ -213,11 +164,12 @@ proc run*( lightClient: LightClient, optimisticHeader: ForkedLightClientHeader ) = withForkyHeader(optimisticHeader): - when lcDataFork > LightClientDataFork.None: + when lcDataFork > LightClientDataFork.Altair: info "New LC optimistic header", optimistic_header = shortLog(forkyHeader) - if headerCallback != nil: + headerStore.add(optimisticHeader) + if ctx != nil: try: - headerCallback(cstring(Json.encode(forkyHeader)), 1) + ctx.onHeader(cstring(Json.encode(forkyHeader)), 1) except SerializationError as e: error "optimisticHeaderCallback exception", error = e.msg @@ -276,11 +228,12 @@ proc run*( blocksGossipState = targetGossipState - proc onSecond(time: Moment) = + proc updateGossipStatus(time: Moment) = let wallSlot = getBeaconTime().slotOrZero() updateBlocksGossipStatus(wallSlot + 1) lightClient.updateGossipStatus(wallSlot + 1) + # updates gossip status every second every second proc runOnSecondLoop() {.async.} = let sleepTime = chronos.seconds(1) while true: @@ -288,15 +241,20 @@ proc run*( await chronos.sleepAsync(sleepTime) let afterSleep = chronos.now(chronos.Moment) let sleepTime = afterSleep - start - onSecond(start) + updateGossipStatus(start) let finished = chronos.now(chronos.Moment) let processingTime = finished - afterSleep trace "onSecond task completed", sleepTime, processingTime - onSecond(Moment.now()) + # update gossip status before starting the light client + updateGossipStatus(Moment.now()) + # start the light client lightClient.start() + # launch a async routine asyncSpawn runOnSecondLoop() + + # run an infinite loop and wait for a stop signal while true: poll() if ctx != nil and ctx.stop: @@ -305,7 +263,7 @@ proc run*( waitFor rpcProxy.stop() ctx.cleanup() # Notify client that cleanup is finished - headerCallback(nil, 2) + ctx.onHeader(nil, 2) break when isMainModule: diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 067ddd4a8..b0d0504b8 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -13,6 +13,7 @@ import chronicles, json_rpc/[rpcserver, rpcclient, rpcproxy], eth/common/accounts, + eth/common/addresses, web3/[primitives, eth_api_types, eth_api], ../../execution_chain/beacon/web3_eth_conv, ../../execution_chain/common/common, @@ -20,7 +21,7 @@ import ../../execution_chain/transaction/call_evm, ../../execution_chain/[evm/types, evm/state], ../validate_proof, - ../block_cache + ../header_store logScope: topics = "verified_proxy" @@ -28,178 +29,106 @@ logScope: type VerifiedRpcProxy* = ref object proxy: RpcProxy - blockCache: BlockCache - chainId: UInt256 - - QuantityTagKind = enum - LatestBlock - BlockNumber + headerStore: HeaderStore + chainId: Quantity BlockTag = eth_api_types.RtBlockIdentifier - QuantityTag = object - case kind: QuantityTagKind - of LatestBlock: - discard - of BlockNumber: - blockNumber: Quantity +template checkPreconditions(proxy: VerifiedRpcProxy) = + if proxy.headerStore.isEmpty(): + raise newException(ValueError, "Syncing") + +template rpcClient(lcProxy: VerifiedRpcProxy): RpcClient = + lcProxy.proxy.getClient() + +proc resolveTag( + self: VerifiedRpcProxy, blockTag: BlockTag +): base.BlockNumber {.raises: [ValueError].} = + self.checkPreconditions() -func parseQuantityTag(blockTag: BlockTag): Result[QuantityTag, string] = if blockTag.kind == bidAlias: - let tag = blockTag.alias.toLowerAscii + let tag = blockTag.alias.toLowerAscii() case tag of "latest": - return ok(QuantityTag(kind: LatestBlock)) + let hLatest = self.headerStore.latest() + if hLatest.isSome: + return hLatest.get().number + else: + raise newException(ValueError, "No block stored for given tag " & $blockTag) else: - return err("Unsupported blockTag: " & tag) + raise newException(ValueError, "No support for block tag " & $blockTag) else: - let quantity = blockTag.number - return ok(QuantityTag(kind: BlockNumber, blockNumber: quantity)) + return base.BlockNumber(distinctBase(blockTag.number)) -template checkPreconditions(proxy: VerifiedRpcProxy) = - if proxy.blockCache.isEmpty(): - raise newException(ValueError, "Syncing") - -template rpcClient(lcProxy: VerifiedRpcProxy): RpcClient = - lcProxy.proxy.getClient() - -proc getBlockByTag( - proxy: VerifiedRpcProxy, quantityTag: BlockTag -): results.Opt[BlockObject] {.raises: [ValueError].} = - checkPreconditions(proxy) - - let tag = parseQuantityTag(quantityTag).valueOr: - raise newException(ValueError, error) - - case tag.kind - of LatestBlock: - # this will always return some block, as we always checkPreconditions - proxy.blockCache.latest - of BlockNumber: - proxy.blockCache.getByNumber(tag.blockNumber) - -proc getBlockByHash( - proxy: VerifiedRpcProxy, blockHash: Hash32 -): results.Opt[BlockObject] {.raises: [ValueError].} = - checkPreconditions(proxy) - proxy.blockCache.getPayloadByHash(blockHash) - -proc getBlockByTagOrThrow( - proxy: VerifiedRpcProxy, quantityTag: BlockTag -): BlockObject {.raises: [ValueError].} = - getBlockByTag(proxy, quantityTag).valueOr: - raise newException(ValueError, "No block stored for given tag " & $quantityTag) - -proc getBlockByHashOrThrow( - proxy: VerifiedRpcProxy, blockHash: Hash32 -): BlockObject {.raises: [ValueError].} = - getBlockByHash(proxy, blockHash).valueOr: - raise newException(ValueError, "No block stored for given hash " & $blockHash) - -proc getBlockHeaderByTagOrThrow( - proxy: VerifiedRpcProxy, quantityTag: BlockTag -): Header {.raises: [ValueError].} = - let blk = getBlockByTag(proxy, quantityTag).valueOr: - raise newException(ValueError, "No block stored for given tag " & $quantityTag) - - return Header( - parentHash: blk.parentHash, - ommersHash: blk.sha3Uncles, - coinbase: blk.miner, - stateRoot: blk.stateRoot, - transactionsRoot: blk.transactionsRoot, - receiptsRoot: blk.receiptsRoot, - logsBloom: blk.logsBloom, - difficulty: blk.difficulty, - number: distinctBase(blk.number), - gasLimit: distinctBase(blk.gasLimit), - gasUsed: distinctBase(blk.gasUsed), - timestamp: blk.timestamp.ethTime, - extraData: distinctBase(blk.extraData), - mixHash: Bytes32(distinctBase(blk.mixHash)), - nonce: blk.nonce.get, - baseFeePerGas: blk.baseFeePerGas, - withdrawalsRoot: blk.withdrawalsRoot, - blobGasUsed: blk.blobGasUsed.u64, - excessBlobGas: blk.excessBlobGas.u64, - parentBeaconBlockRoot: blk.parentBeaconBlockRoot, - requestsHash: blk.requestsHash - ) - -proc getBlockHeaderByHashOrThrow( - proxy: VerifiedRpcProxy, blockHash: Hash32 +# TODO: pull a header from the RPC if not in cache +proc getHeaderByHash( + self: VerifiedRpcProxy, blockHash: Hash32 ): Header {.raises: [ValueError].} = - let blk = getBlockByHash(proxy, blockHash).valueOr: - raise newException(ValueError, "No block stored for given hash " & $blockHash) - - return Header( - parentHash: blk.parentHash, - ommersHash: blk.sha3Uncles, - coinbase: blk.miner, - stateRoot: blk.stateRoot, - transactionsRoot: blk.transactionsRoot, - receiptsRoot: blk.receiptsRoot, - logsBloom: blk.logsBloom, - difficulty: blk.difficulty, - number: distinctBase(blk.number), - gasLimit: distinctBase(blk.gasLimit), - gasUsed: distinctBase(blk.gasUsed), - timestamp: blk.timestamp.ethTime, - extraData: distinctBase(blk.extraData), - mixHash: Bytes32(distinctBase(blk.mixHash)), - nonce: blk.nonce.get, - baseFeePerGas: blk.baseFeePerGas, - withdrawalsRoot: blk.withdrawalsRoot, - blobGasUsed: blk.blobGasUsed.u64, - excessBlobGas: blk.excessBlobGas.u64, - parentBeaconBlockRoot: blk.parentBeaconBlockRoot, - requestsHash: blk.requestsHash - ) - -proc getAccount(lcProxy: VerifiedRpcProxy, address: Address, quantityTag: BlockTag): Future[Account] {.async: (raises: [ValueError, CatchableError]).} = - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 + self.checkPreconditions() + self.headerStore.get(blockHash).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockHash) +# TODO: pull a header from the RPC if not in cache +proc getHeaderByTag( + self: VerifiedRpcProxy, blockTag: BlockTag +): Header {.raises: [ValueError].} = + let n = self.resolveTag(blockTag) + self.headerStore.get(n).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + +proc getAccount( + lcProxy: VerifiedRpcProxy, + address: addresses.Address, + blockNumber: base.BlockNumber, + stateRoot: Root +): Future[Account] {.async: (raises: [ValueError, CatchableError]).} = let proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) account = getAccountFromProof( - blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, + stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, proof.storageHash, proof.accountProof, ).valueOr: raise newException(ValueError, error) return account -proc getCode(lcProxy: VerifiedRpcProxy, address: Address, quantityTag: BlockTag): Future[seq[byte]] {.async: (raises: [ValueError, CatchableError]).} = - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 - account = await lcProxy.getAccount(address, quantityTag) - - info "Forwarding eth_getCode", blockNumber +proc getCode( + lcProxy: VerifiedRpcProxy, + address: addresses.Address, + blockNumber: base.BlockNumber, + stateRoot: Root +): Future[seq[byte]] {.async: (raises: [ValueError, CatchableError]).} = + # get verified account details for the address at blockNumber + let account = await lcProxy.getAccount(address, blockNumber, stateRoot) + # if the account does not have any code, return empty hex data if account.codeHash == EMPTY_CODE_HASH: - # account does not have any code, return empty hex data return @[] + info "Forwarding eth_getCode", blockNumber + let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber)) + # verify the byte code. since we verified the account against + # the state root we just need to verify the code hash if isValidCode(account, code): return code else: raise newException(ValueError, "received code doesn't match the account code hash") -proc getStorageAt(lcProxy: VerifiedRpcProxy, address: Address, slot: UInt256, quantityTag: BlockTag): Future[UInt256] {.async: (raises: [ValueError, CatchableError]).} = - let - blk = lcProxy.getBlockByTagOrThrow(quantityTag) - blockNumber = blk.number.uint64 +proc getStorageAt( + lcProxy: VerifiedRpcProxy, + address: addresses.Address, + slot: UInt256, + blockNumber: base.BlockNumber, + stateRoot: Root +): Future[UInt256] {.async: (raises: [ValueError, CatchableError]).} = info "Forwarding eth_getStorageAt", blockNumber let proof = await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber)) - slotValue = getStorageData(blk.stateRoot, slot, proof).valueOr: + slotValue = getStorageData(stateRoot, slot, proof).valueOr: raise newException(ValueError, error) slotValue @@ -208,54 +137,80 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = lcProxy.proxy.rpc("eth_chainId") do() -> UInt256: lcProxy.chainId - lcProxy.proxy.rpc("eth_blockNumber") do() -> uint64: - ## Returns the number of the most recent block. - let latest = lcProxy.blockCache.latest.valueOr: + lcProxy.proxy.rpc("eth_blockNumber") do() -> Quantity: + # Returns the number of the most recent block seen by the light client. + lcProxy.checkPreconditions() + + let hLatest = lcProxy.headerStore.latest() + if hLatest.isNone: raise newException(ValueError, "Syncing") - latest.number.uint64 + return Quantity(hLatest.get().number) lcProxy.proxy.rpc("eth_getBalance") do( - address: Address, quantityTag: BlockTag + address: addresses.Address, blockTag: BlockTag ) -> UInt256: - let account = await lcProxy.getAccount(address, quantityTag) + let + blockNumber = lcProxy.resolveTag(blockTag) + header = lcProxy.headerStore.get(blockNumber).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + account = await lcProxy.getAccount(address, blockNumber, header.stateRoot) + account.balance lcProxy.proxy.rpc("eth_getStorageAt") do( - address: Address, slot: UInt256, quantityTag: BlockTag + address: addresses.Address, slot: UInt256, blockTag: BlockTag ) -> UInt256: - await lcProxy.getStorageAt(address, slot, quantityTag) + let + blockNumber = lcProxy.resolveTag(blockTag) + header = lcProxy.headerStore.get(blockNumber).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + + await lcProxy.getStorageAt(address, slot, blockNumber, header.stateRoot) lcProxy.proxy.rpc("eth_getTransactionCount") do( - address: Address, quantityTag: BlockTag + address: addresses.Address, blockTag: BlockTag ) -> Quantity: - let account = await lcProxy.getAccount(address, quantityTag) + let + blockNumber = lcProxy.resolveTag(blockTag) + header = lcProxy.headerStore.get(blockNumber).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + + account = await lcProxy.getAccount(address, blockNumber, header.stateRoot) + Quantity(account.nonce) lcProxy.proxy.rpc("eth_getCode") do( - address: Address, quantityTag: BlockTag + address: addresses.Address, blockTag: BlockTag ) -> seq[byte]: - await lcProxy.getCode(address, quantityTag) + let + blockNumber = lcProxy.resolveTag(blockTag) + header = lcProxy.headerStore.get(blockNumber).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + + await lcProxy.getCode(address, blockNumber, header.stateRoot) lcProxy.proxy.rpc("eth_call") do( - args: TransactionArgs, quantityTag: BlockTag + args: TransactionArgs, blockTag: BlockTag ) -> seq[byte]: # eth_call # 1. get the code with proof - let to = if args.to.isSome(): args.to.get() - else: raise newException(ValueError, "contract address missing in transaction args") + let + to = if args.to.isSome(): args.to.get() + else: raise newException(ValueError, "contract address missing in transaction args") + blockNumber = lcProxy.resolveTag(blockTag) + header = lcProxy.headerStore.get(blockNumber).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + code = await lcProxy.getCode(to, blockNumber, header.stateRoot) # 2. get all storage locations that are accessed let - code = await lcProxy.getCode(to, quantityTag) - header = lcProxy.getBlockHeaderByTagOrThrow(quantityTag) - blkNumber = header.number.uint64 - parent = lcProxy.getBlockHeaderByHashOrThrow(header.parentHash) - accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(blkNumber)) - - let accessList = if not accessListResult.error.isSome(): accessListResult.accessList - else: raise newException(ValueError, "couldn't get an access list for eth call") + parent = lcProxy.headerStore.get(header.parentHash).valueOr: + raise newException(ValueError, "No block stored for given tag " & $blockTag) + accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(blockNumber)) + accessList = if not accessListResult.error.isSome(): accessListResult.accessList + else: raise newException(ValueError, "couldn't get an access list for eth call") # 3. pull the storage values that are access along with their accounts and initialize db let @@ -268,16 +223,17 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = for accessPair in accessList: let accountAddr = accessPair.address - acc = await lcProxy.getAccount(accountAddr, quantityTag) - accCode = await lcProxy.getCode(accountAddr, quantityTag) + acc = await lcProxy.getAccount(accountAddr, blockNumber, header.stateRoot) + accCode = await lcProxy.getCode(accountAddr, blockNumber, header.stateRoot) db.setNonce(accountAddr, acc.nonce) db.setBalance(accountAddr, acc.balance) db.setCode(accountAddr, accCode) for slot in accessPair.storageKeys: - let slotInt = UInt256.fromHex(toHex(slot)) - let slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, quantityTag) + let + slotInt = UInt256.fromHex(toHex(slot)) + slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, blockNumber, header.stateRoot) db.setStorage(accountAddr, slotInt, slotValue) db.persist(clearEmptyAccount = false) # settle accounts storage @@ -296,20 +252,20 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = # TODO currently we do not handle fullTransactions flag. It require updates on # nim-web3 side - lcProxy.proxy.rpc("eth_getBlockByNumber") do( - quantityTag: BlockTag, fullTransactions: bool - ) -> Opt[BlockObject]: - lcProxy.getBlockByTag(quantityTag) - - lcProxy.proxy.rpc("eth_getBlockByHash") do( - blockHash: Hash32, fullTransactions: bool - ) -> Opt[BlockObject]: - lcProxy.blockCache.getPayloadByHash(blockHash) +# lcProxy.proxy.rpc("eth_getBlockByNumber") do( +# blockTag: BlockTag, fullTransactions: bool +# ) -> Opt[BlockObject]: +# lcProxy.getBlockByTag(blockTag) +# +# lcProxy.proxy.rpc("eth_getBlockByHash") do( +# blockHash: Hash32, fullTransactions: bool +# ) -> Opt[BlockObject]: +# lcProxy.blockCache.getPayloadByHash(blockHash) proc new*( - T: type VerifiedRpcProxy, proxy: RpcProxy, blockCache: BlockCache, chainId: UInt256 + T: type VerifiedRpcProxy, proxy: RpcProxy, headerStore: HeaderStore, chainId: Quantity ): T = - VerifiedRpcProxy(proxy: proxy, blockCache: blockCache, chainId: chainId) + VerifiedRpcProxy(proxy: proxy, headerStore: headerStore, chainId: chainId) # Used to be in eth1_monitor.nim; not sure why it was deleted, # so I copied it here. --Adam From 87f80e38ae03a129df3de99d6f3036a55b7f9092 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 4 Mar 2025 01:45:14 +0530 Subject: [PATCH 04/11] fixes --- nimbus_verified_proxy/nimbus_verified_proxy.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nimbus_verified_proxy/nimbus_verified_proxy.nim b/nimbus_verified_proxy/nimbus_verified_proxy.nim index 8a4347b0a..bdc87f3dc 100644 --- a/nimbus_verified_proxy/nimbus_verified_proxy.nim +++ b/nimbus_verified_proxy/nimbus_verified_proxy.nim @@ -55,7 +55,6 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): UInt256 = proc run*( config: VerifiedProxyConf, ctx: ptr Context ) {.raises: [CatchableError], gcsafe.} = - # Required as both Eth2Node and LightClient requires correct config type {.gcsafe.}: setupLogging(config.logLevel, config.logStdout, none(OutFile)) @@ -140,6 +139,11 @@ proc run*( LightClientFinalizationMode.Optimistic, ) + # find out what this does + network.registerProtocol( + PeerSync, PeerSync.NetworkState.init( + cfg, forkDigests, genesisBlockRoot, getBeaconTime)) + # start the p2p network and rpcProxy waitFor network.startListening() waitFor network.start() @@ -176,6 +180,7 @@ proc run*( lightClient.onFinalizedHeader = onFinalizedHeader lightClient.onOptimisticHeader = onOptimisticHeader lightClient.trustedBlockRoot = some config.trustedBlockRoot + lightClient.installMessageValidators() func shouldSyncOptimistically(wallSlot: Slot): bool = let optimisticHeader = lightClient.optimisticHeader From 55beea97a3c86bd5c71a08b2cbf8eb2239dcae69 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 14 Mar 2025 10:43:57 +0530 Subject: [PATCH 05/11] download headers on the fly --- nimbus_verified_proxy/header_store.nim | 10 ++ nimbus_verified_proxy/rpc/rpc_eth_api.nim | 153 ++++++++++++++++------ 2 files changed, 126 insertions(+), 37 deletions(-) diff --git a/nimbus_verified_proxy/header_store.nim b/nimbus_verified_proxy/header_store.nim index 7cf8d976c..670db20d5 100644 --- a/nimbus_verified_proxy/header_store.nim +++ b/nimbus_verified_proxy/header_store.nim @@ -107,6 +107,16 @@ proc latest*(self: HeaderStore): Opt[Header] = Opt.none(Header) +proc earliest*(self: HeaderStore): Opt[Header] = + if self.headers.len() == 0: + return Opt.none(Header) + + var hash: Hash32 + for h in self.headers.keys: + hash = h + + self.headers.peek(hash) + proc get*(self: HeaderStore, number: base.BlockNumber): Opt[Header] = let hash = try: diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index b0d0504b8..5b3f3cd48 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -60,21 +60,112 @@ proc resolveTag( else: return base.BlockNumber(distinctBase(blockTag.number)) -# TODO: pull a header from the RPC if not in cache +proc convHeader(blk: BlockObject): Header = + let + nonce = if blk.nonce.isSome: blk.nonce.get + else: default(Bytes8) + + return Header( + parentHash: blk.parentHash, + ommersHash: blk.sha3Uncles, + coinbase: blk.miner, + stateRoot: blk.stateRoot, + transactionsRoot: blk.transactionsRoot, + receiptsRoot: blk.receiptsRoot, + logsBloom: blk.logsBloom, + difficulty: blk.difficulty, + number: base.BlockNumber(distinctBase(blk.number)), + gasLimit: GasInt(blk.gasLimit.uint64), + gasUsed: GasInt(blk.gasUsed.uint64), + timestamp: ethTime(blk.timestamp), + extraData: seq[byte](blk.extraData), + mixHash: Bytes32(distinctBase(blk.mixHash)), + nonce: nonce, + baseFeePerGas: blk.baseFeePerGas, + withdrawalsRoot: blk.withdrawalsRoot, + blobGasUsed: blk.blobGasUsed.u64, + excessBlobGas: blk.excessBlobGas.u64, + parentBeaconBlockRoot: blk.parentBeaconBlockRoot, + requestsHash: blk.requestsHash + ) + +proc walkBlocks( + self: VerifiedRpcProxy, + sourceNum: base.BlockNumber, + targetNum: base.BlockNumber, + sourceHash: Hash32, + targetHash: Hash32): Future[bool] {.async: (raises: [ValueError, CatchableError]).} = + + var nextHash = sourceHash + info "starting block walk to verify", blockHash=targetHash + + # TODO: use batch calls to get all blocks at once by number + for i in 0 ..< sourceNum - targetNum: + # TODO: use a verified hash cache + let blk = await self.rpcClient.eth_getBlockByHash(nextHash, false) + info "getting next block", hash=nextHash, number=blk.number, remaining=sourceNum-base.BlockNumber(distinctBase(blk.number)) + + if blk.parentHash == targetHash: + return true + + nextHash = blk.parentHash + + return false + proc getHeaderByHash( self: VerifiedRpcProxy, blockHash: Hash32 -): Header {.raises: [ValueError].} = +): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = self.checkPreconditions() - self.headerStore.get(blockHash).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockHash) + let cachedHeader = self.headerStore.get(blockHash) + + if cachedHeader.isNone(): + debug "did not find the header in the cache", blockHash=blockHash + else: + return cachedHeader.get() + + # get the source block + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # get the target block + let blk = await self.rpcClient.eth_getBlockByHash(blockHash, false) + let header = convHeader(blk) + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blockHash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + return header -# TODO: pull a header from the RPC if not in cache proc getHeaderByTag( self: VerifiedRpcProxy, blockTag: BlockTag -): Header {.raises: [ValueError].} = - let n = self.resolveTag(blockTag) - self.headerStore.get(n).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) +): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = + let + n = self.resolveTag(blockTag) + cachedHeader = self.headerStore.get(n) + + if cachedHeader.isNone(): + debug "did not find the header in the cache", blockTag=blockTag + else: + return cachedHeader.get() + + # get the source block + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # get the target block + let blk = await self.rpcClient.eth_getBlockByNumber(blockTag, false) + let header = convHeader(blk) + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blk.hash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + return header proc getAccount( lcProxy: VerifiedRpcProxy, @@ -151,32 +242,25 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = address: addresses.Address, blockTag: BlockTag ) -> UInt256: let - blockNumber = lcProxy.resolveTag(blockTag) - header = lcProxy.headerStore.get(blockNumber).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) - account = await lcProxy.getAccount(address, blockNumber, header.stateRoot) + header = await lcProxy.getHeaderByTag(blockTag) + account = await lcProxy.getAccount(address, header.number, header.stateRoot) account.balance lcProxy.proxy.rpc("eth_getStorageAt") do( address: addresses.Address, slot: UInt256, blockTag: BlockTag ) -> UInt256: - let - blockNumber = lcProxy.resolveTag(blockTag) - header = lcProxy.headerStore.get(blockNumber).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) + let header = await lcProxy.getHeaderByTag(blockTag) + - await lcProxy.getStorageAt(address, slot, blockNumber, header.stateRoot) + await lcProxy.getStorageAt(address, slot, header.number, header.stateRoot) lcProxy.proxy.rpc("eth_getTransactionCount") do( address: addresses.Address, blockTag: BlockTag ) -> Quantity: let - blockNumber = lcProxy.resolveTag(blockTag) - header = lcProxy.headerStore.get(blockNumber).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) - - account = await lcProxy.getAccount(address, blockNumber, header.stateRoot) + header = await lcProxy.getHeaderByTag(blockTag) + account = await lcProxy.getAccount(address, header.number, header.stateRoot) Quantity(account.nonce) @@ -184,11 +268,9 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = address: addresses.Address, blockTag: BlockTag ) -> seq[byte]: let - blockNumber = lcProxy.resolveTag(blockTag) - header = lcProxy.headerStore.get(blockNumber).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) + header = await lcProxy.getHeaderByTag(blockTag) - await lcProxy.getCode(address, blockNumber, header.stateRoot) + await lcProxy.getCode(address,header.number, header.stateRoot) lcProxy.proxy.rpc("eth_call") do( args: TransactionArgs, blockTag: BlockTag @@ -199,16 +281,13 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = let to = if args.to.isSome(): args.to.get() else: raise newException(ValueError, "contract address missing in transaction args") - blockNumber = lcProxy.resolveTag(blockTag) - header = lcProxy.headerStore.get(blockNumber).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) - code = await lcProxy.getCode(to, blockNumber, header.stateRoot) + header = await lcProxy.getHeaderByTag(blockTag) + code = await lcProxy.getCode(to, header.number, header.stateRoot) # 2. get all storage locations that are accessed let - parent = lcProxy.headerStore.get(header.parentHash).valueOr: - raise newException(ValueError, "No block stored for given tag " & $blockTag) - accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(blockNumber)) + parent = await lcProxy.getHeaderByHash(header.parentHash) + accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(header.number)) accessList = if not accessListResult.error.isSome(): accessListResult.accessList else: raise newException(ValueError, "couldn't get an access list for eth call") @@ -223,8 +302,8 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = for accessPair in accessList: let accountAddr = accessPair.address - acc = await lcProxy.getAccount(accountAddr, blockNumber, header.stateRoot) - accCode = await lcProxy.getCode(accountAddr, blockNumber, header.stateRoot) + acc = await lcProxy.getAccount(accountAddr, header.number, header.stateRoot) + accCode = await lcProxy.getCode(accountAddr, header.number, header.stateRoot) db.setNonce(accountAddr, acc.nonce) db.setBalance(accountAddr, acc.balance) @@ -233,7 +312,7 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = for slot in accessPair.storageKeys: let slotInt = UInt256.fromHex(toHex(slot)) - slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, blockNumber, header.stateRoot) + slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, header.number, header.stateRoot) db.setStorage(accountAddr, slotInt, slotValue) db.persist(clearEmptyAccount = false) # settle accounts storage From b7bd704ba9b29efab92383b29947629fb938a22d Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 14 Mar 2025 11:31:59 +0530 Subject: [PATCH 06/11] verify headers --- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 5b3f3cd48..860b7c05a 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -103,7 +103,7 @@ proc walkBlocks( for i in 0 ..< sourceNum - targetNum: # TODO: use a verified hash cache let blk = await self.rpcClient.eth_getBlockByHash(nextHash, false) - info "getting next block", hash=nextHash, number=blk.number, remaining=sourceNum-base.BlockNumber(distinctBase(blk.number)) + info "getting next block", hash=nextHash, number=blk.number, remaining=distinctBase(blk.number) - targetNum if blk.parentHash == targetHash: return true @@ -131,6 +131,13 @@ proc getHeaderByHash( let blk = await self.rpcClient.eth_getBlockByHash(blockHash, false) let header = convHeader(blk) + # verify header hash + if header.rlpHash != blk.hash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if blockHash != blk.hash: + raise newException(ValueError, "the blk.hash(downloaded) doesn't match with the provided hash") + # walk blocks backwards(time) from source to target let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blockHash) @@ -159,6 +166,13 @@ proc getHeaderByTag( let blk = await self.rpcClient.eth_getBlockByNumber(blockTag, false) let header = convHeader(blk) +# verify header hash + if header.rlpHash != blk.hash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if blockHash != blk.hash: + raise newException(ValueError, "the blk.hash(downloaded) doesn't match with the provided hash") + # walk blocks backwards(time) from source to target let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blk.hash) From 80df93a7a9dac6b39c7b4f785d93cb8e3b431e25 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 19 Mar 2025 14:59:08 +0530 Subject: [PATCH 07/11] reorg --- nimbus_verified_proxy/header_store.nim | 2 +- .../nimbus_verified_proxy.nim | 1 + nimbus_verified_proxy/rpc/accounts.nim | 158 +++++++++ nimbus_verified_proxy/rpc/blocks.nim | 162 ++++++++++ nimbus_verified_proxy/rpc/rpc_eth_api.nim | 301 +++--------------- nimbus_verified_proxy/types.nim | 22 ++ 6 files changed, 388 insertions(+), 258 deletions(-) create mode 100644 nimbus_verified_proxy/rpc/accounts.nim create mode 100644 nimbus_verified_proxy/rpc/blocks.nim create mode 100644 nimbus_verified_proxy/types.nim diff --git a/nimbus_verified_proxy/header_store.nim b/nimbus_verified_proxy/header_store.nim index 670db20d5..56a8d3ac6 100644 --- a/nimbus_verified_proxy/header_store.nim +++ b/nimbus_verified_proxy/header_store.nim @@ -14,7 +14,7 @@ import std/tables, beacon_chain/spec/beaconstate, beacon_chain/spec/datatypes/[phase0, altair, bellatrix], - beacon_chain/[light_client, nimbus_binary_common, version], + beacon_chain/[light_client, nimbus_binary_common], beacon_chain/el/engine_api_conversions, minilru, results diff --git a/nimbus_verified_proxy/nimbus_verified_proxy.nim b/nimbus_verified_proxy/nimbus_verified_proxy.nim index bdc87f3dc..2963cee50 100644 --- a/nimbus_verified_proxy/nimbus_verified_proxy.nim +++ b/nimbus_verified_proxy/nimbus_verified_proxy.nim @@ -23,6 +23,7 @@ import beacon_chain/[light_client, nimbus_binary_common, version], ../execution_chain/rpc/cors, ./rpc/rpc_eth_api, + ./types, ./nimbus_verified_proxy_conf, ./header_store diff --git a/nimbus_verified_proxy/rpc/accounts.nim b/nimbus_verified_proxy/rpc/accounts.nim new file mode 100644 index 000000000..4f4c89ec7 --- /dev/null +++ b/nimbus_verified_proxy/rpc/accounts.nim @@ -0,0 +1,158 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/sequtils, + stint, + results, + chronicles, + eth/common/[base_rlp, accounts_rlp, hashes_rlp], + ../../execution_chain/beacon/web3_eth_conv, + eth/common/addresses, + eth/common/eth_types_rlp, + eth/trie/[hexary_proof_verification], + json_rpc/[rpcproxy, rpcserver, rpcclient], + web3/[primitives, eth_api_types, eth_api], + ../types, + ../header_store + +export results, stint, hashes_rlp, accounts_rlp, eth_api_types + +template rpcClient(vp: VerifiedRpcProxy): RpcClient = + vp.proxy.getClient() + +proc getAccountFromProof( + stateRoot: Hash32, + accountAddress: Address, + accountBalance: UInt256, + accountNonce: Quantity, + accountCodeHash: Hash32, + accountStorageRoot: Hash32, + mptNodes: seq[RlpEncodedBytes], +): Result[Account, string] = + let + mptNodesBytes = mptNodes.mapIt(distinctBase(it)) + acc = Account( + nonce: distinctBase(accountNonce), + balance: accountBalance, + storageRoot: accountStorageRoot, + codeHash: accountCodeHash, + ) + accountEncoded = rlp.encode(acc) + accountKey = toSeq(keccak256((accountAddress.data)).data) + + let proofResult = verifyMptProof(mptNodesBytes, stateRoot, accountKey, accountEncoded) + + case proofResult.kind + of MissingKey: + return ok(EMPTY_ACCOUNT) + of ValidProof: + return ok(acc) + of InvalidProof: + return err(proofResult.errorMsg) + +proc getStorageFromProof( + account: Account, storageProof: StorageProof +): Result[UInt256, string] = + let + storageMptNodes = storageProof.proof.mapIt(distinctBase(it)) + key = toSeq(keccak256(toBytesBE(storageProof.key)).data) + encodedValue = rlp.encode(storageProof.value) + proofResult = + verifyMptProof(storageMptNodes, account.storageRoot, key, encodedValue) + + case proofResult.kind + of MissingKey: + return ok(UInt256.zero) + of ValidProof: + return ok(storageProof.value) + of InvalidProof: + return err(proofResult.errorMsg) + +proc getStorageFromProof( + stateRoot: Hash32, requestedSlot: UInt256, proof: ProofResponse +): Result[UInt256, string] = + let account = + ?getAccountFromProof( + stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, + proof.storageHash, proof.accountProof, + ) + + if account.storageRoot == EMPTY_ROOT_HASH: + # valid account with empty storage, in that case getStorageAt + # return 0 value + return ok(u256(0)) + + if len(proof.storageProof) != 1: + return err("no storage proof for requested slot") + + let storageProof = proof.storageProof[0] + + if len(storageProof.proof) == 0: + return err("empty mpt proof for account with not empty storage") + + if storageProof.key != requestedSlot: + return err("received proof for invalid slot") + + getStorageFromProof(account, storageProof) + +proc getAccount*( + lcProxy: VerifiedRpcProxy, + address: Address, + blockNumber: base.BlockNumber, + stateRoot: Root): Future[Account] {.async: (raises: [ValueError, CatchableError]).} = + let + proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) + account = getAccountFromProof( + stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, + proof.storageHash, proof.accountProof, + ).valueOr: + raise newException(ValueError, error) + + return account + +proc getCode*( + lcProxy: VerifiedRpcProxy, + address: Address, + blockNumber: base.BlockNumber, + stateRoot: Root): Future[seq[byte]] {.async: (raises: [ValueError, CatchableError]).} = + # get verified account details for the address at blockNumber + let account = await lcProxy.getAccount(address, blockNumber, stateRoot) + + # if the account does not have any code, return empty hex data + if account.codeHash == EMPTY_CODE_HASH: + return @[] + + info "Forwarding eth_getCode", blockNumber + + let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber)) + + # verify the byte code. since we verified the account against + # the state root we just need to verify the code hash + if account.codeHash == keccak256(code): + return code + else: + raise newException(ValueError, "received code doesn't match the account code hash") + +proc getStorageAt*( + lcProxy: VerifiedRpcProxy, + address: Address, + slot: UInt256, + blockNumber: base.BlockNumber, + stateRoot: Root +): Future[UInt256] {.async: (raises: [ValueError, CatchableError]).} = + + info "Forwarding eth_getStorageAt", blockNumber + + let + proof = await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber)) + slotValue = getStorageFromProof(stateRoot, slot, proof).valueOr: + raise newException(ValueError, error) + + slotValue diff --git a/nimbus_verified_proxy/rpc/blocks.nim b/nimbus_verified_proxy/rpc/blocks.nim new file mode 100644 index 000000000..b35539b85 --- /dev/null +++ b/nimbus_verified_proxy/rpc/blocks.nim @@ -0,0 +1,162 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/strutils, + results, + chronicles, + web3/[primitives, eth_api_types, eth_api], + json_rpc/[rpcproxy, rpcserver, rpcclient], + eth/common/addresses, + eth/common/eth_types_rlp, + ../../execution_chain/beacon/web3_eth_conv, + ../types, + ../header_store + +type BlockTag* = eth_api_types.RtBlockIdentifier + +template rpcClient(vp: VerifiedRpcProxy): RpcClient = + vp.proxy.getClient() + +proc resolveTag( + self: VerifiedRpcProxy, blockTag: BlockTag +): base.BlockNumber {.raises: [ValueError].} = + if blockTag.kind == bidAlias: + let tag = blockTag.alias.toLowerAscii() + case tag + of "latest": + let hLatest = self.headerStore.latest() + if hLatest.isSome: + return hLatest.get().number + else: + raise newException(ValueError, "Couldn't get the latest block number from header store") + else: + raise newException(ValueError, "No support for block tag " & $blockTag) + else: + return base.BlockNumber(distinctBase(blockTag.number)) + +proc convHeader(blk: BlockObject): Header = + let + nonce = if blk.nonce.isSome: blk.nonce.get + else: default(Bytes8) + + return Header( + parentHash: blk.parentHash, + ommersHash: blk.sha3Uncles, + coinbase: blk.miner, + stateRoot: blk.stateRoot, + transactionsRoot: blk.transactionsRoot, + receiptsRoot: blk.receiptsRoot, + logsBloom: blk.logsBloom, + difficulty: blk.difficulty, + number: base.BlockNumber(distinctBase(blk.number)), + gasLimit: GasInt(blk.gasLimit.uint64), + gasUsed: GasInt(blk.gasUsed.uint64), + timestamp: ethTime(blk.timestamp), + extraData: seq[byte](blk.extraData), + mixHash: Bytes32(distinctBase(blk.mixHash)), + nonce: nonce, + baseFeePerGas: blk.baseFeePerGas, + withdrawalsRoot: blk.withdrawalsRoot, + blobGasUsed: blk.blobGasUsed.u64, + excessBlobGas: blk.excessBlobGas.u64, + parentBeaconBlockRoot: blk.parentBeaconBlockRoot, + requestsHash: blk.requestsHash + ) + +proc walkBlocks( + self: VerifiedRpcProxy, + sourceNum: base.BlockNumber, + targetNum: base.BlockNumber, + sourceHash: Hash32, + targetHash: Hash32): Future[bool] {.async: (raises: [ValueError, CatchableError]).} = + + var nextHash = sourceHash + info "starting block walk to verify", blockHash=targetHash + + # TODO: use batch calls to get all blocks at once by number + for i in 0 ..< sourceNum - targetNum: + # TODO: use a verified hash cache + let blk = await self.rpcClient.eth_getBlockByHash(nextHash, false) + info "getting next block", hash=nextHash, number=blk.number, remaining=distinctBase(blk.number) - targetNum + + if blk.parentHash == targetHash: + return true + + nextHash = blk.parentHash + + return false + +proc getHeaderByHash*( + self: VerifiedRpcProxy, blockHash: Hash32 +): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = + let cachedHeader = self.headerStore.get(blockHash) + + if cachedHeader.isNone(): + debug "did not find the header in the cache", blockHash=blockHash + else: + return cachedHeader.get() + + # get the source block + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # get the target block + let blk = await self.rpcClient.eth_getBlockByHash(blockHash, false) + let header = convHeader(blk) + + # verify header hash + if header.rlpHash != blk.hash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if blockHash != blk.hash: + raise newException(ValueError, "the blk.hash(downloaded) doesn't match with the provided hash") + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blockHash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + return header + +proc getHeaderByTag*( + self: VerifiedRpcProxy, blockTag: BlockTag +): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = + let + n = self.resolveTag(blockTag) + cachedHeader = self.headerStore.get(n) + + if cachedHeader.isNone(): + debug "did not find the header in the cache", blockTag=blockTag + else: + return cachedHeader.get() + + # get the source block + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # get the target block + let blk = await self.rpcClient.eth_getBlockByNumber(blockTag, false) + let header = convHeader(blk) + + # verify header hash + if header.rlpHash != blk.hash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if n != header.number: + raise newException(ValueError, "the downloaded block number doesn't match with the requested block number") + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blk.hash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + return header diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 860b7c05a..110eb040d 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -21,272 +21,64 @@ import ../../execution_chain/transaction/call_evm, ../../execution_chain/[evm/types, evm/state], ../validate_proof, - ../header_store + ../header_store, + ../types, + ./blocks, + ./accounts logScope: topics = "verified_proxy" -type - VerifiedRpcProxy* = ref object - proxy: RpcProxy - headerStore: HeaderStore - chainId: Quantity - - BlockTag = eth_api_types.RtBlockIdentifier - -template checkPreconditions(proxy: VerifiedRpcProxy) = - if proxy.headerStore.isEmpty(): - raise newException(ValueError, "Syncing") - -template rpcClient(lcProxy: VerifiedRpcProxy): RpcClient = - lcProxy.proxy.getClient() - -proc resolveTag( - self: VerifiedRpcProxy, blockTag: BlockTag -): base.BlockNumber {.raises: [ValueError].} = - self.checkPreconditions() - - if blockTag.kind == bidAlias: - let tag = blockTag.alias.toLowerAscii() - case tag - of "latest": - let hLatest = self.headerStore.latest() - if hLatest.isSome: - return hLatest.get().number - else: - raise newException(ValueError, "No block stored for given tag " & $blockTag) - else: - raise newException(ValueError, "No support for block tag " & $blockTag) - else: - return base.BlockNumber(distinctBase(blockTag.number)) - -proc convHeader(blk: BlockObject): Header = - let - nonce = if blk.nonce.isSome: blk.nonce.get - else: default(Bytes8) - - return Header( - parentHash: blk.parentHash, - ommersHash: blk.sha3Uncles, - coinbase: blk.miner, - stateRoot: blk.stateRoot, - transactionsRoot: blk.transactionsRoot, - receiptsRoot: blk.receiptsRoot, - logsBloom: blk.logsBloom, - difficulty: blk.difficulty, - number: base.BlockNumber(distinctBase(blk.number)), - gasLimit: GasInt(blk.gasLimit.uint64), - gasUsed: GasInt(blk.gasUsed.uint64), - timestamp: ethTime(blk.timestamp), - extraData: seq[byte](blk.extraData), - mixHash: Bytes32(distinctBase(blk.mixHash)), - nonce: nonce, - baseFeePerGas: blk.baseFeePerGas, - withdrawalsRoot: blk.withdrawalsRoot, - blobGasUsed: blk.blobGasUsed.u64, - excessBlobGas: blk.excessBlobGas.u64, - parentBeaconBlockRoot: blk.parentBeaconBlockRoot, - requestsHash: blk.requestsHash - ) - -proc walkBlocks( - self: VerifiedRpcProxy, - sourceNum: base.BlockNumber, - targetNum: base.BlockNumber, - sourceHash: Hash32, - targetHash: Hash32): Future[bool] {.async: (raises: [ValueError, CatchableError]).} = - - var nextHash = sourceHash - info "starting block walk to verify", blockHash=targetHash - - # TODO: use batch calls to get all blocks at once by number - for i in 0 ..< sourceNum - targetNum: - # TODO: use a verified hash cache - let blk = await self.rpcClient.eth_getBlockByHash(nextHash, false) - info "getting next block", hash=nextHash, number=blk.number, remaining=distinctBase(blk.number) - targetNum - - if blk.parentHash == targetHash: - return true - - nextHash = blk.parentHash - - return false - -proc getHeaderByHash( - self: VerifiedRpcProxy, blockHash: Hash32 -): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = - self.checkPreconditions() - let cachedHeader = self.headerStore.get(blockHash) - - if cachedHeader.isNone(): - debug "did not find the header in the cache", blockHash=blockHash - else: - return cachedHeader.get() - - # get the source block - let earliestHeader = self.headerStore.earliest.valueOr: - raise newException(ValueError, "Syncing") - - # get the target block - let blk = await self.rpcClient.eth_getBlockByHash(blockHash, false) - let header = convHeader(blk) - - # verify header hash - if header.rlpHash != blk.hash: - raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") - - if blockHash != blk.hash: - raise newException(ValueError, "the blk.hash(downloaded) doesn't match with the provided hash") - - # walk blocks backwards(time) from source to target - let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blockHash) - - if not isLinked: - raise newException(ValueError, "the requested block is not part of the canonical chain") - - return header - -proc getHeaderByTag( - self: VerifiedRpcProxy, blockTag: BlockTag -): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = - let - n = self.resolveTag(blockTag) - cachedHeader = self.headerStore.get(n) - - if cachedHeader.isNone(): - debug "did not find the header in the cache", blockTag=blockTag - else: - return cachedHeader.get() - - # get the source block - let earliestHeader = self.headerStore.earliest.valueOr: - raise newException(ValueError, "Syncing") - - # get the target block - let blk = await self.rpcClient.eth_getBlockByNumber(blockTag, false) - let header = convHeader(blk) - -# verify header hash - if header.rlpHash != blk.hash: - raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") - - if blockHash != blk.hash: - raise newException(ValueError, "the blk.hash(downloaded) doesn't match with the provided hash") - - # walk blocks backwards(time) from source to target - let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blk.hash) - - if not isLinked: - raise newException(ValueError, "the requested block is not part of the canonical chain") - - return header - -proc getAccount( - lcProxy: VerifiedRpcProxy, - address: addresses.Address, - blockNumber: base.BlockNumber, - stateRoot: Root -): Future[Account] {.async: (raises: [ValueError, CatchableError]).} = - let - proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber)) - account = getAccountFromProof( - stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash, - proof.storageHash, proof.accountProof, - ).valueOr: - raise newException(ValueError, error) - - return account - -proc getCode( - lcProxy: VerifiedRpcProxy, - address: addresses.Address, - blockNumber: base.BlockNumber, - stateRoot: Root -): Future[seq[byte]] {.async: (raises: [ValueError, CatchableError]).} = - # get verified account details for the address at blockNumber - let account = await lcProxy.getAccount(address, blockNumber, stateRoot) - - # if the account does not have any code, return empty hex data - if account.codeHash == EMPTY_CODE_HASH: - return @[] - - info "Forwarding eth_getCode", blockNumber - - let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber)) - - # verify the byte code. since we verified the account against - # the state root we just need to verify the code hash - if isValidCode(account, code): - return code - else: - raise newException(ValueError, "received code doesn't match the account code hash") - -proc getStorageAt( - lcProxy: VerifiedRpcProxy, - address: addresses.Address, - slot: UInt256, - blockNumber: base.BlockNumber, - stateRoot: Root -): Future[UInt256] {.async: (raises: [ValueError, CatchableError]).} = - - info "Forwarding eth_getStorageAt", blockNumber - - let - proof = await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber)) - slotValue = getStorageData(stateRoot, slot, proof).valueOr: - raise newException(ValueError, error) - - slotValue - -proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = - lcProxy.proxy.rpc("eth_chainId") do() -> UInt256: - lcProxy.chainId - - lcProxy.proxy.rpc("eth_blockNumber") do() -> Quantity: - # Returns the number of the most recent block seen by the light client. - lcProxy.checkPreconditions() +template rpcClient*(vp: VerifiedRpcProxy): RpcClient = + vp.proxy.getClient() + +proc installEthApiHandlers*(vp: VerifiedRpcProxy) = + vp.proxy.rpc("eth_chainId") do() -> Quantity: + vp.chainId - let hLatest = lcProxy.headerStore.latest() + # eth_blockNumber - get latest tag from header store + vp.proxy.rpc("eth_blockNumber") do() -> Quantity: + # Returns the number of the most recent block seen by the light client. + let hLatest = vp.headerStore.latest() if hLatest.isNone: raise newException(ValueError, "Syncing") return Quantity(hLatest.get().number) - lcProxy.proxy.rpc("eth_getBalance") do( + vp.proxy.rpc("eth_getBalance") do( address: addresses.Address, blockTag: BlockTag ) -> UInt256: let - header = await lcProxy.getHeaderByTag(blockTag) - account = await lcProxy.getAccount(address, header.number, header.stateRoot) + header = await vp.getHeaderByTag(blockTag) + account = await vp.getAccount(address, header.number, header.stateRoot) account.balance - lcProxy.proxy.rpc("eth_getStorageAt") do( + vp.proxy.rpc("eth_getStorageAt") do( address: addresses.Address, slot: UInt256, blockTag: BlockTag ) -> UInt256: - let header = await lcProxy.getHeaderByTag(blockTag) - + let header = await vp.getHeaderByTag(blockTag) - await lcProxy.getStorageAt(address, slot, header.number, header.stateRoot) + await vp.getStorageAt(address, slot, header.number, header.stateRoot) - lcProxy.proxy.rpc("eth_getTransactionCount") do( + vp.proxy.rpc("eth_getTransactionCount") do( address: addresses.Address, blockTag: BlockTag ) -> Quantity: let - header = await lcProxy.getHeaderByTag(blockTag) - account = await lcProxy.getAccount(address, header.number, header.stateRoot) + header = await vp.getHeaderByTag(blockTag) + account = await vp.getAccount(address, header.number, header.stateRoot) Quantity(account.nonce) - lcProxy.proxy.rpc("eth_getCode") do( + vp.proxy.rpc("eth_getCode") do( address: addresses.Address, blockTag: BlockTag ) -> seq[byte]: let - header = await lcProxy.getHeaderByTag(blockTag) + header = await vp.getHeaderByTag(blockTag) - await lcProxy.getCode(address,header.number, header.stateRoot) + await vp.getCode(address,header.number, header.stateRoot) - lcProxy.proxy.rpc("eth_call") do( + vp.proxy.rpc("eth_call") do( args: TransactionArgs, blockTag: BlockTag ) -> seq[byte]: @@ -295,13 +87,13 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = let to = if args.to.isSome(): args.to.get() else: raise newException(ValueError, "contract address missing in transaction args") - header = await lcProxy.getHeaderByTag(blockTag) - code = await lcProxy.getCode(to, header.number, header.stateRoot) + header = await vp.getHeaderByTag(blockTag) + code = await vp.getCode(to, header.number, header.stateRoot) # 2. get all storage locations that are accessed let - parent = await lcProxy.getHeaderByHash(header.parentHash) - accessListResult = await lcProxy.rpcClient.eth_createAccessList(args, blockId(header.number)) + parent = await vp.getHeaderByHash(header.parentHash) + accessListResult = await vp.rpcClient.eth_createAccessList(args, blockId(header.number)) accessList = if not accessListResult.error.isSome(): accessListResult.accessList else: raise newException(ValueError, "couldn't get an access list for eth call") @@ -316,8 +108,8 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = for accessPair in accessList: let accountAddr = accessPair.address - acc = await lcProxy.getAccount(accountAddr, header.number, header.stateRoot) - accCode = await lcProxy.getCode(accountAddr, header.number, header.stateRoot) + acc = await vp.getAccount(accountAddr, header.number, header.stateRoot) + accCode = await vp.getCode(accountAddr, header.number, header.stateRoot) db.setNonce(accountAddr, acc.nonce) db.setBalance(accountAddr, acc.balance) @@ -326,7 +118,7 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = for slot in accessPair.storageKeys: let slotInt = UInt256.fromHex(toHex(slot)) - slotValue = await lcProxy.getStorageAt(accountAddr, slotInt, header.number, header.stateRoot) + slotValue = await vp.getStorageAt(accountAddr, slotInt, header.number, header.stateRoot) db.setStorage(accountAddr, slotInt, slotValue) db.persist(clearEmptyAccount = false) # settle accounts storage @@ -339,26 +131,21 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) = # TODO: # Following methods are forwarded directly to the web3 provider and therefore # are not validated in any way. - lcProxy.proxy.registerProxyMethod("net_version") - lcProxy.proxy.registerProxyMethod("eth_sendRawTransaction") - lcProxy.proxy.registerProxyMethod("eth_getTransactionReceipt") + vp.proxy.registerProxyMethod("net_version") + vp.proxy.registerProxyMethod("eth_sendRawTransaction") + vp.proxy.registerProxyMethod("eth_getTransactionReceipt") # TODO currently we do not handle fullTransactions flag. It require updates on # nim-web3 side -# lcProxy.proxy.rpc("eth_getBlockByNumber") do( +# vp.proxy.rpc("eth_getBlockByNumber") do( # blockTag: BlockTag, fullTransactions: bool # ) -> Opt[BlockObject]: -# lcProxy.getBlockByTag(blockTag) +# vp.getBlockByTag(blockTag) # -# lcProxy.proxy.rpc("eth_getBlockByHash") do( +# vp.proxy.rpc("eth_getBlockByHash") do( # blockHash: Hash32, fullTransactions: bool # ) -> Opt[BlockObject]: -# lcProxy.blockCache.getPayloadByHash(blockHash) - -proc new*( - T: type VerifiedRpcProxy, proxy: RpcProxy, headerStore: HeaderStore, chainId: Quantity -): T = - VerifiedRpcProxy(proxy: proxy, headerStore: headerStore, chainId: chainId) +# vp.blockCache.getPayloadByHash(blockHash) # Used to be in eth1_monitor.nim; not sure why it was deleted, # so I copied it here. --Adam @@ -396,13 +183,13 @@ template awaitWithRetries*[T]( read(f) -proc verifyChaindId*(p: VerifiedRpcProxy): Future[void] {.async.} = - let localId = p.chainId +proc verifyChaindId*(vp: VerifiedRpcProxy): Future[void] {.async.} = + let localId = vp.chainId # retry 2 times, if the data provider fails despite the re-tries, propagate # exception to the caller. let providerId = - awaitWithRetries(p.rpcClient.eth_chainId(), retries = 2, timeout = seconds(30)) + awaitWithRetries(vp.rpcClient.eth_chainId(), retries = 2, timeout = seconds(30)) # This is a chain/network mismatch error between the Nimbus verified proxy and # the application using it. Fail fast to avoid misusage. The user must fix diff --git a/nimbus_verified_proxy/types.nim b/nimbus_verified_proxy/types.nim new file mode 100644 index 000000000..01cfaaa2d --- /dev/null +++ b/nimbus_verified_proxy/types.nim @@ -0,0 +1,22 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + json_rpc/[rpcproxy], + web3/[primitives, eth_api_types, eth_api], + ./header_store + +type + VerifiedRpcProxy* = ref object + proxy*: RpcProxy + headerStore*: HeaderStore + chainId*: Quantity + +proc new*( + T: type VerifiedRpcProxy, proxy: RpcProxy, headerStore: HeaderStore, chainId: Quantity +): T = + VerifiedRpcProxy(proxy: proxy, headerStore: headerStore, chainId: chainId) From b983b9a9de31cd255ab7a2aebcb4220e8cb12cce Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 25 Mar 2025 21:03:07 +0530 Subject: [PATCH 08/11] transactions support --- nimbus_verified_proxy/rpc/blocks.nim | 83 ++++++++++++++- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 93 +++++++++++++--- nimbus_verified_proxy/rpc/transactions.nim | 118 +++++++++++++++++++++ 3 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 nimbus_verified_proxy/rpc/transactions.nim diff --git a/nimbus_verified_proxy/rpc/blocks.nim b/nimbus_verified_proxy/rpc/blocks.nim index b35539b85..1e745afba 100644 --- a/nimbus_verified_proxy/rpc/blocks.nim +++ b/nimbus_verified_proxy/rpc/blocks.nim @@ -15,9 +15,11 @@ import json_rpc/[rpcproxy, rpcserver, rpcclient], eth/common/addresses, eth/common/eth_types_rlp, + eth/trie/[hexary, ordered_trie, db, trie_defs], ../../execution_chain/beacon/web3_eth_conv, ../types, - ../header_store + ../header_store, + ./transactions type BlockTag* = eth_api_types.RtBlockIdentifier @@ -93,6 +95,83 @@ proc walkBlocks( return false +proc getBlockByHash*( + self: VerifiedRpcProxy, blockHash: Hash32, fullTransactions: bool +): Future[BlockObject] {.async: (raises: [ValueError, CatchableError]).} = + # get the target block + let blk = await self.rpcClient.eth_getBlockByHash(blockHash, fullTransactions) + let header = convHeader(blk) + + # verify header hash + if header.rlpHash != blockHash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if blockHash != blk.hash: + raise newException(ValueError, "the downloaded block hash doesn't match with the requested hash") + + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blockHash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + # verify transactions + if fullTransactions: + let verified = verifyTransactions(header.transactionsRoot, blk.transactions).valueOr: + raise newException(ValueError, "error while verifying transactions root") + if not verified: + raise newException(ValueError, "transactions within the block do not yield the same transaction root") + + # verify withdrawals + if blk.withdrawals.isSome(): + if blk.withdrawalsRoot.get() != orderedTrieRoot(blk.withdrawals.get()): + raise newException(ValueError, "withdrawals within the block do not yield the same withdrawals root") + + return blk + +proc getBlockByTag*( + self: VerifiedRpcProxy, blockTag: BlockTag, fullTransactions: bool +): Future[BlockObject] {.async: (raises: [ValueError, CatchableError]).} = + let n = self.resolveTag(blockTag) + + # get the target block + let blk = await self.rpcClient.eth_getBlockByNumber(blockTag, false) + let header = convHeader(blk) + + # verify header hash + if header.rlpHash != blk.hash: + raise newException(ValueError, "hashed block header doesn't match with blk.hash(downloaded)") + + if n != header.number: + raise newException(ValueError, "the downloaded block number doesn't match with the requested block number") + + # get the source block + let earliestHeader = self.headerStore.earliest.valueOr: + raise newException(ValueError, "Syncing") + + # walk blocks backwards(time) from source to target + let isLinked = await self.walkBlocks(earliestHeader.number, header.number, earliestHeader.parentHash, blk.hash) + + if not isLinked: + raise newException(ValueError, "the requested block is not part of the canonical chain") + + # verify transactions + if fullTransactions: + let verified = verifyTransactions(header.transactionsRoot, blk.transactions).valueOr: + raise newException(ValueError, "error while verifying transactions root") + if not verified: + raise newException(ValueError, "transactions within the block do not yield the same transaction root") + + # verify withdrawals + if blk.withdrawals.isSome(): + if blk.withdrawalsRoot.get() != orderedTrieRoot(blk.withdrawals.get()): + raise newException(ValueError, "withdrawals within the block do not yield the same withdrawals root") + + return blk + proc getHeaderByHash*( self: VerifiedRpcProxy, blockHash: Hash32 ): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = @@ -129,7 +208,7 @@ proc getHeaderByHash*( proc getHeaderByTag*( self: VerifiedRpcProxy, blockTag: BlockTag ): Future[Header] {.async: (raises: [ValueError, CatchableError]).} = - let + let n = self.resolveTag(blockTag) cachedHeader = self.headerStore.get(n) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 110eb040d..ff94d02fe 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -24,7 +24,8 @@ import ../header_store, ../types, ./blocks, - ./accounts + ./accounts, + ./transactions logScope: topics = "verified_proxy" @@ -36,6 +37,84 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) = vp.proxy.rpc("eth_chainId") do() -> Quantity: vp.chainId + vp.proxy.rpc("eth_getBlockByNumber") do( + blockTag: BlockTag, fullTransactions: bool + ) -> Opt[BlockObject]: + try: + let blk = await vp.getBlockByTag(blockTag, fullTransactions) + return Opt.some(blk) + except ValueError as e: + # should return Opt.none but we also want to transmit error related info + # return Opt.none(BlockObject) + raise newException(ValueError, e.msg) # raising an exception will return the error message + + vp.proxy.rpc("eth_getBlockByHash") do( + blockHash: Hash32, fullTransactions: bool + ) -> Opt[BlockObject]: + try: + let blk = await vp.getBlockByHash(blockHash, fullTransactions) + return Opt.some(blk) + except ValueError as e: + # should return Opt.none but we also want to transmit error related info + # return Opt.none(BlockObject) + raise newException(ValueError, e.msg) # raising an exception will return the error message + + vp.proxy.rpc("eth_getUncleCountByBlockNumber") do( + blockTag: BlockTag + ) -> Quantity: + let blk = await vp.getBlockByTag(blockTag, false) + return Quantity(blk.uncles.len()) + + vp.proxy.rpc("eth_getUncleCountByBlockHash") do( + blockHash: Hash32 + ) -> Quantity: + let blk = await vp.getBlockByHash(blockHash, false) + return Quantity(blk.uncles.len()) + + vp.proxy.rpc("eth_getBlockTransactionCountByNumber") do( + blockTag: BlockTag + ) -> Quantity: + let blk = await vp.getBlockByTag(blockTag, true) + return Quantity(blk.transactions.len) + + vp.proxy.rpc("eth_getBlockTransactionCountByHash") do( + blockHash: Hash32 + ) -> Quantity: + let blk = await vp.getBlockByHash(blockHash, true) + return Quantity(blk.transactions.len) + + vp.proxy.rpc("eth_getTransactionByBlockNumberAndIndex") do( + blockTag: BlockTag, index: Quantity + ) -> TransactionObject: + let blk = await vp.getBlockByTag(blockTag, true) + if distinctBase(index) >= uint64(blk.transactions.len): + raise newException(ValueError, "provided transaction index is outside bounds") + let x = blk.transactions[distinctBase(index)] + doAssert x.kind == tohTx + return x.tx + + vp.proxy.rpc("eth_getTransactionByBlockHashAndIndex") do( + blockHash: Hash32, index: Quantity + ) -> TransactionObject: + let blk = await vp.getBlockByHash(blockHash, true) + if distinctBase(index) >= uint64(blk.transactions.len): + raise newException(ValueError, "provided transaction index is outside bounds") + let x = blk.transactions[distinctBase(index)] + doAssert x.kind == tohTx + return x.tx + + vp.proxy.rpc("eth_getTransactionByHash") do( + txHash: Hash32 + ) -> TransactionObject: + let tx = await vp.rpcClient.eth_getTransactionByHash(txHash) + if tx.hash != txHash: + raise newException(ValueError, "the downloaded transaction hash doesn't match the requested transaction hash") + + if not checkTxHash(tx, txHash): + raise newException(ValueError, "the transaction doesn't hash to the provided hash") + + return tx + # eth_blockNumber - get latest tag from header store vp.proxy.rpc("eth_blockNumber") do() -> Quantity: # Returns the number of the most recent block seen by the light client. @@ -135,18 +214,6 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) = vp.proxy.registerProxyMethod("eth_sendRawTransaction") vp.proxy.registerProxyMethod("eth_getTransactionReceipt") - # TODO currently we do not handle fullTransactions flag. It require updates on - # nim-web3 side -# vp.proxy.rpc("eth_getBlockByNumber") do( -# blockTag: BlockTag, fullTransactions: bool -# ) -> Opt[BlockObject]: -# vp.getBlockByTag(blockTag) -# -# vp.proxy.rpc("eth_getBlockByHash") do( -# blockHash: Hash32, fullTransactions: bool -# ) -> Opt[BlockObject]: -# vp.blockCache.getPayloadByHash(blockHash) - # Used to be in eth1_monitor.nim; not sure why it was deleted, # so I copied it here. --Adam template awaitWithRetries*[T]( diff --git a/nimbus_verified_proxy/rpc/transactions.nim b/nimbus_verified_proxy/rpc/transactions.nim new file mode 100644 index 000000000..4e280f537 --- /dev/null +++ b/nimbus_verified_proxy/rpc/transactions.nim @@ -0,0 +1,118 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/sequtils, + stint, + results, + chronicles, + eth/common/[base_rlp, transactions_rlp, receipts_rlp, hashes_rlp], + ../../execution_chain/beacon/web3_eth_conv, + eth/common/addresses, + eth/common/eth_types_rlp, + eth/trie/[hexary, ordered_trie, db, trie_defs], + json_rpc/[rpcproxy, rpcserver, rpcclient], + web3/[primitives, eth_api_types, eth_api], + ../types, + ../header_store + +export results, stint, hashes_rlp, accounts_rlp, eth_api_types + +template rpcClient(vp: VerifiedRpcProxy): RpcClient = + vp.proxy.getClient() + +template calcWithdrawalsRoot*(withdrawals: openArray[Withdrawal]): Root = + orderedTrieRoot(withdrawals) + +func vHashes(x: Opt[seq[Hash32]]): seq[VersionedHash] = + if x.isNone: return + else: x.get + +func authList(x: Opt[seq[Authorization]]): seq[Authorization] = + if x.isNone: return + else: x.get + +proc toTransaction(tx: TransactionObject): Transaction = + Transaction( + txType : tx.`type`.get(0.Web3Quantity).TxType, + chainId : tx.chainId.get(0.Web3Quantity).ChainId, + nonce : tx.nonce.AccountNonce, + gasPrice : tx.gasPrice.GasInt, + maxPriorityFeePerGas: tx.maxPriorityFeePerGas.get(0.Web3Quantity).GasInt, + maxFeePerGas : tx.maxFeePerGas.get(0.Web3Quantity).GasInt, + gasLimit : tx.gas.GasInt, + to : tx.to, + value : tx.value, + payload : tx.input, + accessList : tx.accessList.get(@[]), + maxFeePerBlobGas: tx.maxFeePerBlobGas.get(0.u256), + versionedHashes : vHashes(tx.blobVersionedHashes), + V : tx.v.uint64, + R : tx.r, + S : tx.s, + authorizationList: authList(tx.authorizationList), + ) + +proc toTransactions(txs: openArray[TxOrHash]): seq[Transaction] {.raises: [ValueError].} = + for x in txs: + if x.kind == tohTx: + result.add toTransaction(x.tx) + else: + raise newException(ValueError, "cannot construct a transaction trie using only txhashes") + +proc checkTxHash*(txObj: TransactionObject, txHash: Hash32): bool = + let tx = toTransaction(txObj) + if tx.rlpHash != txHash: + return false + + return true + +template toLog(lg: LogObject): Log = + Log( + address: lg.address, + topics: lg.topics, + data: lg.data + ) + +proc toLogs(logs: openArray[LogObject]): seq[Log] = + result = map(logs, proc(x: LogObject): Log = toLog(x)) + +proc toReceipt(rec: ReceiptObject): Receipt = + let isHash = if rec.status.isSome: false + else: true + + var status = false + if rec.status.isSome: + if rec.status.get() == 1.Quantity: + status = true + + return Receipt( + hash: rec.transactionHash, + isHash: isHash, + status: status, + cumulativeGasUsed: rec.cumulativeGasUsed.GasInt, + logs: toLogs(rec.logs), + logsBloom: rec.logsBloom, + receiptType: rec.`type`.get(0.Web3Quantity).ReceiptType + ) + +proc verifyTransactions*( + txRoot: Hash32, + transactions: seq[TxOrHash], +): Result[bool, string] = + + try: + let txs = toTransactions(transactions) + let rootHash = orderedTrieRoot(txs) + if rootHash == txRoot: + return ok(true) + except ValueError as e: + return err(e.msg) + + ok(false) From d1f000e05e627943e5acd0ae797a7899efc9037d Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 31 Mar 2025 14:51:05 +0530 Subject: [PATCH 09/11] receipt fix --- nimbus_verified_proxy/rpc/receipts.nim | 95 ++++++++++++++++++++++ nimbus_verified_proxy/rpc/rpc_eth_api.nim | 23 +++++- nimbus_verified_proxy/rpc/transactions.nim | 29 ------- 3 files changed, 117 insertions(+), 30 deletions(-) create mode 100644 nimbus_verified_proxy/rpc/receipts.nim diff --git a/nimbus_verified_proxy/rpc/receipts.nim b/nimbus_verified_proxy/rpc/receipts.nim new file mode 100644 index 000000000..a98b5583a --- /dev/null +++ b/nimbus_verified_proxy/rpc/receipts.nim @@ -0,0 +1,95 @@ +# nimbus_verified_proxy +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/sequtils, + stint, + results, + chronicles, + eth/common/[base_rlp, transactions_rlp, receipts_rlp, hashes_rlp], + ../../execution_chain/beacon/web3_eth_conv, + eth/common/addresses, + eth/common/eth_types_rlp, + eth/trie/[hexary, ordered_trie, db, trie_defs], + json_rpc/[rpcproxy, rpcserver, rpcclient], + web3/[primitives, eth_api_types, eth_api], + ../types, + ./blocks, + ../header_store + +export results, stint, hashes_rlp, accounts_rlp, eth_api_types + +template rpcClient(vp: VerifiedRpcProxy): RpcClient = + vp.proxy.getClient() + +template toLog(lg: LogObject): Log = + Log( + address: lg.address, + topics: lg.topics, + data: lg.data + ) + +proc toLogs(logs: openArray[LogObject]): seq[Log] = + result = map(logs, proc(x: LogObject): Log = toLog(x)) + +proc toReceipt(rec: ReceiptObject): Receipt = + let isHash = if rec.status.isSome: false + else: true + + var status = false + if rec.status.isSome: + if rec.status.get() == 1.Quantity: + status = true + + return Receipt( + hash: rec.transactionHash, + isHash: isHash, + status: status, + cumulativeGasUsed: rec.cumulativeGasUsed.GasInt, + logs: toLogs(rec.logs), + logsBloom: rec.logsBloom, + receiptType: rec.`type`.get(0.Web3Quantity).ReceiptType + ) + +proc toReceipts(recs: openArray[ReceiptObject]): seq[Receipt] = + for r in recs: + result.add(toReceipt(r)) + +proc getReceiptsByBlockTag*( + vp: VerifiedRpcProxy, blockTag: BlockTag +): Future[seq[ReceiptObject]] {.async: (raises: [ValueError, CatchableError]).} = + + let + header = await vp.getHeaderByTag(blockTag) + rxs = await vp.rpcClient.eth_getBlockReceipts(blockTag) + + if rxs.isSome(): + if orderedTrieRoot(toReceipts(rxs.get())) != header.receiptsRoot: + raise newException(ValueError, "downloaded receipts do not evaluate to the receipts root of the block") + else: + raise newException(ValueError, "error downloading the receipts") + + return rxs.get() + +proc getReceiptsByBlockHash*( + vp: VerifiedRpcProxy, blockHash: Hash32 +): Future[seq[ReceiptObject]] {.async: (raises: [ValueError, CatchableError]).} = + + let + header = await vp.getHeaderByHash(blockHash) + blockTag = BlockTag(RtBlockIdentifier(kind: bidNumber, number: Quantity(header.number))) + rxs = await vp.rpcClient.eth_getBlockReceipts(blockTag) + + if rxs.isSome(): + if orderedTrieRoot(toReceipts(rxs.get())) != header.receiptsRoot: + raise newException(ValueError, "downloaded receipts do not evaluate to the receipts root of the block") + else: + raise newException(ValueError, "error downloading the receipts") + + return rxs.get() diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index ff94d02fe..f3238bc53 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -25,7 +25,8 @@ import ../types, ./blocks, ./accounts, - ./transactions + ./transactions, + ./receipts logScope: topics = "verified_proxy" @@ -115,6 +116,26 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) = return tx + # TODO: this method should also support block hashes. For that, the client call should also suupport block hashes + vp.proxy.rpc("eth_getBlockReceipts") do( + blockTag: BlockTag + ) -> Opt[seq[ReceiptObject]]: + let rxs = await vp.getReceiptsByBlockTag(blockTag) + return Opt.some(rxs) + + vp.proxy.rpc("eth_getTransactionReceipt") do( + txHash: Hash32 + ) -> ReceiptObject: + let + rx = await vp.rpcClient.eth_getTransactionReceipt(txHash) + rxs = await vp.getReceiptsByBlockHash(rx.blockHash) + + for r in rxs: + if r.transactionHash == txHash: + return r + + raise newException(ValueError, "receipt couldn't be verified") + # eth_blockNumber - get latest tag from header store vp.proxy.rpc("eth_blockNumber") do() -> Quantity: # Returns the number of the most recent block seen by the light client. diff --git a/nimbus_verified_proxy/rpc/transactions.nim b/nimbus_verified_proxy/rpc/transactions.nim index 4e280f537..8a5468779 100644 --- a/nimbus_verified_proxy/rpc/transactions.nim +++ b/nimbus_verified_proxy/rpc/transactions.nim @@ -73,35 +73,6 @@ proc checkTxHash*(txObj: TransactionObject, txHash: Hash32): bool = return true -template toLog(lg: LogObject): Log = - Log( - address: lg.address, - topics: lg.topics, - data: lg.data - ) - -proc toLogs(logs: openArray[LogObject]): seq[Log] = - result = map(logs, proc(x: LogObject): Log = toLog(x)) - -proc toReceipt(rec: ReceiptObject): Receipt = - let isHash = if rec.status.isSome: false - else: true - - var status = false - if rec.status.isSome: - if rec.status.get() == 1.Quantity: - status = true - - return Receipt( - hash: rec.transactionHash, - isHash: isHash, - status: status, - cumulativeGasUsed: rec.cumulativeGasUsed.GasInt, - logs: toLogs(rec.logs), - logsBloom: rec.logsBloom, - receiptType: rec.`type`.get(0.Web3Quantity).ReceiptType - ) - proc verifyTransactions*( txRoot: Hash32, transactions: seq[TxOrHash], From 2a2ffee0a31f01231522a536d65f8cf699b018cf Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 31 Mar 2025 17:27:57 +0530 Subject: [PATCH 10/11] rebase fix --- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 2 +- nimbus_verified_proxy/rpc/transactions.nim | 2 +- nimbus_verified_proxy/types.nim | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index f3238bc53..2e998d0a0 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -35,7 +35,7 @@ template rpcClient*(vp: VerifiedRpcProxy): RpcClient = vp.proxy.getClient() proc installEthApiHandlers*(vp: VerifiedRpcProxy) = - vp.proxy.rpc("eth_chainId") do() -> Quantity: + vp.proxy.rpc("eth_chainId") do() -> UInt256: vp.chainId vp.proxy.rpc("eth_getBlockByNumber") do( diff --git a/nimbus_verified_proxy/rpc/transactions.nim b/nimbus_verified_proxy/rpc/transactions.nim index 8a5468779..a630d1720 100644 --- a/nimbus_verified_proxy/rpc/transactions.nim +++ b/nimbus_verified_proxy/rpc/transactions.nim @@ -41,7 +41,7 @@ func authList(x: Opt[seq[Authorization]]): seq[Authorization] = proc toTransaction(tx: TransactionObject): Transaction = Transaction( txType : tx.`type`.get(0.Web3Quantity).TxType, - chainId : tx.chainId.get(0.Web3Quantity).ChainId, + chainId : tx.chainId.get(0.u256), nonce : tx.nonce.AccountNonce, gasPrice : tx.gasPrice.GasInt, maxPriorityFeePerGas: tx.maxPriorityFeePerGas.get(0.Web3Quantity).GasInt, diff --git a/nimbus_verified_proxy/types.nim b/nimbus_verified_proxy/types.nim index 01cfaaa2d..8e9aa1975 100644 --- a/nimbus_verified_proxy/types.nim +++ b/nimbus_verified_proxy/types.nim @@ -7,16 +7,16 @@ import json_rpc/[rpcproxy], - web3/[primitives, eth_api_types, eth_api], + stint, ./header_store type VerifiedRpcProxy* = ref object proxy*: RpcProxy headerStore*: HeaderStore - chainId*: Quantity + chainId*: UInt256 proc new*( - T: type VerifiedRpcProxy, proxy: RpcProxy, headerStore: HeaderStore, chainId: Quantity + T: type VerifiedRpcProxy, proxy: RpcProxy, headerStore: HeaderStore, chainId: UInt256 ): T = VerifiedRpcProxy(proxy: proxy, headerStore: headerStore, chainId: chainId) From c146a47d73a274b30123a612180e81a4c4eec911 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 1 Apr 2025 15:05:58 +0530 Subject: [PATCH 11/11] fixes --- nimbus_verified_proxy/rpc/rpc_eth_api.nim | 28 ++++++++--------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/nimbus_verified_proxy/rpc/rpc_eth_api.nim b/nimbus_verified_proxy/rpc/rpc_eth_api.nim index 2e998d0a0..c82eb184f 100644 --- a/nimbus_verified_proxy/rpc/rpc_eth_api.nim +++ b/nimbus_verified_proxy/rpc/rpc_eth_api.nim @@ -40,25 +40,13 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) = vp.proxy.rpc("eth_getBlockByNumber") do( blockTag: BlockTag, fullTransactions: bool - ) -> Opt[BlockObject]: - try: - let blk = await vp.getBlockByTag(blockTag, fullTransactions) - return Opt.some(blk) - except ValueError as e: - # should return Opt.none but we also want to transmit error related info - # return Opt.none(BlockObject) - raise newException(ValueError, e.msg) # raising an exception will return the error message + ) -> BlockObject: + await vp.getBlockByTag(blockTag, fullTransactions) vp.proxy.rpc("eth_getBlockByHash") do( blockHash: Hash32, fullTransactions: bool - ) -> Opt[BlockObject]: - try: - let blk = await vp.getBlockByHash(blockHash, fullTransactions) - return Opt.some(blk) - except ValueError as e: - # should return Opt.none but we also want to transmit error related info - # return Opt.none(BlockObject) - raise newException(ValueError, e.msg) # raising an exception will return the error message + ) -> BlockObject: + await vp.getBlockByHash(blockHash, fullTransactions) vp.proxy.rpc("eth_getUncleCountByBlockNumber") do( blockTag: BlockTag @@ -156,10 +144,12 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) = vp.proxy.rpc("eth_getStorageAt") do( address: addresses.Address, slot: UInt256, blockTag: BlockTag - ) -> UInt256: - let header = await vp.getHeaderByTag(blockTag) + ) -> FixedBytes[32]: + let + header = await vp.getHeaderByTag(blockTag) + storage = await vp.getStorageAt(address, slot, header.number, header.stateRoot) - await vp.getStorageAt(address, slot, header.number, header.stateRoot) + FixedBytes[32](storage.toBytesBE) vp.proxy.rpc("eth_getTransactionCount") do( address: addresses.Address, blockTag: BlockTag