History: in-memory blockhash index

packages/cli/src/rpc/modules/ultralight.ts

@@ -111,7 +111,7 @@ export class ultralight {
     const [blockNum, blockHash] = params
     try {
       this.logger(`Indexed block ${BigInt(blockNum)} / ${blockNum} to ${blockHash} `)
-      await this._history!.indexBlockhash(BigInt(blockNum), blockHash)
+      await this._history!.indexBlockHash(BigInt(blockNum), blockHash)
       return `Added ${blockNum} to block index`
     } catch (err: any) {
       throw {

packages/portalnetwork/src/client/client.ts

@@ -289,13 +289,17 @@ export class PortalNetwork extends (EventEmitter as { new (): PortalNetworkEvent
   public start = async () => {
     await this.discv5.start()
     await this.db.open()
+    const storedIndex = await this.db.getBlockIndex()
     for (const network of this.networks.values()) {
       try {
         // Check for stored radius in db
         const storedRadius = await network.db.db.get('radius')
         await network.setRadius(BigInt(storedRadius))
       } catch {
-        continue
+        // No action
+      }
+      if (network instanceof HistoryNetwork) {
+        network.blockHashIndex = storedIndex
       }
       await network.prune()
       // Start kbucket refresh on 30 second interval

packages/portalnetwork/src/client/eth.ts

@@ -90,10 +90,9 @@ export class ETH {
       this.logger.extend('getBlockByHash')(lookupResponse)
       if (!lookupResponse || !('content' in lookupResponse)) {
         // Header not found by hash, try to find by number if known
-        const blockIndex = await this.history!.blockIndex()
-        const blockNumber = blockIndex.get(blockHash)
+        const blockNumber = this.history!.blockHashToNumber(blockHash)
         if (blockNumber !== undefined) {
-          const block = await this.getBlockByNumber(BigInt(blockNumber), includeTransactions)
+          const block = await this.getBlockByNumber(blockNumber, includeTransactions)
           return block
         }
         return undefined
@@ -191,7 +190,7 @@ export class ETH {
       }
     } else {
       // Header not found by number.  If block hash is known, search for header by hash
-      const blockHash = await this.history!.blockNumberToHash(BigInt(blockNumber))
+      const blockHash = this.history!.blockNumberToHash(BigInt(blockNumber))
       if (blockHash !== undefined) {
         return this.getBlockByHash(blockHash, includeTransactions)
       }

packages/portalnetwork/src/networks/history/history.ts

@@ -38,12 +38,23 @@ export class HistoryNetwork extends BaseNetwork {
   networkName = 'HistoryNetwork'
   logger: Debugger
   gossipManager: GossipManager
+  public blockHashIndex: Map<string, string>
   constructor({ client, db, radius, maxStorage }: BaseNetworkConfig) {
     super({ client, networkId: NetworkId.HistoryNetwork, db, radius, maxStorage })
     this.networkId = NetworkId.HistoryNetwork
     this.logger = debug(this.enr.nodeId.slice(0, 5)).extend('Portal').extend('HistoryNetwork')
     this.gossipManager = new GossipManager(this)
     this.routingTable.setLogger(this.logger)
+    this.blockHashIndex = new Map()
+  }
+
+  public blockNumberToHash(blockNumber: bigint): string | undefined {
+    return this.blockHashIndex.get('0x' + blockNumber.toString(16))
+  }
+
+  public blockHashToNumber(blockHash: string): bigint | undefined {
+    const blockNumber = this.blockHashIndex.get(blockHash)
+    return blockNumber === undefined ? undefined : BigInt(blockNumber)
   }
 
   /**
@@ -55,7 +66,7 @@ export class HistoryNetwork extends BaseNetwork {
     const contentType = contentKey[0]
     if (contentType === HistoryNetworkContentType.BlockHeaderByNumber) {
       const blockNumber = decodeHistoryNetworkContentKey(contentKey).keyOpt
-      const blockHash = await this.blockNumberToHash(<bigint>blockNumber)
+      const blockHash = this.blockNumberToHash(<bigint>blockNumber)
       if (blockHash === undefined) {
         return undefined
       }
@@ -68,12 +79,11 @@ export class HistoryNetwork extends BaseNetwork {
     return value !== undefined ? hexToBytes(value) : undefined
   }
 
-  public indexBlockhash = async (number: bigint, blockHash: string) => {
+  public indexBlockHash = async (number: bigint, blockHash: string) => {
     const blockNumber = '0x' + number.toString(16)
-    const blockindex = await this.blockIndex()
-    blockindex.set(blockNumber, blockHash)
-    blockindex.set(blockHash, blockNumber)
-    await this.setBlockIndex(blockindex)
+    this.blockHashIndex.set(blockNumber, blockHash)
+    this.blockHashIndex.set(blockHash, blockNumber)
+    await this.portal.db.storeBlockIndex(this.blockHashIndex)
   }
 
   /**
@@ -173,7 +183,7 @@ export class HistoryNetwork extends BaseNetwork {
         }
       }
     }
-    await this.indexBlockhash(header.number, toHexString(header.hash()))
+    await this.indexBlockHash(header.number, toHexString(header.hash()))
     return header.hash()
   }
 

packages/portalnetwork/src/networks/history/util.ts

@@ -232,7 +232,7 @@ export const addRLPSerializedBlock = async (
       header: header.serialize(),
       proof: { selector: 0, value: null },
     })
-    await network.indexBlockhash(header.number, toHexString(header.hash()))
+    await network.indexBlockHash(header.number, toHexString(header.hash()))
 
     await network.store(headerKey, headerProof)
   }

packages/portalnetwork/src/networks/network.ts

@@ -57,19 +57,12 @@ export abstract class BaseNetwork extends EventEmitter {
   public networkId: NetworkId
   abstract networkName: string
   public enr: SignableENR
-  public blockIndex: () => Promise<Map<string, string>>
-  public setBlockIndex: (blockIndex: Map<string, string>) => Promise<void>
+
   portal: PortalNetwork
   constructor({ client, networkId, db, radius, maxStorage }: BaseNetworkConfig) {
     super()
     this.networkId = networkId
     this.logger = client.logger.extend(this.constructor.name)
-    this.blockIndex = () => {
-      return client.db.getBlockIndex()
-    }
-    this.setBlockIndex = (blockIndex: Map<string, string>) => {
-      return client.db.storeBlockIndex(blockIndex)
-    }
     this.enr = client.discv5.enr
     this.checkIndex = 0
     this.nodeRadius = radius ?? 2n ** 256n - 1n
@@ -117,17 +110,6 @@ export abstract class BaseNetwork extends EventEmitter {
     return this.portal.discv5.findEnr(nodeId)
   }
 
-  public async blockNumberToHash(blockNumber: bigint): Promise<string | undefined> {
-    const blockIndex = await this.blockIndex()
-    return blockIndex.get('0x' + blockNumber.toString(16))
-  }
-
-  public async blockHashToNumber(blockHash: string): Promise<bigint | undefined> {
-    const blockIndex = await this.blockIndex()
-    const blockNumber = blockIndex.get(blockHash)
-    return blockNumber === undefined ? undefined : BigInt(blockNumber)
-  }
-
   public async put(contentKey: string, content: string) {
     await this.db.put(contentKey, content)
   }

packages/portalnetwork/test/client/eth/ethCall.spec.ts

@@ -88,7 +88,7 @@ describe.skip('ethCall', () => {
   //     toHexString(block.header.hash()),
   //     history,
   //   )
-  //   await history.indexBlockhash(block.header.number, toHexString(block.header.hash()))
+  //   await history.indexBlockHash(block.header.number, toHexString(block.header.hash()))
   //   const greeterInput = '0xcfae3217'
   //   const tx: RpcTx = {
   //     to: address.toString(),

packages/portalnetwork/test/networks/history/blockIndex.spec.ts

@@ -28,14 +28,13 @@ describe('BlockIndex', async () => {
   it('should store block header', () => {
     assert.equal(stored, headerWithProof)
   })
-  it('should save block index', async () => {
+  it('should save block index', () => {
     const numberHex = '0x' + BigInt(1).toString(16)
-    const index = await history.blockIndex()
-    assert.isTrue(index.has(numberHex))
-    assert.equal(index.get(numberHex), hash)
+    assert.isTrue(history.blockHashIndex.has(numberHex))
+    assert.equal(history.blockHashIndex.get(numberHex), hash)
   })
   it('should store blockIndex in DB', async () => {
-    const expected = Array.from(await history.blockIndex())
+    const expected = Array.from(history.blockHashIndex)
     const stored = JSON.parse(await ultralight.db.db.get('block_index'))
     assert.deepEqual(stored, expected)
   })
@@ -45,10 +44,11 @@ describe('BlockIndex', async () => {
     supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }],
     db: ultralight.db.db,
   })
+  await ultralight2.start()
   const history2 = ultralight2.networks.get(NetworkId.HistoryNetwork) as HistoryNetwork
   it('should start with blockIndex in DB', async () => {
-    const expected = await history.blockIndex()
-    const stored = await history2.blockIndex()
-    assert.deepEqual(stored, expected)
+    const expected = history.blockHashIndex
+    const stored = history2.blockHashIndex
+    assert.deepEqual(stored, expected, `Expected ${expected.size} but got ${stored.size}`)
   })
 })