Skip to content

Commit 4f8ec42

Browse files
AndresElizondoAndrés Elizondo
AndresElizondo
and
Andrés Elizondo
authored
Refactors Settler RFQ Order events for faster backfills. (#166)
* Refactors Settler RFQ Order events for faster backfills. * Random updates to docker-compose.yml * Prettier. * Upgrades web3.js to latest version. --------- Co-authored-by: Andrés Elizondo <[email protected]>
1 parent 8d46caa commit 4f8ec42

6 files changed

+505
-230
lines changed

‎docker-compose.yml

+226-15
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ services:
1717
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
1818
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
1919
SCRAPER_MODE: 'BLOCKS'
20+
BLOCK_RECEIPTS_MODE: "standard"
2021
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
2122
EP_DEPLOYMENT_BLOCK: 10247094
2223
SETTLER_DEPLOYMENT_BLOCK: 19582253
@@ -73,7 +74,7 @@ services:
7374
dockerfile: Dockerfile
7475
restart: always
7576
environment:
76-
EVM_RPC_URL: '${RPC_URL_BSC}'
77+
EVM_RPC_URL: '${BSC_RPC_URL_ALCHEMY}'
7778
CHAIN_ID: '56'
7879
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
7980
SCHEMA: 'events_bsc'
@@ -114,15 +115,16 @@ services:
114115
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
115116
SCHEMA: 'events_polygon'
116117
SCRAPER_MODE: 'BLOCKS'
118+
BLOCK_RECEIPTS_MODE: "standard"
117119
ENABLE_PROMETHEUS_METRICS: "true"
118120
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
119121
EP_DEPLOYMENT_BLOCK: 14391480
120122
SETTLER_DEPLOYMENT_BLOCK: 55435171
121-
MAX_BLOCKS_TO_SEARCH: 1000
122-
MAX_BLOCKS_TO_PULL: 251
123-
SECONDS_BETWEEN_RUNS: 2
123+
MAX_BLOCKS_TO_SEARCH: 500
124+
MAX_BLOCKS_TO_PULL: 500
125+
SECONDS_BETWEEN_RUNS: 1
124126
FEAT_LIMIT_ORDERS: "true"
125-
V4_NATIVE_FILL_START_BLOCK: "17587182"
127+
V4_NATIVE_FILL_START_BLOCK: 17587182
126128
FEAT_NFT: "true"
127129
NFT_FEATURE_START_BLOCK: 25697854
128130
FEAT_OTC_ORDERS: "true"
@@ -134,8 +136,8 @@ services:
134136
FEAT_TOKENS_FROM_TRANSFERS: "true"
135137
MAX_BLOCKS_REORG: 400
136138
FEAT_ERC20_TRANSFER_ALL: "true"
137-
FEAT_ERC20_TRANSFER_ALL: "true"
138139
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
140+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
139141

140142
event-pipeline-avalanche:
141143
build:
@@ -159,8 +161,6 @@ services:
159161
MAX_BLOCKS_TO_PULL: 2000
160162
SECONDS_BETWEEN_RUNS: 1
161163
RESCRAPE_BLOCKS: 10
162-
SECONDS_BETWEEN_RUNS: 1
163-
RESCRAPE_BLOCKS: 10
164164
FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true"
165165
FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498"
166166
FLASHWALLET_DEPLOYMENT_BLOCK: 11805869
@@ -178,10 +178,12 @@ services:
178178
dockerfile: Dockerfile
179179
restart: always
180180
environment:
181-
EVM_RPC_URL: '${RPC_URL_FANTOM}'
181+
EVM_RPC_URL: '${FANTOM_RPC_URL_ALCHEMY}'
182182
CHAIN_ID: '250'
183183
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
184184
SCHEMA: 'events_fantom'
185+
SCRAPER_MODE: 'BLOCKS'
186+
BLOCK_RECEIPTS_MODE: "standard"
185187
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
186188
#KAFKA_SSL: '${KAFKA_SSL}'
187189
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
@@ -190,6 +192,7 @@ services:
190192
ENABLE_PROMETHEUS_METRICS: "true"
191193
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
192194
EP_DEPLOYMENT_BLOCK: 18855765
195+
SETTLER_DEPLOYMENT_BLOCK: 18855765
193196
MAX_BLOCKS_TO_SEARCH: 2000
194197
MAX_BLOCKS_TO_PULL: 1000
195198
SECONDS_BETWEEN_RUNS: 1
@@ -208,19 +211,22 @@ services:
208211
dockerfile: Dockerfile
209212
restart: always
210213
environment:
211-
EVM_RPC_URL: '${RPC_URL_CELO}'
214+
EVM_RPC_URL: '${CELO_RPC_URL_ALCHEMY}'
212215
CHAIN_ID: '42220'
213216
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
214217
SCHEMA: 'events_celo'
218+
SCRAPER_MODE: 'EVENTS'
219+
BLOCK_RECEIPTS_MODE: "standard"
215220
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
216221
#KAFKA_SSL: '${KAFKA_SSL}'
217222
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
218223
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
219224
ENABLE_PROMETHEUS_METRICS: "true"
220225
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
221226
EP_DEPLOYMENT_BLOCK: 9350111
222-
MAX_BLOCKS_TO_SEARCH: 5000
223-
MAX_BLOCKS_TO_PULL: 2000
227+
SETTLER_DEPLOYMENT_BLOCK: 9350111
228+
MAX_BLOCKS_TO_SEARCH: 100
229+
MAX_BLOCKS_TO_PULL: 20
224230
SECONDS_BETWEEN_RUNS: 1
225231
RESCRAPE_BLOCKS: 10
226232
FEAT_NFT: "true"
@@ -232,10 +238,12 @@ services:
232238
dockerfile: Dockerfile
233239
restart: always
234240
environment:
241+
useLevel: "debug"
235242
EVM_RPC_URL: '${RPC_URL_OPTIMISM}'
236243
CHAIN_ID: '10'
237244
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
238245
SCHEMA: 'events_optimism'
246+
SCRAPER_MODE: "BLOCKS"
239247
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
240248
#KAFKA_SSL: '${KAFKA_SSL}'
241249
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
@@ -265,20 +273,24 @@ services:
265273
dockerfile: Dockerfile
266274
restart: always
267275
environment:
276+
LOG_LEVEL: "debug"
268277
EVM_RPC_URL: '${EVM_RPC_URL_ARBITRUM}'
269278
CHAIN_ID: '42161'
270279
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
271280
SCHEMA: 'events_arbitrum'
281+
SCRAPER_MODE: "BLOCKS"
282+
BLOCK_RECEIPTS_MODE: "standard"
272283
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
273284
#KAFKA_SSL: '${KAFKA_SSL}'
274285
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
275286
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
276287
FEAT_ZEROEX_EXCHANGE_PROXY: "true"
277288
EP_DEPLOYMENT_BLOCK: 4050733
278289
SETTLER_DEPLOYMENT_BLOCK: 197531334
279-
MAX_BLOCKS_TO_SEARCH: 1000
280-
MAX_BLOCKS_TO_PULL: 1000
281-
SECONDS_BETWEEN_RUNS: 30
290+
MAX_BLOCKS_TO_SEARCH: 500
291+
MAX_BLOCKS_TO_PULL: 102
292+
MAX_BLOCKS_REORG: 300
293+
SECONDS_BETWEEN_RUNS: 1
282294
RESCRAPE_BLOCKS: 10
283295
EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff"
284296
FEAT_NFT: "true"
@@ -287,6 +299,20 @@ services:
287299
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1'
288300
FEAT_ERC20_TRANSFER_ALL: "true"
289301
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
302+
FEAT_TRANSFORMED_ERC20_EVENT: "true"
303+
FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true"
304+
FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498"
305+
FLASHWALLET_DEPLOYMENT_BLOCK: 4050733
306+
FEAT_OTC_ORDERS: "true"
307+
OTC_ORDERS_FEATURE_START_BLOCK: 87341520
308+
BLOCK_FINALITY_THRESHOLD: 0
309+
FEAT_SOCKET_BRIDGE_EVENT: "true"
310+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
311+
SOCKET_BRIDGE_EVENT_START_BLOCK: 145873113
312+
FEAT_META_TRANSACTION_EXECUTED_EVENT: "true"
313+
META_TRANSACTION_EXECUTED_START_BLOCK: 168492783
314+
FEAT_TOKENS_FROM_TRANSFERS: "true"
315+
290316

291317
event-pipeline-base:
292318
build:
@@ -318,6 +344,191 @@ services:
318344
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
319345
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
320346

347+
event-pipeline-scroll:
348+
build:
349+
context: .
350+
dockerfile: Dockerfile
351+
restart: always
352+
environment:
353+
EVM_RPC_URL: '${SCROLL_RPC_URL_ALCHEMY}'
354+
CHAIN_ID: '534352'
355+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
356+
SCHEMA: 'events_scroll'
357+
SCRAPER_MODE: 'BLOCKS'
358+
BLOCK_RECEIPTS_MODE: "standard"
359+
FEAT_ZEROEX_EXCHANGE_PROXY: "false"
360+
# EP_DEPLOYMENT_BLOCK:
361+
SETTLER_DEPLOYMENT_BLOCK: 7474150
362+
MAX_BLOCKS_TO_SEARCH: 1000
363+
MAX_BLOCKS_TO_PULL: 100
364+
MAX_BLOCKS_REORG: 300
365+
SECONDS_BETWEEN_RUNS: 1
366+
RESCRAPE_BLOCKS: 10
367+
FEAT_NFT: "false"
368+
NFT_FEATURE_START_BLOCK: 1410394
369+
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
370+
#KAFKA_SSL: '${KAFKA_SSL}'
371+
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
372+
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
373+
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
374+
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x5300000000000000000000000000000000000004'
375+
FEAT_ERC20_TRANSFER_ALL: "true"
376+
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
377+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
378+
# FEAT_TOKENS_FROM_TRANSFERS: "true"
379+
# TOKENS_FROM_TRANSFERS_START_BLOCK: 1
380+
FEAT_SOCKET_BRIDGE_EVENT: "true"
381+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
382+
SOCKET_BRIDGE_EVENT_START_BLOCK: 9527830
383+
384+
event-pipeline-blast:
385+
build:
386+
context: .
387+
dockerfile: Dockerfile
388+
restart: always
389+
environment:
390+
EVM_RPC_URL: '${BLAST_RPC_URL_ALCHEMY}'
391+
CHAIN_ID: '81457'
392+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
393+
SCHEMA: 'events_blast'
394+
SCRAPER_MODE: 'BLOCKS'
395+
BLOCK_RECEIPTS_MODE: "standard"
396+
FEAT_ZEROEX_EXCHANGE_PROXY: "false"
397+
# EP_DEPLOYMENT_BLOCK:
398+
SETTLER_DEPLOYMENT_BLOCK: 6124420
399+
MAX_BLOCKS_TO_SEARCH: 1000
400+
MAX_BLOCKS_TO_PULL: 100
401+
MAX_BLOCKS_REORG: 500
402+
SECONDS_BETWEEN_RUNS: 1
403+
RESCRAPE_BLOCKS: 10
404+
FEAT_NFT: "false"
405+
# NFT_FEATURE_START_BLOCK: 1410394
406+
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
407+
#KAFKA_SSL: '${KAFKA_SSL}'
408+
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
409+
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
410+
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
411+
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4300000000000000000000000000000000000004'
412+
FEAT_ERC20_TRANSFER_ALL: "true"
413+
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
414+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
415+
FEAT_TOKENS_FROM_TRANSFERS: "true"
416+
# TOKENS_FROM_TRANSFERS_START_BLOCK: 1
417+
FEAT_SOCKET_BRIDGE_EVENT: "true"
418+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
419+
SOCKET_BRIDGE_EVENT_START_BLOCK: 6124420
420+
421+
event-pipeline-linea:
422+
build:
423+
context: .
424+
dockerfile: Dockerfile
425+
restart: always
426+
environment:
427+
EVM_RPC_URL: '${LINEA_RPC_URL_ALCHEMY}'
428+
CHAIN_ID: '59144'
429+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
430+
SCHEMA: 'events_linea'
431+
SCRAPER_MODE: 'BLOCKS'
432+
BLOCK_RECEIPTS_MODE: "standard"
433+
FEAT_ZEROEX_EXCHANGE_PROXY: "false"
434+
# EP_DEPLOYMENT_BLOCK:
435+
SETTLER_DEPLOYMENT_BLOCK: 6917652
436+
MAX_BLOCKS_TO_SEARCH: 125
437+
MAX_BLOCKS_TO_PULL: 25
438+
MAX_BLOCKS_REORG: 125
439+
SECONDS_BETWEEN_RUNS: 1
440+
RESCRAPE_BLOCKS: 10
441+
FEAT_NFT: "false"
442+
# NFT_FEATURE_START_BLOCK: 1410394
443+
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
444+
#KAFKA_SSL: '${KAFKA_SSL}'
445+
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
446+
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
447+
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
448+
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f'
449+
FEAT_ERC20_TRANSFER_ALL: "true"
450+
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
451+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
452+
# FEAT_TOKENS_FROM_TRANSFERS: "true"
453+
# TOKENS_FROM_TRANSFERS_START_BLOCK: 1
454+
FEAT_SOCKET_BRIDGE_EVENT: "true"
455+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
456+
SOCKET_BRIDGE_EVENT_START_BLOCK: 6917652
457+
458+
event-pipeline-mode:
459+
build:
460+
context: .
461+
dockerfile: Dockerfile
462+
restart: always
463+
environment:
464+
EVM_RPC_URL: '${MODE_RPC_URL_ALCHEMY}'
465+
CHAIN_ID: '34443'
466+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
467+
SCHEMA: 'events_mode'
468+
SCRAPER_MODE: 'BLOCKS'
469+
BLOCK_RECEIPTS_MODE: "standard"
470+
FEAT_ZEROEX_EXCHANGE_PROXY: "false"
471+
# EP_DEPLOYMENT_BLOCK:
472+
SETTLER_DEPLOYMENT_BLOCK: 14121097
473+
MAX_BLOCKS_TO_SEARCH: 450
474+
MAX_BLOCKS_TO_PULL: 200
475+
MAX_BLOCKS_REORG: 450
476+
SECONDS_BETWEEN_RUNS: 1
477+
RESCRAPE_BLOCKS: 10
478+
FEAT_NFT: "false"
479+
# NFT_FEATURE_START_BLOCK: 1410394
480+
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
481+
#KAFKA_SSL: '${KAFKA_SSL}'
482+
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
483+
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
484+
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
485+
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4200000000000000000000000000000000000006'
486+
FEAT_ERC20_TRANSFER_ALL: "true"
487+
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
488+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
489+
FEAT_TOKENS_FROM_TRANSFERS: "true"
490+
# TOKENS_FROM_TRANSFERS_START_BLOCK: 1
491+
FEAT_SOCKET_BRIDGE_EVENT: "true"
492+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
493+
SOCKET_BRIDGE_EVENT_START_BLOCK: 6124420
494+
495+
event-pipeline-mantle:
496+
build:
497+
context: .
498+
dockerfile: Dockerfile
499+
restart: always
500+
environment:
501+
EVM_RPC_URL: '${MANTLE_RPC_URL_QUICKNODE}'
502+
CHAIN_ID: '5000'
503+
POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
504+
SCHEMA: 'events_mantle'
505+
SCRAPER_MODE: 'BLOCKS'
506+
BLOCK_RECEIPTS_MODE: "standard"
507+
FEAT_ZEROEX_EXCHANGE_PROXY: "false"
508+
# EP_DEPLOYMENT_BLOCK:
509+
SETTLER_DEPLOYMENT_BLOCK: 6124420
510+
MAX_BLOCKS_TO_SEARCH: 125
511+
MAX_BLOCKS_TO_PULL: 25
512+
MAX_BLOCKS_REORG: 450
513+
SECONDS_BETWEEN_RUNS: 1
514+
RESCRAPE_BLOCKS: 10
515+
FEAT_NFT: "false"
516+
# NFT_FEATURE_START_BLOCK: 1410394
517+
#KAFKA_BROKERS: '${KAFKA_BROKERS}'
518+
#KAFKA_SSL: '${KAFKA_SSL}'
519+
#KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}'
520+
#KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
521+
FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
522+
WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x78c1b0C915c4FAA5FffA6CAbf0219DA63d7f4cb8'
523+
FEAT_ERC20_TRANSFER_ALL: "true"
524+
FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
525+
FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
526+
FEAT_TOKENS_FROM_TRANSFERS: "true"
527+
# TOKENS_FROM_TRANSFERS_START_BLOCK: 1
528+
FEAT_SOCKET_BRIDGE_EVENT: "true"
529+
SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
530+
SOCKET_BRIDGE_EVENT_START_BLOCK: 6124420
531+
321532
token-scraper-ethereum:
322533
build:
323534
context: .

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,6 @@
5353
"reflect-metadata": "^0.1.13",
5454
"typeorm": "^0.2.20",
5555
"typescript": "4",
56-
"web3": "^1.9.0"
56+
"web3": "1.10.4"
5757
}
5858
}

0 commit comments

Comments
 (0)