From 8125c3bf9feee499413dfafd5eda1748ad46d9ee Mon Sep 17 00:00:00 2001 From: Matt Rivas Date: Wed, 13 Dec 2023 14:17:33 -0700 Subject: [PATCH 1/3] chore: update config refs --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 6e34c5b6..a7048273 160000 --- a/config +++ b/config @@ -1 +1 @@ -Subproject commit 6e34c5b68fa3fba7cad3b140f8676dcbdab687c5 +Subproject commit a70482731c55f8c2079da0df66d084c2468bf3ad From 9959716b707e8a874c0e4da3e4a21a55ab9bc312 Mon Sep 17 00:00:00 2001 From: Matt Rivas Date: Wed, 13 Dec 2023 14:20:10 -0700 Subject: [PATCH 2/3] Revert "chore: update config refs" This reverts commit 8125c3bf9feee499413dfafd5eda1748ad46d9ee. --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index a7048273..6e34c5b6 160000 --- a/config +++ b/config @@ -1 +1 @@ -Subproject commit a70482731c55f8c2079da0df66d084c2468bf3ad +Subproject commit 6e34c5b68fa3fba7cad3b140f8676dcbdab687c5 From 2afef552f7c4f73c5933dd8e241d4d8cd25934b4 Mon Sep 17 00:00:00 2001 From: Matt Rivas Date: Thu, 14 Dec 2023 11:52:41 -0700 Subject: [PATCH 3/3] feat: implement state overrides and block height --- snapshotter/utils/rpc.py | 51 ++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/snapshotter/utils/rpc.py b/snapshotter/utils/rpc.py index 801548f6..b31042f7 100644 --- a/snapshotter/utils/rpc.py +++ b/snapshotter/utils/rpc.py @@ -344,14 +344,22 @@ async def f(node_idx): return current_block return await f(node_idx=0) - async def _async_web3_call(self, contract_function, redis_conn, from_address=None): + async def _async_web3_call(self, contract_function, redis_conn, from_address=None, block=None, overrides=None): """ Executes a web3 call asynchronously. + If state overrides are provided, the raw byte data is returned since + it is more efficient to decode large amounts of data + using eth_abi rather than the web3 codec and this functionality is + typically used to retrieve large amounts of on chain data ie ticks in a uniswap v3 pool. + Args: contract_function: The contract function to call. redis_conn: The Redis connection object. from_address: The address to send the transaction from. + block: The block number to execute the call at. + overrides: The overrides to use for the call. Ex: {'0xcontract_address': '0xcontract_bytecode'} + Returns: The result of the web3 call. @@ -365,7 +373,6 @@ async def _async_web3_call(self, contract_function, redis_conn, from_address=Non ) async def f(node_idx): try: - node = self._nodes[node_idx] rpc_url = node.get('rpc_url') @@ -381,12 +388,12 @@ async def f(node_idx): rate_limit_lua_script_shas=self._rate_limit_lua_script_shas, limit_incr_by=1, ) - + params: TxParams = {'gas': Wei(0), 'gasPrice': Wei(0)} if not contract_function.address: raise ValueError( - f'Missing address for batch_call in `{contract_function.fn_name}`', + f'Missing address for batch_call in `{[contract_function.fn_name]}`', ) output_type = [ @@ -419,14 +426,22 @@ async def f(node_idx): if from_address: payload['from'] = from_address - data = await node['web3_client_async'].eth.call(payload) + data = await node['web3_client_async'].eth.call( + payload, block_identifier=block, state_override=overrides + ) + + if overrides is not None: + return data - decoded_data = node['web3_client_async'].codec.decode_abi( - output_type, HexBytes(data), + decoded_data = node['web3_client_async'].codec.decode( + output_type, + HexBytes(data), ) normalized_data = map_abi_data( - BASE_RETURN_NORMALIZERS, output_type, decoded_data, + BASE_RETURN_NORMALIZERS, + output_type, + decoded_data, ) if len(normalized_data) == 1: @@ -440,10 +455,9 @@ async def f(node_idx): underlying_exception=e, extra_info={'msg': str(e)}, ) - self._logger.opt(exception=settings.logs.trace_enabled).error( - ( - 'Error while making web3 batch call' - ), + + self._logger.opt(lazy=True).error( + ('Error while making web3 batch call'), err=lambda: str(exc), ) raise exc @@ -577,7 +591,7 @@ async def get_current_block(self, redis_conn, node_idx=0): ) return current_block - async def web3_call(self, tasks, redis_conn, from_address=None): + async def web3_call(self, tasks, redis_conn, from_address=None, block=None, overrides=None): """ Calls the given tasks asynchronously using web3 and returns the response. @@ -585,6 +599,8 @@ async def web3_call(self, tasks, redis_conn, from_address=None): tasks (list): List of contract functions to call. redis_conn: Redis connection object. from_address (str, optional): Address to use as the transaction sender. Defaults to None. + block: (int, optional): Block number to execute the call at. Defaults to None. + overrides: (dict, optional): State overrides to use for the call. Defaults to None. Ex: {"0xcontract_address": "0xcontract_bytecode"} Returns: list: List of responses from the contract function calls. @@ -595,8 +611,13 @@ async def web3_call(self, tasks, redis_conn, from_address=None): try: web3_tasks = [ self._async_web3_call( - contract_function=task, redis_conn=redis_conn, from_address=from_address, - ) for task in tasks + contract_function=task, + redis_conn=redis_conn, + from_address=from_address, + block=block, + overrides=overrides + ) + for task in tasks ] response = await asyncio.gather(*web3_tasks) return response