[EXP-16-218] Treasury and SMS on fantom (#218)

* feat: env variable to skip wallet stats for speed * feat: postgres tx caching * fix: price edge cases for wallet exporter * feat: add new exporters to make logs * fix: treasury params for multichain * feat: make wallet exporter loop instead of restarting container * fix: fantom env vars * feat: make logs-all Co-authored-by: banteg <[email protected]> Co-authored-by: banteg <[email protected]>
yearn · Feb 2, 2022 · adf3c4a · adf3c4a
1 parent 2922916
commit adf3c4a
Show file tree

Hide file tree

Showing 13 changed files with 234 additions and 81 deletions.
diff --git a/Makefile b/Makefile
@@ -51,10 +51,13 @@ rebuild: down build up
 scratch: clean-volumes build up
 
 logs:
-	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter transactions-exporter wallet-exporter sms-exporter historical-sms-exporter
+	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter
 
 test:
 	$(test_command) up
 
 all:
 	$(all_command) down && $(all_command) build --no-cache && $(all_command) up $(flags)
+
+logs-all:
+	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter wallet-exporter
diff --git a/scripts/historical_sms_exporter.py b/scripts/historical_sms_exporter.py
@@ -1,22 +1,34 @@
 import logging
 from datetime import datetime, timezone
 
+from brownie import chain
 from yearn.historical_helper import export_historical, time_tracking
+from yearn.networks import Network
 from yearn.treasury.treasury import StrategistMultisig
 from yearn.utils import closest_block_after_timestamp
 
 logger = logging.getLogger('yearn.historical_sms_exporter')
 
+
 def main():
     start = datetime.now(tz=timezone.utc)
-    # end: 2021-01-28 09:09:48 first inbound sms tx
-    end = datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc)
+
+    end = {
+        Network.Mainnet: datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc), # first inbound sms tx
+        Network.Fantom: datetime(2021, 6, 17, tzinfo=timezone.utc), # Fantom SMS deployed
+    }[chain.id]
+
+    data_query = {
+        Network.Mainnet: 'sms_assets{network="ETH"}',
+        Network.Fantom: 'sms_assets{network="FTM"}',
+    }[chain.id]
+
     export_historical(
         start,
         end,
         export_chunk,
         export_snapshot,
-        'sms_assets'
+        data_query
     )
 
 

diff --git a/scripts/historical_treasury_exporter.py b/scripts/historical_treasury_exporter.py
@@ -1,22 +1,33 @@
 import logging
 from datetime import datetime, timezone
 
+from brownie import chain
 from yearn.historical_helper import export_historical, time_tracking
+from yearn.networks import Network
 from yearn.treasury.treasury import YearnTreasury
 from yearn.utils import closest_block_after_timestamp
 
 logger = logging.getLogger('yearn.historical_treasury_exporter')
 
 def main():
     start = datetime.now(tz=timezone.utc)
-    # end: 2020-07-21 first treasury tx
-    end = datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc)
+
+    end = {
+        Network.Mainnet: datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc), # first treasury tx
+        Network.Fantom: datetime(2021, 10, 12, tzinfo=timezone.utc), # Fantom Multisig deployed
+    }[chain.id]
+
+    data_query = {
+        Network.Mainnet: 'treasury_assets{network="ETH"}',
+        Network.Fantom: 'treasury_assets{network="FTM"}',
+    }[chain.id]
+
     export_historical(
         start,
         end,
         export_chunk,
         export_snapshot,
-        'treasury_assets'
+        data_query
     )
 
 

diff --git a/scripts/transactions_exporter.py b/scripts/transactions_exporter.py
@@ -7,11 +7,11 @@
 from brownie import ZERO_ADDRESS, Contract, chain, web3
 from brownie.exceptions import BrownieEnvironmentWarning
 from pony.orm import db_session
-from joblib import Parallel, delayed
 from web3._utils.abi import filter_by_name
 from web3._utils.events import construct_event_topic_set
 from yearn.entities import UserTx  # , TreasuryTx
 from yearn.events import decode_logs, get_logs_asap
+from yearn.networks import Network
 from yearn.outputs.postgres.utils import (cache_address, cache_token,
                                           last_recorded_block)
 from yearn.prices import magic
@@ -25,6 +25,11 @@
 
 BATCH_SIZE = 5000
 
+FIRST_END_BLOCK = {
+    Network.Mainnet: 9480000, # NOTE block some arbitrary time after iearn's first deployment
+    Network.Fantom: 5000000, # NOTE block some arbitrary time after v2's first deployment
+}[chain.id]
+
 def main():
     for block in chain.new_blocks(height_buffer=1):
         process_and_cache_user_txs(last_recorded_block(UserTx))
@@ -36,7 +41,7 @@ def process_and_cache_user_txs(last_saved_block=None):
     max_block_to_cache = chain.height - 50
     start_block = last_saved_block + 1 if last_saved_block else None
     end_block = (
-        9480000 if start_block is None # NOTE block some arbitrary time after iearn's first deployment
+        FIRST_END_BLOCK if start_block is None
         else start_block + BATCH_SIZE if start_block + BATCH_SIZE < max_block_to_cache
         else max_block_to_cache
     )
@@ -130,8 +135,8 @@ def _get_price(event, token_entity):
                 logger.warn('trying again...')
                 time.sleep(5)
             else:
-                logger.warn(f'vault: {token_entity.token.address}')
-                raise Exception(str(e))
+                logger.warn(f'vault: {token_entity.address.address}')
+                raise
 
 
 def _event_type(sender, receiver, vault_address) -> str:

diff --git a/scripts/wallet_exporter.py b/scripts/wallet_exporter.py
@@ -12,17 +12,18 @@
 logger = logging.getLogger('yearn.wallet_exporter')
 
 def main():
-    start = datetime.now(tz=timezone.utc)
-    # end: 2020-02-12 first iearn deployment
-    end = datetime(2020, 2, 12, tzinfo=timezone.utc)
-    export_historical(
-        start,
-        end,
-        export_chunk,
-        export_snapshot,
-        'aggregate{param="total wallets"}',
-        _generate_snapshot_range
-    )
+    while True:
+        start = datetime.now(tz=timezone.utc)
+        # end: 2020-02-12 first iearn deployment
+        end = datetime(2020, 2, 12, tzinfo=timezone.utc)
+        export_historical(
+            start,
+            end,
+            export_chunk,
+            export_snapshot,
+            'aggregate{param="total wallets"}',
+            _generate_snapshot_range
+        )
 
 
 def export_chunk(chunk, export_snapshot_func):

diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml
@@ -91,30 +91,31 @@ services:
       - yearn-exporter
     restart: on-failure
 
-  wallet-exporter:
+  treasury-exporter:
     image: yearn-exporter
-    command: wallet_exporter
+    command: treasury_exporter
     environment:
       - WEB3_PROVIDER
       - ETHERSCAN_TOKEN
       - EXPLORER
       - SLEEP_SECONDS
-      - PGHOST=postgres
-      - PGDATABASE=postgres
-      - PGUSER=postgres
-      - PGPASSWORD=yearn-exporter
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
       - brownie:/root/.brownie
       - cache:/app/yearn-exporter/cache
     networks:
       - yearn-exporter
-    restart: always
+    restart: on-failure
+    depends_on:
+      - postgres
 
-  treasury-exporter:
+  historical-treasury-exporter:
     image: yearn-exporter
-    command: treasury_exporter
+    command: historical_treasury_exporter
     environment:
       - WEB3_PROVIDER
       - ETHERSCAN_TOKEN
@@ -123,6 +124,28 @@ services:
       - POOL_SIZE
       - CHUNK_SIZE
       - RESOLUTION
+      - SKIP_WALLET_STATS
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
+  ftm-treasury-exporter:
+    image: yearn-exporter
+    command: treasury_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
@@ -134,18 +157,18 @@ services:
     depends_on:
       - postgres
 
-  historical-treasury-exporter:
+  historical-ftm-treasury-exporter:
     image: yearn-exporter
     command: historical_treasury_exporter
     environment:
-      - WEB3_PROVIDER
-      - ETHERSCAN_TOKEN
-      - EXPLORER
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
       - SLEEP_SECONDS
+      - NETWORK=ftm-main
       - POOL_SIZE
       - CHUNK_SIZE
       - RESOLUTION
-      - SKIP_WALLET_STATS
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
@@ -189,10 +212,55 @@ services:
       - yearn-exporter
     restart: on-failure
 
+  ftm-sms-exporter:
+    image: yearn-exporter
+    command: sms_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
+  historical-ftm-sms-exporter:
+    image: yearn-exporter
+    command: historical_sms_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
   transactions-exporter:
     image: yearn-exporter
     command: transactions_exporter
     environment:
+      - WEB3_PROVIDER
+      - ETHERSCAN_TOKEN
+      - EXPLORER
       - PGHOST=postgres
       - PGDATABASE=postgres
       - PGUSER=postgres
@@ -208,6 +276,31 @@ services:
     depends_on:
       - postgres
 
+  wallet-exporter:
+    image: yearn-exporter
+    command: wallet_exporter
+    environment:
+      - WEB3_PROVIDER
+      - ETHERSCAN_TOKEN
+      - EXPLORER
+      - SLEEP_SECONDS
+      - PGHOST=postgres
+      - PGDATABASE=postgres
+      - PGUSER=postgres
+      - PGPASSWORD=yearn-exporter
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: always
+
+
   vmagent:
     image: victoriametrics/vmagent
     volumes:

diff --git a/yearn/constants.py b/yearn/constants.py
@@ -143,4 +143,4 @@
     Network.Fantom: {
         "0x72a34AbafAB09b15E7191822A679f28E067C4a16",
     }
-}.get(chain.id,set())
+}.get(chain.id,set())
diff --git a/yearn/outputs/describers/vault.py b/yearn/outputs/describers/vault.py
@@ -1,10 +1,10 @@
 import logging
+
 from brownie import chain
+from yearn.exceptions import PriceError
 from yearn.networks import Network
-
 from yearn.outputs.postgres.utils import fetch_balances
 from yearn.prices import magic
-from yearn.exceptions import PriceError
 
 logger = logging.getLogger(__name__)