[EXP-16-218] Treasury and SMS on fantom

Makefile

@@ -51,10 +51,13 @@ rebuild: down build up
 scratch: clean-volumes build up
 
 logs:
-	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter transactions-exporter wallet-exporter sms-exporter historical-sms-exporter
+	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter
 
 test:
 	$(test_command) up
 
 all:
 	$(all_command) down && $(all_command) build --no-cache && $(all_command) up $(flags)
+
+logs-all:
+	$(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter wallet-exporter

scripts/historical_sms_exporter.py

@@ -1,22 +1,31 @@
 import logging
 from datetime import datetime, timezone
 
+from brownie import chain
 from yearn.historical_helper import export_historical, time_tracking
+from yearn.networks import Network
 from yearn.treasury.treasury import StrategistMultisig
 from yearn.utils import closest_block_after_timestamp
 
 logger = logging.getLogger('yearn.historical_sms_exporter')
 
 def main():
     start = datetime.now(tz=timezone.utc)
-    # end: 2021-01-28 09:09:48 first inbound sms tx
-    end = datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc)
+    if Network(chain.id) == Network.Fantom:
+        # end: 2021-06-17 Fantom SMS deployed
+        end = datetime(2021, 6, 17, tzinfo=timezone.utc)
+        data_query = 'sms_assets{network="FTM"}'
+    else:
+        # end: 2021-01-28 09:09:48 first inbound sms tx
+        end = datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc)
+        data_query = 'sms_assets{network="ETH"}'
+
     export_historical(
         start,
         end,
         export_chunk,
         export_snapshot,
-        'sms_assets'
+        data_query
     )
 
 

scripts/historical_treasury_exporter.py

@@ -1,22 +1,31 @@
 import logging
 from datetime import datetime, timezone
 
+from brownie import chain
 from yearn.historical_helper import export_historical, time_tracking
+from yearn.networks import Network
 from yearn.treasury.treasury import YearnTreasury
 from yearn.utils import closest_block_after_timestamp
 
 logger = logging.getLogger('yearn.historical_treasury_exporter')
 
 def main():
     start = datetime.now(tz=timezone.utc)
-    # end: 2020-07-21 first treasury tx
-    end = datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc)
+    if Network(chain.id) == Network.Fantom:
+        # end: 2021-10-12 Fantom Multisig deployed
+        end = datetime(2021, 10, 12, tzinfo=timezone.utc)
+        data_query = 'treasury_assets{network="FTM"}'
+    else:
+        # end: 2020-07-21 first treasury tx
+        end = datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc)
+        data_query = 'treasury_assets{network="ETH"}'
+
     export_historical(
         start,
         end,
         export_chunk,
         export_snapshot,
-        'treasury_assets'
+        data_query
     )
 
 

scripts/transactions_exporter.py

@@ -7,11 +7,11 @@
 from brownie import ZERO_ADDRESS, Contract, chain, web3
 from brownie.exceptions import BrownieEnvironmentWarning
 from pony.orm import db_session
-from joblib import Parallel, delayed
 from web3._utils.abi import filter_by_name
 from web3._utils.events import construct_event_topic_set
 from yearn.entities import UserTx  # , TreasuryTx
 from yearn.events import decode_logs, get_logs_asap
+from yearn.networks import Network
 from yearn.outputs.postgres.utils import (cache_address, cache_token,
                                           last_recorded_block)
 from yearn.prices import magic
@@ -25,6 +25,11 @@
 
 BATCH_SIZE = 5000
 
+FIRST_END_BLOCK = {
+    Network.Mainnet: 9480000, # NOTE block some arbitrary time after iearn's first deployment
+    Network.Fantom: 5000000, # NOTE block some arbitrary time after v2's first deployment
+}[chain.id]
+
 def main():
     for block in chain.new_blocks(height_buffer=1):
         process_and_cache_user_txs(last_recorded_block(UserTx))
@@ -36,7 +41,7 @@ def process_and_cache_user_txs(last_saved_block=None):
     max_block_to_cache = chain.height - 50
     start_block = last_saved_block + 1 if last_saved_block else None
     end_block = (
-        9480000 if start_block is None # NOTE block some arbitrary time after iearn's first deployment
+        FIRST_END_BLOCK if start_block is None
         else start_block + BATCH_SIZE if start_block + BATCH_SIZE < max_block_to_cache
         else max_block_to_cache
     )
@@ -130,7 +135,7 @@ def _get_price(event, token_entity):
                 logger.warn('trying again...')
                 time.sleep(5)
             else:
-                logger.warn(f'vault: {token_entity.token.address}')
+                logger.warn(f'vault: {token_entity.address.address}')
                 raise Exception(str(e))
 
 

scripts/wallet_exporter.py

@@ -12,17 +12,18 @@
 logger = logging.getLogger('yearn.wallet_exporter')
 
 def main():
-    start = datetime.now(tz=timezone.utc)
-    # end: 2020-02-12 first iearn deployment
-    end = datetime(2020, 2, 12, tzinfo=timezone.utc)
-    export_historical(
-        start,
-        end,
-        export_chunk,
-        export_snapshot,
-        'aggregate{param="total wallets"}',
-        _generate_snapshot_range
-    )
+    while True:
+        start = datetime.now(tz=timezone.utc)
+        # end: 2020-02-12 first iearn deployment
+        end = datetime(2020, 2, 12, tzinfo=timezone.utc)
+        export_historical(
+            start,
+            end,
+            export_chunk,
+            export_snapshot,
+            'aggregate{param="total wallets"}',
+            _generate_snapshot_range
+        )
 
 
 def export_chunk(chunk, export_snapshot_func):

services/dashboard/docker-compose.yml

@@ -91,30 +91,31 @@ services:
       - yearn-exporter
     restart: on-failure
 
-  wallet-exporter:
+  treasury-exporter:
     image: yearn-exporter
-    command: wallet_exporter
+    command: treasury_exporter
     environment:
       - WEB3_PROVIDER
       - ETHERSCAN_TOKEN
       - EXPLORER
       - SLEEP_SECONDS
-      - PGHOST=postgres
-      - PGDATABASE=postgres
-      - PGUSER=postgres
-      - PGPASSWORD=yearn-exporter
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
       - brownie:/root/.brownie
       - cache:/app/yearn-exporter/cache
     networks:
       - yearn-exporter
-    restart: always
+    restart: on-failure
+    depends_on:
+      - postgres
 
-  treasury-exporter:
+  historical-treasury-exporter:
     image: yearn-exporter
-    command: treasury_exporter
+    command: historical_treasury_exporter
     environment:
       - WEB3_PROVIDER
       - ETHERSCAN_TOKEN
@@ -123,6 +124,28 @@ services:
       - POOL_SIZE
       - CHUNK_SIZE
       - RESOLUTION
+      - SKIP_WALLET_STATS
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
+  ftm-treasury-exporter:
+    image: yearn-exporter
+    command: treasury_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
@@ -134,18 +157,18 @@ services:
     depends_on:
       - postgres
 
-  historical-treasury-exporter:
+  historical-ftm-treasury-exporter:
     image: yearn-exporter
     command: historical_treasury_exporter
     environment:
-      - WEB3_PROVIDER
-      - ETHERSCAN_TOKEN
-      - EXPLORER
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
       - SLEEP_SECONDS
+      - NETWORK=ftm-main
       - POOL_SIZE
       - CHUNK_SIZE
       - RESOLUTION
-      - SKIP_WALLET_STATS
     volumes:
       - solidity_compilers:/root/.solcx
       - vyper_compilers:/root/.vvm
@@ -189,10 +212,55 @@ services:
       - yearn-exporter
     restart: on-failure
 
+  ftm-sms-exporter:
+    image: yearn-exporter
+    command: sms_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
+  historical-ftm-sms-exporter:
+    image: yearn-exporter
+    command: historical_sms_exporter
+    environment:
+      - WEB3_PROVIDER=$FTM_WEB3_PROVIDER
+      - EXPLORER=$FTM_EXPLORER
+      - FTMSCAN_TOKEN
+      - SLEEP_SECONDS
+      - NETWORK=ftm-main
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: on-failure
+
   transactions-exporter:
     image: yearn-exporter
     command: transactions_exporter
     environment:
+      - WEB3_PROVIDER
+      - ETHERSCAN_TOKEN
+      - EXPLORER
       - PGHOST=postgres
       - PGDATABASE=postgres
       - PGUSER=postgres
@@ -208,6 +276,31 @@ services:
     depends_on:
       - postgres
 
+  wallet-exporter:
+    image: yearn-exporter
+    command: wallet_exporter
+    environment:
+      - WEB3_PROVIDER
+      - ETHERSCAN_TOKEN
+      - EXPLORER
+      - SLEEP_SECONDS
+      - PGHOST=postgres
+      - PGDATABASE=postgres
+      - PGUSER=postgres
+      - PGPASSWORD=yearn-exporter
+      - POOL_SIZE
+      - CHUNK_SIZE
+      - RESOLUTION
+    volumes:
+      - solidity_compilers:/root/.solcx
+      - vyper_compilers:/root/.vvm
+      - brownie:/root/.brownie
+      - cache:/app/yearn-exporter/cache
+    networks:
+      - yearn-exporter
+    restart: always
+
+
   vmagent:
     image: victoriametrics/vmagent
     volumes:

yearn/constants.py

@@ -143,4 +143,4 @@
     Network.Fantom: {
         "0x72a34AbafAB09b15E7191822A679f28E067C4a16",
     }
-}.get(chain.id,set())
+}.get(chain.id,set())

yearn/outputs/describers/vault.py

@@ -1,10 +1,10 @@
 import logging
+
 from brownie import chain
+from yearn.exceptions import PriceError
 from yearn.networks import Network
-
 from yearn.outputs.postgres.utils import fetch_balances
 from yearn.prices import magic
-from yearn.exceptions import PriceError
 
 logger = logging.getLogger(__name__)
 

yearn/outputs/postgres/utils.py

@@ -1,27 +1,23 @@
-from brownie import Contract, chain, ZERO_ADDRESS
-from pony.orm import ObjectNotFound, db_session, TransactionIntegrityError, select
-from yearn.cache import memory
-import logging
+from brownie import chain, ZERO_ADDRESS
+from pony.orm import db_session, select
 from yearn.entities import Address, Token, UserTx, db
 from yearn.multicall2 import fetch_multicall
-from yearn.utils import is_contract
+from yearn.utils import is_contract, contract
 
 @db_session
-@memory.cache()
 def cache_address(address: str) -> Address:
     address_entity = Address.get(chainid=chain.id, address=address)
     if not address_entity:
         address_entity = Address(chainid=chain.id, address=address, is_contract=is_contract(address))
     return address_entity
 
 @db_session
-@memory.cache()
 def cache_token(address: str) -> Token:
     address_entity = cache_address(address)
     token = Token.get(address=address_entity)
     if not token:
-        contract = Contract(address)
-        symbol, name, decimals = fetch_multicall([contract,'symbol'],[contract,'name'],[contract,'decimals'])
+        token = contract(address)
+        symbol, name, decimals = fetch_multicall([token,'symbol'],[token,'name'],[token,'decimals'])
         token = Token(address=address_entity, symbol=symbol, name=name, decimals=decimals)
         print(f'token {symbol} added to postgres')
     return token