diff --git a/Makefile b/Makefile index 6c2c207c0..af27e7c4d 100644 --- a/Makefile +++ b/Makefile @@ -51,10 +51,13 @@ rebuild: down build up scratch: clean-volumes build up logs: - $(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter transactions-exporter wallet-exporter sms-exporter historical-sms-exporter + $(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter test: $(test_command) up all: $(all_command) down && $(all_command) build --no-cache && $(all_command) up $(flags) + +logs-all: + $(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter ftm-treasury-exporter historical-ftm-treasury-exporter sms-exporter historical-sms-exporter ftm-sms-exporter historical-ftm-sms-exporter transactions-exporter wallet-exporter \ No newline at end of file diff --git a/scripts/historical_sms_exporter.py b/scripts/historical_sms_exporter.py index 7cc7988d6..96b8a615b 100644 --- a/scripts/historical_sms_exporter.py +++ b/scripts/historical_sms_exporter.py @@ -1,22 +1,34 @@ import logging from datetime import datetime, timezone +from brownie import chain from yearn.historical_helper import export_historical, time_tracking +from yearn.networks import Network from yearn.treasury.treasury import StrategistMultisig from yearn.utils import closest_block_after_timestamp logger = logging.getLogger('yearn.historical_sms_exporter') + def main(): start = datetime.now(tz=timezone.utc) - # end: 2021-01-28 09:09:48 first inbound sms tx - end = datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc) + + end = { + Network.Mainnet: datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc), # first inbound sms tx + Network.Fantom: datetime(2021, 6, 17, tzinfo=timezone.utc), # Fantom SMS deployed + }[chain.id] + + data_query = { + Network.Mainnet: 'sms_assets{network="ETH"}', + Network.Fantom: 'sms_assets{network="FTM"}', + }[chain.id] + export_historical( start, end, export_chunk, export_snapshot, - 'sms_assets' + data_query ) diff --git a/scripts/historical_treasury_exporter.py b/scripts/historical_treasury_exporter.py index 71091c07c..a4acb6ce5 100644 --- a/scripts/historical_treasury_exporter.py +++ b/scripts/historical_treasury_exporter.py @@ -1,7 +1,9 @@ import logging from datetime import datetime, timezone +from brownie import chain from yearn.historical_helper import export_historical, time_tracking +from yearn.networks import Network from yearn.treasury.treasury import YearnTreasury from yearn.utils import closest_block_after_timestamp @@ -9,14 +11,23 @@ def main(): start = datetime.now(tz=timezone.utc) - # end: 2020-07-21 first treasury tx - end = datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc) + + end = { + Network.Mainnet: datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc), # first treasury tx + Network.Fantom: datetime(2021, 10, 12, tzinfo=timezone.utc), # Fantom Multisig deployed + }[chain.id] + + data_query = { + Network.Mainnet: 'treasury_assets{network="ETH"}', + Network.Fantom: 'treasury_assets{network="FTM"}', + }[chain.id] + export_historical( start, end, export_chunk, export_snapshot, - 'treasury_assets' + data_query ) diff --git a/scripts/transactions_exporter.py b/scripts/transactions_exporter.py index 6b4ed4b03..68f805502 100644 --- a/scripts/transactions_exporter.py +++ b/scripts/transactions_exporter.py @@ -7,11 +7,11 @@ from brownie import ZERO_ADDRESS, Contract, chain, web3 from brownie.exceptions import BrownieEnvironmentWarning from pony.orm import db_session -from joblib import Parallel, delayed from web3._utils.abi import filter_by_name from web3._utils.events import construct_event_topic_set from yearn.entities import UserTx # , TreasuryTx from yearn.events import decode_logs, get_logs_asap +from yearn.networks import Network from yearn.outputs.postgres.utils import (cache_address, cache_token, last_recorded_block) from yearn.prices import magic @@ -25,6 +25,11 @@ BATCH_SIZE = 5000 +FIRST_END_BLOCK = { + Network.Mainnet: 9480000, # NOTE block some arbitrary time after iearn's first deployment + Network.Fantom: 5000000, # NOTE block some arbitrary time after v2's first deployment +}[chain.id] + def main(): for block in chain.new_blocks(height_buffer=1): process_and_cache_user_txs(last_recorded_block(UserTx)) @@ -36,7 +41,7 @@ def process_and_cache_user_txs(last_saved_block=None): max_block_to_cache = chain.height - 50 start_block = last_saved_block + 1 if last_saved_block else None end_block = ( - 9480000 if start_block is None # NOTE block some arbitrary time after iearn's first deployment + FIRST_END_BLOCK if start_block is None else start_block + BATCH_SIZE if start_block + BATCH_SIZE < max_block_to_cache else max_block_to_cache ) @@ -130,8 +135,8 @@ def _get_price(event, token_entity): logger.warn('trying again...') time.sleep(5) else: - logger.warn(f'vault: {token_entity.token.address}') - raise Exception(str(e)) + logger.warn(f'vault: {token_entity.address.address}') + raise def _event_type(sender, receiver, vault_address) -> str: diff --git a/scripts/wallet_exporter.py b/scripts/wallet_exporter.py index 37efe1707..e5b1b0a09 100644 --- a/scripts/wallet_exporter.py +++ b/scripts/wallet_exporter.py @@ -12,17 +12,18 @@ logger = logging.getLogger('yearn.wallet_exporter') def main(): - start = datetime.now(tz=timezone.utc) - # end: 2020-02-12 first iearn deployment - end = datetime(2020, 2, 12, tzinfo=timezone.utc) - export_historical( - start, - end, - export_chunk, - export_snapshot, - 'aggregate{param="total wallets"}', - _generate_snapshot_range - ) + while True: + start = datetime.now(tz=timezone.utc) + # end: 2020-02-12 first iearn deployment + end = datetime(2020, 2, 12, tzinfo=timezone.utc) + export_historical( + start, + end, + export_chunk, + export_snapshot, + 'aggregate{param="total wallets"}', + _generate_snapshot_range + ) def export_chunk(chunk, export_snapshot_func): diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index 916bdb26c..f4085b591 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -91,18 +91,17 @@ services: - yearn-exporter restart: on-failure - wallet-exporter: + treasury-exporter: image: yearn-exporter - command: wallet_exporter + command: treasury_exporter environment: - WEB3_PROVIDER - ETHERSCAN_TOKEN - EXPLORER - SLEEP_SECONDS - - PGHOST=postgres - - PGDATABASE=postgres - - PGUSER=postgres - - PGPASSWORD=yearn-exporter + - POOL_SIZE + - CHUNK_SIZE + - RESOLUTION volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -110,11 +109,13 @@ services: - cache:/app/yearn-exporter/cache networks: - yearn-exporter - restart: always + restart: on-failure + depends_on: + - postgres - treasury-exporter: + historical-treasury-exporter: image: yearn-exporter - command: treasury_exporter + command: historical_treasury_exporter environment: - WEB3_PROVIDER - ETHERSCAN_TOKEN @@ -123,6 +124,28 @@ services: - POOL_SIZE - CHUNK_SIZE - RESOLUTION + - SKIP_WALLET_STATS + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: on-failure + + ftm-treasury-exporter: + image: yearn-exporter + command: treasury_exporter + environment: + - WEB3_PROVIDER=$FTM_WEB3_PROVIDER + - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN + - SLEEP_SECONDS + - NETWORK=ftm-main + - POOL_SIZE + - CHUNK_SIZE + - RESOLUTION volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -134,18 +157,18 @@ services: depends_on: - postgres - historical-treasury-exporter: + historical-ftm-treasury-exporter: image: yearn-exporter command: historical_treasury_exporter environment: - - WEB3_PROVIDER - - ETHERSCAN_TOKEN - - EXPLORER + - WEB3_PROVIDER=$FTM_WEB3_PROVIDER + - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN - SLEEP_SECONDS + - NETWORK=ftm-main - POOL_SIZE - CHUNK_SIZE - RESOLUTION - - SKIP_WALLET_STATS volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -189,10 +212,55 @@ services: - yearn-exporter restart: on-failure + ftm-sms-exporter: + image: yearn-exporter + command: sms_exporter + environment: + - WEB3_PROVIDER=$FTM_WEB3_PROVIDER + - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN + - SLEEP_SECONDS + - NETWORK=ftm-main + - POOL_SIZE + - CHUNK_SIZE + - RESOLUTION + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: on-failure + + historical-ftm-sms-exporter: + image: yearn-exporter + command: historical_sms_exporter + environment: + - WEB3_PROVIDER=$FTM_WEB3_PROVIDER + - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN + - SLEEP_SECONDS + - NETWORK=ftm-main + - POOL_SIZE + - CHUNK_SIZE + - RESOLUTION + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: on-failure + transactions-exporter: image: yearn-exporter command: transactions_exporter environment: + - WEB3_PROVIDER + - ETHERSCAN_TOKEN + - EXPLORER - PGHOST=postgres - PGDATABASE=postgres - PGUSER=postgres @@ -208,6 +276,31 @@ services: depends_on: - postgres + wallet-exporter: + image: yearn-exporter + command: wallet_exporter + environment: + - WEB3_PROVIDER + - ETHERSCAN_TOKEN + - EXPLORER + - SLEEP_SECONDS + - PGHOST=postgres + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=yearn-exporter + - POOL_SIZE + - CHUNK_SIZE + - RESOLUTION + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: always + + vmagent: image: victoriametrics/vmagent volumes: diff --git a/yearn/constants.py b/yearn/constants.py index e6365872d..050f81a5b 100644 --- a/yearn/constants.py +++ b/yearn/constants.py @@ -143,4 +143,4 @@ Network.Fantom: { "0x72a34AbafAB09b15E7191822A679f28E067C4a16", } -}.get(chain.id,set()) \ No newline at end of file +}.get(chain.id,set()) diff --git a/yearn/outputs/describers/vault.py b/yearn/outputs/describers/vault.py index be71278c6..f589ef0b1 100644 --- a/yearn/outputs/describers/vault.py +++ b/yearn/outputs/describers/vault.py @@ -1,10 +1,10 @@ import logging + from brownie import chain +from yearn.exceptions import PriceError from yearn.networks import Network - from yearn.outputs.postgres.utils import fetch_balances from yearn.prices import magic -from yearn.exceptions import PriceError logger = logging.getLogger(__name__) diff --git a/yearn/outputs/postgres/utils.py b/yearn/outputs/postgres/utils.py index 935675cb5..1e95c2d97 100644 --- a/yearn/outputs/postgres/utils.py +++ b/yearn/outputs/postgres/utils.py @@ -1,13 +1,10 @@ -from brownie import Contract, chain, ZERO_ADDRESS -from pony.orm import ObjectNotFound, db_session, TransactionIntegrityError, select -from yearn.cache import memory -import logging +from brownie import chain, ZERO_ADDRESS +from pony.orm import db_session, select from yearn.entities import Address, Token, UserTx, db from yearn.multicall2 import fetch_multicall -from yearn.utils import is_contract +from yearn.utils import is_contract, contract @db_session -@memory.cache() def cache_address(address: str) -> Address: address_entity = Address.get(chainid=chain.id, address=address) if not address_entity: @@ -15,13 +12,12 @@ def cache_address(address: str) -> Address: return address_entity @db_session -@memory.cache() def cache_token(address: str) -> Token: address_entity = cache_address(address) token = Token.get(address=address_entity) if not token: - contract = Contract(address) - symbol, name, decimals = fetch_multicall([contract,'symbol'],[contract,'name'],[contract,'decimals']) + token = contract(address) + symbol, name, decimals = fetch_multicall([token,'symbol'],[token,'name'],[token,'decimals']) token = Token(address=address_entity, symbol=symbol, name=name, decimals=decimals) print(f'token {symbol} added to postgres') return token diff --git a/yearn/treasury/buckets.py b/yearn/treasury/buckets.py index 55a25254c..b62f77f3b 100644 --- a/yearn/treasury/buckets.py +++ b/yearn/treasury/buckets.py @@ -1,31 +1,49 @@ -from brownie import Contract +from brownie import chain from yearn.cache import memory -from yearn.constants import BTC_LIKE, ETH_LIKE as _ETH_LIKE -import yearn.prices.balancer as bal +from yearn.constants import BTC_LIKE +from yearn.constants import ETH_LIKE as _ETH_LIKE +from yearn.networks import Network +from yearn.prices import balancer as bal +from yearn.prices.aave import aave from yearn.prices.compound import compound from yearn.prices.constants import stablecoins, weth -from yearn.prices.fixed_forex import fixed_forex -from yearn.prices.aave import aave from yearn.prices.curve import curve +from yearn.prices.fixed_forex import fixed_forex from yearn.prices.yearn import yearn_lens from yearn.utils import contract YFI_LIKE = { - '0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e', # YFI - '0xD0660cD418a64a1d44E9214ad8e459324D8157f1', # WOOFY -} + Network.Mainnet: { + '0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e', # YFI + '0xD0660cD418a64a1d44E9214ad8e459324D8157f1', # WOOFY + }, + Network.Fantom: { + '0x29b0Da86e484E1C0029B56e817912d778aC0EC69', # YFI + '0xD0660cD418a64a1d44E9214ad8e459324D8157f1', # WOOFY + }, +}.get(chain.id, set()) INTL_STABLECOINS = { - '0xD71eCFF9342A5Ced620049e616c5035F1dB98620', # sEUR - '0xC581b735A1688071A1746c968e0798D642EDE491', # EURT - '0xdB25f211AB05b1c97D595516F45794528a807ad8', # EURS - '0x96E61422b6A9bA0e068B6c5ADd4fFaBC6a4aae27', # ibEUR -} + Network.Mainnet: { + '0xD71eCFF9342A5Ced620049e616c5035F1dB98620', # sEUR + '0xC581b735A1688071A1746c968e0798D642EDE491', # EURT + '0xdB25f211AB05b1c97D595516F45794528a807ad8', # EURS + '0x96E61422b6A9bA0e068B6c5ADd4fFaBC6a4aae27', # ibEUR + }, + Network.Fantom: { + '', # + }, +}.get(chain.id, set()) OTHER_LONG_TERM_ASSETS = { - '0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44', # KP3R - '0xaf988afF99d3d0cb870812C325C588D8D8CB7De8', # SLP (KP3R/ETH) -} + Network.Mainnet: { + '0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44', # KP3R + '0xaf988afF99d3d0cb870812C325C588D8D8CB7De8', # SLP (KP3R/ETH) + }, + Network.Fantom: { + '', # + }, +}.get(chain.id, set()) ETH_LIKE = _ETH_LIKE.union( { @@ -40,7 +58,7 @@ def get_token_bucket(token) -> str: token = str(token) token = str(_unwrap_token(token)) if ( - token in stablecoins or token in INTL_STABLECOINS or token in fixed_forex + token in stablecoins or token in INTL_STABLECOINS or (fixed_forex and token in fixed_forex) ): # or token == '0x9ba60bA98413A60dB4C651D4afE5C937bbD8044B': # yla return 'Cash & cash equivalents' if token in ETH_LIKE: @@ -75,7 +93,7 @@ def _unwrap_token(token) -> str: str(_unwrap_token(coin)) for coin in contract(token).getCurrentTokens() ) return _pool_bucket(pool_tokens) - if token in aave: + if aave and token in aave: return aave.atoken_underlying(token) if token in compound: try: diff --git a/yearn/treasury/treasury.py b/yearn/treasury/treasury.py index c0d6fa336..fe314784d 100644 --- a/yearn/treasury/treasury.py +++ b/yearn/treasury/treasury.py @@ -13,6 +13,7 @@ from yearn.events import decode_logs from yearn.exceptions import PriceError from yearn.multicall2 import fetch_multicall +from yearn.networks import Network from yearn.outputs.victoria import output_treasury from yearn.partners.partners import partners from yearn.partners.snapshot import WildcardWrapper, Wrapper @@ -197,6 +198,7 @@ def collateral(self, block=None) -> dict: return collateral def maker_collateral(self, block=None) -> dict: + if chain.id != Network.Mainnet: return proxy_registry = contract('0x4678f0a6958e4D2Bc4F1BAF7Bc52E8F3564f3fE4') cdp_manager = contract('0x5ef30b9986345249bc32d8928B7ee64DE9435E39') # ychad = contract('ychad.eth') @@ -220,9 +222,10 @@ def maker_collateral(self, block=None) -> dict: return collateral def unit_collateral(self, block=None) -> dict: + if chain.id != Network.Mainnet: return + if block and block < 11315910: return + # NOTE: This only works for YFI collateral, must extend before using for other collaterals - if block and block < 11315910: - return unitVault = contract("0xb1cff81b9305166ff1efc49a129ad2afcd7bcf19") yfi = "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e" collateral = {} @@ -262,12 +265,14 @@ def debt(self, block=None) -> dict: return debt def maker_debt(self, block=None) -> dict: + if chain.id != Network.Mainnet: return proxy_registry = contract('0x4678f0a6958e4D2Bc4F1BAF7Bc52E8F3564f3fE4') cdp_manager = contract('0x5ef30b9986345249bc32d8928B7ee64DE9435E39') vat = contract('0x35D1b3F3D7966A1DFe207aa4514C12a259A0492B') ilk = encode_single('bytes32', b'YFI-A') dai = '0x6B175474E89094C44Da98b954EedeAC495271d0F' maker_debt = {} + ilk = encode_single('bytes32', b'YFI-A') for address in self.addresses: proxy = proxy_registry.proxies(address) cdp = cdp_manager.first(proxy) @@ -279,9 +284,9 @@ def maker_debt(self, block=None) -> dict: return maker_debt def unit_debt(self, block=None) -> dict: + if chain.id != Network.Mainnet: return + if block and block < 11315910: return # NOTE: This only works for YFI based debt, must extend before using for other collaterals - if block and block < 11315910: - return unitVault = contract("0xb1cff81b9305166ff1efc49a129ad2afcd7bcf19") yfi = "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e" usdp = '0x1456688345527bE1f37E9e627DA0837D6f08C925' @@ -385,7 +390,11 @@ def export(self, block, ts): class YearnTreasury(Treasury): def __init__(self,watch_events_forever=False): - super().__init__('treasury',TREASURY_WALLETS,watch_events_forever=watch_events_forever,start_block=10502337) + start_block = { + Network.Mainnet: 10502337, + Network.Fantom: 18950072, + }[chain.id] + super().__init__('treasury',TREASURY_WALLETS,watch_events_forever=watch_events_forever,start_block=start_block) def partners_debt(self, block=None) -> dict: for i, partner in enumerate(partners): @@ -405,4 +414,8 @@ def partners_debt(self, block=None) -> dict: class StrategistMultisig(Treasury): def __init__(self,watch_events_forever=False): - super().__init__('sms',STRATEGIST_MULTISIG,watch_events_forever=watch_events_forever,start_block=11507716) + start_block = { + Network.Mainnet: 11507716, + Network.Fantom: 10836306, + }[chain.id] + super().__init__('sms',STRATEGIST_MULTISIG,watch_events_forever=watch_events_forever,start_block=start_block) diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 740b0c6b7..0386d6d25 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -1,5 +1,4 @@ import logging -import os import re import threading import time diff --git a/yearn/yearn.py b/yearn/yearn.py index 5565ba3ad..943c49951 100644 --- a/yearn/yearn.py +++ b/yearn/yearn.py @@ -1,4 +1,5 @@ import logging +import os from collections import Counter from time import time @@ -14,7 +15,7 @@ from yearn.networks import Network from yearn.outputs.victoria import output_base, output_wallets from yearn.prices import constants -from yearn.utils import contract +from yearn.utils import contract_creation_block logger = logging.getLogger(__name__) @@ -61,9 +62,14 @@ def active_vaults_at(self, block=None): vault for registry in self.registries.values() for vault in registry.active_vaults_at(block=block) - # [yGov] Doesn't count for this context - if vault.vault != contract("0xBa37B002AbaFDd8E89a1995dA52740bbC013D992") ] + + # [yGov] Doesn't count for this context + if chain.id == Network.Mainnet and ( + block is None + or block > contract_creation_block(yearn.special.Ygov().vault.address) + ): active.remove(yearn.special.Ygov()) + return active @@ -74,11 +80,8 @@ def describe(self, block=None): ) return dict(zip(self.registries, desc)) - def describe_wallets(self, block=None): - from yearn.outputs.describers.registry import RegistryWalletDescriber - describer = RegistryWalletDescriber() - data = Parallel(4,'threading')(delayed(describer.describe_wallets)(registry, block=block) for registry in self.registries.items()) + data = Parallel(4,'threading')(delayed(self.registries[key].describe_wallets)(block=block) for key in self.registries) data = {registry:desc for registry,desc in zip(self.registries,data)} wallet_balances = Counter() @@ -94,7 +97,6 @@ def describe_wallets(self, block=None): "wallet balances usd": wallet_balances } } - data.update(agg_stats) return data