diff --git a/Dockerfile b/Dockerfile index faddd7727..db6a2435a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,6 @@ FROM python:3.9-bullseye RUN mkdir -p /app/yearn-exporter WORKDIR /app/yearn-exporter -RUN apt-get update -RUN apt-get install -y build-essential odbc-postgresql python3-dev unixodbc-dev - ADD requirements.txt ./ RUN pip3 install -r requirements.txt diff --git a/Makefile b/Makefile index 95eb6635d..6c2c207c0 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ rebuild: down build up scratch: clean-volumes build up logs: - $(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter transactions-exporter wallet-exporter + $(dashboards_command) logs -f -t eth-exporter historical-eth-exporter ftm-exporter historical-ftm-exporter treasury-exporter historical-treasury-exporter transactions-exporter wallet-exporter sms-exporter historical-sms-exporter test: $(test_command) up diff --git a/grafana/provisioning/dashboards/yearn/treasury.json b/grafana/provisioning/dashboards/yearn/treasury.json new file mode 100644 index 000000000..c0c7bc743 --- /dev/null +++ b/grafana/provisioning/dashboards/yearn/treasury.json @@ -0,0 +1,1962 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 1, + "id": 26, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"balance\",token=\"YFI\"})", + "interval": "", + "legendFormat": "YFI Balance", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",token=\"YFI\"})", + "hide": false, + "interval": "", + "legendFormat": "YFI Value", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",token=\"yvYFI\"})", + "hide": false, + "interval": "", + "legendFormat": "yvYFI Value", + "refId": "C" + } + ], + "title": "YFI in Treasury", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "yfi price", + "binary": { + "left": "YFI Value", + "operator": "/", + "reducer": "sum", + "right": "YFI Balance" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "yvYFI Balance in YFI", + "binary": { + "left": "yvYFI Value", + "operator": "/", + "reducer": "sum", + "right": "yfi price" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "YFI Balance", + "binary": { + "left": "yvYFI Balance in YFI", + "operator": "+", + "reducer": "sum", + "right": "YFI Balance" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",token=\"YFI\"}) + sum(treasury_assets{param=\"usd value\",token=\"yvYFI\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "YFI in Treasury ($)", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket!=\"YFI\", bucket!=\"Other long term assets\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Liquid Assets", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"ETH\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "ETH", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"BTC\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "BTC", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\", bucket=\"Cash & cash equivalents\"}) ", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cash", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_debt{param=\"usd value\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Treasury Debt", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Cash & cash equivalents\"}) - sum(treasury_debt{param=\"usd value\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cash - Debt", + "type": "stat" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket!=\"YFI\", bucket!=\"Other long term assets\"}) by (bucket)", + "hide": false, + "interval": "", + "legendFormat": "{{bucket}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",token=\"YFI\"}) by (bucket)", + "hide": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Liquid Assets", + "type": "timeseries" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_debt{param=\"usd value\"}) by (token)", + "interval": "", + "legendFormat": "{{token}}", + "refId": "A" + } + ], + "title": "Protocol Debt", + "type": "timeseries" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket!=\"YFI\",bucket!=\"Other long term assets\"})", + "hide": false, + "interval": "", + "legendFormat": "Liquid Assets", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Cash & cash equivalents\"})", + "hide": false, + "interval": "", + "legendFormat": "Cash", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(treasury_debt{param=\"usd value\"})", + "hide": false, + "interval": "", + "legendFormat": "Debt", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Cash & cash equivalents\"}) - sum(treasury_debt{param=\"usd value\"})", + "hide": false, + "interval": "", + "legendFormat": "Cash - Debt", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket!=\"YFI\"}) - sum(treasury_debt{param=\"usd value\"})", + "hide": false, + "interval": "", + "legendFormat": "Liquid Assets - Debt", + "refId": "A" + } + ], + "title": "Assets - Debt", + "type": "timeseries" + }, + { + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_debt{param=\"usd value\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket!=\"Other long term assets\"})", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Debt to Asset Ratio", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Debt to Asset Ratio", + "binary": { + "left": "sum(treasury_debt{param=\"usd value\"})", + "operator": "/", + "reducer": "sum", + "right": "sum(treasury_assets{param=\"usd value\",bucket!=\"Other long term assets\"})" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 33, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Cash & cash equivalents\"}) by (token)", + "hide": false, + "interval": "", + "legendFormat": "{{token}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid Cash", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid Cash", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 31, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"ETH\"}) by (token)", + "hide": false, + "interval": "", + "legendFormat": "{{token}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid ETH", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid ETH", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 29, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"BTC\"}) by (token)", + "hide": false, + "interval": "", + "legendFormat": "{{token}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid BTC", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid BTC", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 27, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Other short term assets\"}) by (token)", + "hide": false, + "interval": "", + "legendFormat": "{{token}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid Everything Else", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid Everything Else", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 40, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Cash & cash equivalents\"}) by (token,wallet)", + "hide": false, + "interval": "", + "legendFormat": "{{token}} | {{wallet}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid Cash by Wallet", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid Cash by Wallet", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 42, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"ETH\"}) by (token,wallet)", + "hide": false, + "interval": "", + "legendFormat": "{{token}} | {{wallet}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid ETH by Wallet", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid ETH by Wallet", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 44, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"BTC\"}) by (token, wallet)", + "hide": false, + "interval": "", + "legendFormat": "{{token}} | {{wallet}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid BTC by Wallet", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid BTC by Wallet", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 47, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(treasury_assets{param=\"usd value\",bucket=\"Other short term assets\"}) by (token, wallet)", + "hide": false, + "interval": "", + "legendFormat": "{{token}} | {{wallet}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Liquid Everything Else by Wallet", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:72", + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:73", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Liquid Everything Else by Wallet", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 31, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Treasury", + "uid": "9_ppKhd7z", + "version": 45 +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 860136cad..925222af5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ boto3==1.17.88 rich>=10.11.0 matplotlib>=3.4.3 pandas>=1.3.0 -pyodbc>=4.0.32 +pony>=0.6.7 sqlalchemy>=1.4.26 diff --git a/scripts/exporter.py b/scripts/exporter.py index a4ed67aa5..15fc89268 100644 --- a/scripts/exporter.py +++ b/scripts/exporter.py @@ -4,7 +4,7 @@ from brownie import chain from yearn.yearn import Yearn -from yearn.outputs import victoria +from yearn.outputs.victoria import output_duration from yearn.prices import constants logger = logging.getLogger('yearn.exporter') @@ -16,7 +16,7 @@ def main(): start_time = time.time() yearn.export(block.number, block.timestamp) duration = time.time() - start_time - victoria.export_duration(duration, 1, "forwards", block.timestamp) + output_duration.export(duration, 1, "forwards", block.timestamp) time.sleep(sleep_interval) diff --git a/scripts/historical_exporter.py b/scripts/historical_exporter.py index 1b86a31ac..b1e9c50ca 100644 --- a/scripts/historical_exporter.py +++ b/scripts/historical_exporter.py @@ -12,9 +12,10 @@ def main(): start = datetime.now(tz=timezone.utc) if Network(chain.id) == Network.Fantom: - # end: 2021-04-30 first possible date after the fantom network upgrade + # end: 2021-04-30 first possible date after the Fantom network upgrade end = datetime(2021, 4, 30, tzinfo=timezone.utc) - data_query = 'yearn_vault{network="FTM"}' + # ironbank first product deployed on Fantom + data_query = 'ironbank{network="FTM"}' else: # end: 2020-02-12 first iearn deployment end = datetime(2020, 2, 12, 0, 1, tzinfo=timezone.utc) diff --git a/scripts/historical_sms_exporter.py b/scripts/historical_sms_exporter.py new file mode 100644 index 000000000..7cc7988d6 --- /dev/null +++ b/scripts/historical_sms_exporter.py @@ -0,0 +1,44 @@ +import logging +from datetime import datetime, timezone + +from yearn.historical_helper import export_historical, time_tracking +from yearn.treasury.treasury import StrategistMultisig +from yearn.utils import closest_block_after_timestamp + +logger = logging.getLogger('yearn.historical_sms_exporter') + +def main(): + start = datetime.now(tz=timezone.utc) + # end: 2021-01-28 09:09:48 first inbound sms tx + end = datetime(2021, 1, 28, 9, 10, tzinfo=timezone.utc) + export_historical( + start, + end, + export_chunk, + export_snapshot, + 'sms_assets' + ) + + +def export_chunk(chunk, export_snapshot_func): + sms = StrategistMultisig() + for snapshot in chunk: + ts = snapshot.timestamp() + export_snapshot_func( + { + 'treasury': sms, + 'snapshot': snapshot, + 'ts': ts, + 'exporter_name': 'historical_sms' + } + ) + + +@time_tracking +def export_snapshot(sms, snapshot, ts, exporter_name): + block = closest_block_after_timestamp(ts) + assert block is not None, "no block after timestamp found" + sms.export(block, ts) + logger.info("exported SMS snapshot %s", snapshot) + + diff --git a/scripts/historical_treasury_exporter.py b/scripts/historical_treasury_exporter.py index 4298af1ba..71091c07c 100644 --- a/scripts/historical_treasury_exporter.py +++ b/scripts/historical_treasury_exporter.py @@ -1,15 +1,16 @@ import logging from datetime import datetime, timezone + from yearn.historical_helper import export_historical, time_tracking +from yearn.treasury.treasury import YearnTreasury from yearn.utils import closest_block_after_timestamp -from yearn.treasury.treasury import Treasury logger = logging.getLogger('yearn.historical_treasury_exporter') def main(): start = datetime.now(tz=timezone.utc) - # end: 2020-02-12 first treasury tx - end = datetime(2020, 7, 21, tzinfo=timezone.utc) + # end: 2020-07-21 first treasury tx + end = datetime(2020, 7, 21, 10, 1, tzinfo=timezone.utc) export_historical( start, end, @@ -20,7 +21,7 @@ def main(): def export_chunk(chunk, export_snapshot_func): - treasury = Treasury() + treasury = YearnTreasury() for snapshot in chunk: ts = snapshot.timestamp() export_snapshot_func( diff --git a/scripts/sms_exporter.py b/scripts/sms_exporter.py new file mode 100644 index 000000000..d75834be1 --- /dev/null +++ b/scripts/sms_exporter.py @@ -0,0 +1,20 @@ +import logging +import os +import time + +from brownie import chain +from yearn.treasury.treasury import StrategistMultisig +from yearn.outputs.victoria import output_duration + +logger = logging.getLogger('yearn.sms_exporter') +sleep_interval = int(os.environ.get('SLEEP_SECONDS', '0')) + + +def main(): + treasury = StrategistMultisig(watch_events_forever=True) + for block in chain.new_blocks(height_buffer=12): + start_time = time.time() + treasury.export(block.number, block.timestamp) + duration = time.time() - start_time + output_duration.export(duration, 1, "sms_forwards", block.timestamp) + time.sleep(sleep_interval) diff --git a/scripts/transactions_exporter.py b/scripts/transactions_exporter.py index ba3b755d6..6b4ed4b03 100644 --- a/scripts/transactions_exporter.py +++ b/scripts/transactions_exporter.py @@ -1,63 +1,72 @@ +import logging import time import warnings from decimal import Decimal -import logging import pandas as pd from brownie import ZERO_ADDRESS, Contract, chain, web3 from brownie.exceptions import BrownieEnvironmentWarning +from pony.orm import db_session from joblib import Parallel, delayed -import sqlalchemy -from tqdm import tqdm from web3._utils.abi import filter_by_name from web3._utils.events import construct_event_topic_set -from yearn.events import create_filter, decode_logs, get_logs_asap -from yearn.outputs.postgres.postgres import postgres +from yearn.entities import UserTx # , TreasuryTx +from yearn.events import decode_logs, get_logs_asap +from yearn.outputs.postgres.utils import (cache_address, cache_token, + last_recorded_block) from yearn.prices import magic -from yearn.utils import contract -from yearn.v1.registry import Registry as RegistryV1 -from yearn.v2.registry import Registry as RegistryV2 -from yearn.treasury.treasury import Treasury - -treasury = Treasury() +from yearn.yearn import Yearn warnings.simplefilter("ignore", BrownieEnvironmentWarning) -registryV1 = RegistryV1() -registryV2 = RegistryV2() +yearn = Yearn(load_strategies=False) -logger = logging.getLogger(__name__) +logger = logging.getLogger('yearn.transactions_exporter') +BATCH_SIZE = 5000 def main(): for block in chain.new_blocks(height_buffer=1): - process_and_cache_user_txs(postgres.last_recorded_block('user_txs')) - - -def active_vaults_at(end_block) -> set: - v1 = {vault.vault for vault in registryV1.active_vaults_at(end_block)} - v2 = {vault.vault for vault in registryV2.active_vaults_at(end_block)} - return v1.union(v2) + process_and_cache_user_txs(last_recorded_block(UserTx)) +@db_session def process_and_cache_user_txs(last_saved_block=None): - max_block_to_cache = ( - chain.height - 50 - ) # We look 50 blocks back to avoid uncles and reorgs + # NOTE: We look 50 blocks back to avoid uncles and reorgs + max_block_to_cache = chain.height - 50 start_block = last_saved_block + 1 if last_saved_block else None end_block = ( - 10650000 - if start_block is None - else start_block + 500 - if start_block + 500 < max_block_to_cache + 9480000 if start_block is None # NOTE block some arbitrary time after iearn's first deployment + else start_block + BATCH_SIZE if start_block + BATCH_SIZE < max_block_to_cache else max_block_to_cache ) df = pd.DataFrame() - for vault in tqdm(active_vaults_at(end_block)): - df = df.append(get_token_transfers(vault, start_block, end_block)) - df = df.rename(columns={'token': 'vault'}) - df.to_sql('user_txs', postgres.sqla_engine, if_exists='append', index=False) - print(f'user txs batch {start_block}-{end_block} exported to postrges') + for vault in yearn.active_vaults_at(end_block): + df = pd.concat([df, get_token_transfers(vault.vault, start_block, end_block)]) + if len(df): + # NOTE: We want to insert txs in the order they took place, so wallet exporter + # won't have issues in the event that transactions exporter fails mid-run. + df = df.sort_values('block') + for index, row in df.iterrows(): + UserTx( + vault=cache_token(row.token), + timestamp=row.timestamp, + block=row.block, + hash=row.hash, + log_index=row.log_index, + type=row.type, + from_address=row['from'], + to_address=row['to'], + amount = row.amount, + price = row.price, + value_usd = row.value_usd, + gas_used = row.gas_used, + gas_price = row.gas_price + ) + if start_block == end_block: + logger.warn(f'{len(df)} user txs exported to postrges [block {start_block}]') + else: + logger.warn(f'{len(df)} user txs exported to postrges [blocks {start_block}-{end_block}]') # Helper functions @@ -66,33 +75,24 @@ def get_token_transfers(token, start_block, end_block) -> pd.DataFrame: filter_by_name('Transfer', token.abi)[0], web3.codec, ) - postgres.cache_token(token.address) - decimals = contract(token.address).decimals() + token_entity = cache_token(token.address) events = decode_logs( get_logs_asap(token.address, topics, from_block=start_block, to_block=end_block) ) - return pd.DataFrame( - Parallel(1, 'threading')( - delayed(_process_transfer_event)(event, token, decimals) for event in events - ) - ) + return pd.DataFrame([_process_transfer_event(event, token_entity) for event in events]) -def _process_transfer_event(event, vault, decimals) -> dict: +def _process_transfer_event(event, token_entity) -> dict: sender, receiver, amount = event.values() - postgres.cache_address(sender) - postgres.cache_address(receiver) - price = _get_price(event, vault) + cache_address(sender) + cache_address(receiver) + price = _get_price(event, token_entity) if ( - vault.address == '0x7F83935EcFe4729c4Ea592Ab2bC1A32588409797' + # NOTE magic.get_price() returns erroneous price due to erroneous ppfs + token_entity.address.address == '0x7F83935EcFe4729c4Ea592Ab2bC1A32588409797' and event.block_number == 12869164 ): - # NOTE magic.get_price() returns erroneous price due to erroneous ppfs price = 99999 - if price > 100000: - logger.warn(f'token: {vault.address}') - logger.warn(f'price: {price}') - logger.warn(f'block: {event.block_number}') txhash = event.transaction_hash.hex() return { 'chainid': chain.id, @@ -100,42 +100,37 @@ def _process_transfer_event(event, vault, decimals) -> dict: 'timestamp': chain[event.block_number].timestamp, 'hash': txhash, 'log_index': event.log_index, - 'token': vault.address, - 'type': _event_type(sender, receiver, vault.address), + 'token': token_entity.address.address, + 'type': _event_type(sender, receiver, token_entity.address.address), 'from': sender, 'to': receiver, - 'amount': Decimal(amount) / Decimal(10 ** decimals), + 'amount': Decimal(amount) / Decimal(10 ** token_entity.decimals), 'price': price, - 'value_usd': Decimal(amount) / Decimal(10 ** decimals) * Decimal(price), + 'value_usd': Decimal(amount) / Decimal(10 ** token_entity.decimals) * Decimal(price), 'gas_used': web3.eth.getTransactionReceipt(txhash).gasUsed, - 'gas_price': web3.eth.getTransaction( - txhash - ).gasPrice, # * 1.0 # force pandas to insert this as decimal not bigint + 'gas_price': web3.eth.getTransaction(txhash).gasPrice } -def _get_price(event, vault): +def _get_price(event, token_entity): while True: try: try: - return magic.get_price(vault.address, event.block_number) + return magic.get_price(token_entity.address.address, event.block_number) except TypeError: # magic.get_price fails because all liquidity was removed for testing and `share_price` returns None - return magic.get_price(vault.token(), event.block_number) + return magic.get_price(Contract(token_entity.address.address).token(), event.block_number) except ConnectionError as e: # Try again - print(f'ConnectionError: {str(e)}') + logger.warn(f'ConnectionError: {str(e)}') time.sleep(1) except ValueError as e: - print(f'ValueError: {str(e)}') - if str(e) in [ - "Failed to retrieve data from API: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached'}", - "Failed to retrieve data from API: {'status': '0', 'message': 'NOTOK', 'result': 'Max rate limit reached, please use API Key for higher rate limit'}", - ]: + logger.warn(f'ValueError: {str(e)}') + if 'Max rate limit reached' in str(e): # Try again - print('trying again...') + logger.warn('trying again...') time.sleep(5) else: - print(f'vault: {vault.address}') + logger.warn(f'vault: {token_entity.token.address}') raise Exception(str(e)) diff --git a/scripts/treasury_exporter.py b/scripts/treasury_exporter.py index a8395f7d0..fcc2d86a4 100644 --- a/scripts/treasury_exporter.py +++ b/scripts/treasury_exporter.py @@ -3,18 +3,18 @@ import time from brownie import chain -from yearn.treasury.treasury import Treasury -from yearn.outputs import victoria +from yearn.treasury.treasury import YearnTreasury +from yearn.outputs.victoria import output_duration logger = logging.getLogger('yearn.treasury_exporter') sleep_interval = int(os.environ.get('SLEEP_SECONDS', '0')) def main(): - treasury = Treasury(watch_events_forever=True) + treasury = YearnTreasury(watch_events_forever=True) for block in chain.new_blocks(height_buffer=12): start_time = time.time() treasury.export(block.number, block.timestamp) duration = time.time() - start_time - victoria.export_duration(duration, 1, "treasury_forwards", block.timestamp) + output_duration.export(duration, 1, "treasury_forwards", block.timestamp) time.sleep(sleep_interval) diff --git a/scripts/wallet_exporter.py b/scripts/wallet_exporter.py index 44261e6de..37efe1707 100644 --- a/scripts/wallet_exporter.py +++ b/scripts/wallet_exporter.py @@ -3,16 +3,14 @@ from itertools import count from brownie import chain -from yearn.outputs.postgres.postgres import postgres +from yearn.entities import UserTx +from yearn.historical_helper import export_historical, has_data, time_tracking +from yearn.outputs.postgres.utils import last_recorded_block from yearn.utils import closest_block_after_timestamp -from yearn.historical_helper import export_historical, time_tracking, has_data from yearn.yearn import Yearn logger = logging.getLogger('yearn.wallet_exporter') -postgres_cached_thru_block = postgres.last_recorded_block('user_txs') -postgres_cached_thru_ts = chain[postgres_cached_thru_block].timestamp - def main(): start = datetime.now(tz=timezone.utc) # end: 2020-02-12 first iearn deployment @@ -50,6 +48,11 @@ def export_snapshot(yearn, snapshot, ts, exporter_name): def _postgres_ready(ts): + postgres_cached_thru_block = last_recorded_block(UserTx) + postgres_cached_thru_ts = 0 + if postgres_cached_thru_block: + postgres_cached_thru_ts = chain[postgres_cached_thru_block].timestamp + return postgres_cached_thru_ts >= ts diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index 4654c914a..916bdb26c 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -42,6 +42,7 @@ services: - POOL_SIZE - CHUNK_SIZE - RESOLUTION + - SKIP_WALLET_STATS volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -57,6 +58,7 @@ services: environment: - WEB3_PROVIDER=$FTM_WEB3_PROVIDER - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN - SLEEP_SECONDS - NETWORK=ftm-main volumes: @@ -74,6 +76,7 @@ services: environment: - WEB3_PROVIDER=$FTM_WEB3_PROVIDER - EXPLORER=$FTM_EXPLORER + - FTMSCAN_TOKEN - SLEEP_SECONDS - NETWORK=ftm-main - POOL_SIZE @@ -96,10 +99,10 @@ services: - ETHERSCAN_TOKEN - EXPLORER - SLEEP_SECONDS - - POOL_SIZE - - CHUNK_SIZE - - RESOLUTION - - SKIP_WALLET_STATS + - PGHOST=postgres + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=yearn-exporter volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -128,6 +131,8 @@ services: networks: - yearn-exporter restart: on-failure + depends_on: + - postgres historical-treasury-exporter: image: yearn-exporter @@ -140,6 +145,41 @@ services: - POOL_SIZE - CHUNK_SIZE - RESOLUTION + - SKIP_WALLET_STATS + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: on-failure + + sms-exporter: + image: yearn-exporter + command: sms_exporter + environment: + - WEB3_PROVIDER + - ETHERSCAN_TOKEN + - EXPLORER + - SLEEP_SECONDS + volumes: + - solidity_compilers:/root/.solcx + - vyper_compilers:/root/.vvm + - brownie:/root/.brownie + - cache:/app/yearn-exporter/cache + networks: + - yearn-exporter + restart: on-failure + + historical-sms-exporter: + image: yearn-exporter + command: historical_sms_exporter + environment: + - WEB3_PROVIDER + - ETHERSCAN_TOKEN + - EXPLORER + - SLEEP_SECONDS volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -153,9 +193,10 @@ services: image: yearn-exporter command: transactions_exporter environment: - - POSTGRES_PASSWORD=yearn-exporter - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres + - PGHOST=postgres + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=yearn-exporter volumes: - solidity_compilers:/root/.solcx - vyper_compilers:/root/.vvm @@ -246,9 +287,9 @@ services: ports: - 127.0.0.1:5432:5432 environment: - - POSTGRES_PASSWORD=yearn-exporter - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres + - POSTGRES_USER=${PGUSER:-postgres} + - POSTGRES_PASSWORD=${PGPASSWORD:-yearn-exporter} + - POSTGRES_DB=${PGDATABASE:-postgres} networks: - yearn-exporter volumes: diff --git a/yearn/constants.py b/yearn/constants.py index e1f5e4dd2..e6365872d 100644 --- a/yearn/constants.py +++ b/yearn/constants.py @@ -1,4 +1,5 @@ -from brownie import interface +from brownie import interface, chain +from yearn.networks import Network CONTROLLER_INTERFACES = { "0x2be5D998C95DE70D9A38b3d78e49751F10F9E88b": interface.ControllerV1, @@ -118,14 +119,28 @@ CURVE_ADDRESSES_PROVIDER = "0x0000000022D53366457F9d5E68Ec105046FC4383" TREASURY_WALLETS = { - "0x5f0845101857d2A91627478e302357860b1598a1", # Yearn KP3R Wallet - "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", # Yearn Treasury - "0xb99a40fcE04cb740EB79fC04976CA15aF69AaaaE", # Yearn Treasury V1 - "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", # Yearn MultiSig - "0x7d2aB9CA511EBD6F03971Fb417d3492aA82513f0", # ySwap Multisig - "0x2C01B4AD51a67E2d8F02208F54dF9aC4c0B778B6", # yMechs Multisig -} + Network.Mainnet: { + "0x5f0845101857d2A91627478e302357860b1598a1", # Yearn KP3R Wallet + "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", # Yearn Treasury + "0xb99a40fcE04cb740EB79fC04976CA15aF69AaaaE", # Yearn Treasury V1 + "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", # Yearn MultiSig + "0x7d2aB9CA511EBD6F03971Fb417d3492aA82513f0", # ySwap Multisig + "0x2C01B4AD51a67E2d8F02208F54dF9aC4c0B778B6", # yMechs Multisig + }, + Network.Fantom: { + "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", # Yearn Multisig + } +}.get(chain.id,set()) # EVENTS ERC20_TRANSFER_EVENT_HASH = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -ERC677_TRANSFER_EVENT_HASH = '0xe19260aff97b920c7df27010903aeb9c8d2be5d310a2c67824cf3f15396e4c16' \ No newline at end of file +ERC677_TRANSFER_EVENT_HASH = '0xe19260aff97b920c7df27010903aeb9c8d2be5d310a2c67824cf3f15396e4c16' + +STRATEGIST_MULTISIG = { + Network.Mainnet: { + "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", + }, + Network.Fantom: { + "0x72a34AbafAB09b15E7191822A679f28E067C4a16", + } +}.get(chain.id,set()) \ No newline at end of file diff --git a/yearn/entities.py b/yearn/entities.py index 355eda48a..b0d662c52 100644 --- a/yearn/entities.py +++ b/yearn/entities.py @@ -1,5 +1,7 @@ import os from datetime import datetime +from decimal import Decimal + from pony.orm import * db = Database() @@ -7,8 +9,9 @@ class Block(db.Entity): _table_ = "blocks" + block_id = PrimaryKey(int, auto=True) - block = PrimaryKey(int) + block = Required(int, unique=True) timestamp = Required(datetime, sql_type="timestamptz") snapshot = Optional(datetime, sql_type="timestamptz") @@ -17,6 +20,7 @@ class Block(db.Entity): class Snapshot(db.Entity): _table_ = "snapshots" + snapshot_id = PrimaryKey(int, auto=True) block = Required(Block) product = Required(str) @@ -25,12 +29,77 @@ class Snapshot(db.Entity): delegated = Optional(float) # to be filled later +class Address(db.Entity): + _table_ = "addresses" + address_id = PrimaryKey(int, auto=True) + + chainid = Required(int) + address = Required(str) + composite_key(chainid, address) + + is_contract = Required(bool) + nickname = Optional(str) + + token = Optional('Token') + + +class Token(db.Entity): + _table_ = "tokens" + token_id = PrimaryKey(int, auto=True) + + symbol = Required(str) + name = Required(str) + decimals = Required(int) + + user_tx = Set('UserTx', reverse="vault") + partners_tx = Set('PartnerHarvestEvent', reverse="wrapper") + address = Required(Address, column="address_id") + + +class UserTx(db.Entity): + _table_ = "user_txs" + user_tx_id = PrimaryKey(int, auto=True) + + timestamp = Required(int) + block = Required(int) + hash = Required(str) + log_index = Required(int) + composite_key(hash, log_index) + vault = Required(Token, reverse="user_tx", column="token_id") + type = Required(str) + from_address = Required(str, column="from") + to_address = Required(str, column="to") + amount = Required(Decimal,38,18) + price = Required(Decimal,38,18) + value_usd = Required(Decimal,38,18) + gas_used = Required(Decimal,38,1) + gas_price = Required(Decimal,38,1) + + + +class PartnerHarvestEvent(db.Entity): + _table_ = 'partners_txs' + partner_id = PrimaryKey(int, auto=True) + + block = Required(int) + timestamp = Required(int) + balance = Required(Decimal,38,18) + total_supply = Required(Decimal,38,18) + vault_price = Required(Decimal,38,18) + balance_usd = Required(Decimal,38,18) + share = Required(Decimal,38,18) + payout_base = Required(Decimal,38,18) + protocol_fee = Required(Decimal,38,18) + wrapper = Required(Token, reverse='partners_tx') + vault = Required(str) + + db.bind( provider="postgres", user=os.environ.get("PGUSER", "postgres"), host=os.environ.get("PGHOST", "127.0.0.1"), - password=os.environ.get("PGPASS", None), - database="yearn", + password=os.environ.get("PGPASSWORD", ""), + database=os.environ.get("PGDATABASE", "postgres"), ) db.generate_mapping(create_tables=True) diff --git a/yearn/historical_helper.py b/yearn/historical_helper.py index bd60dda96..f7afb0b9a 100644 --- a/yearn/historical_helper.py +++ b/yearn/historical_helper.py @@ -9,7 +9,7 @@ import requests from joblib import Parallel, delayed from toolz import partition_all -from yearn.outputs import victoria +from yearn.outputs.victoria import output_duration from yearn.yearn import Yearn from datetime import timedelta @@ -66,7 +66,7 @@ def wrap(*args, **kwargs): result = export_snapshot_func(*le_args, **kwargs) end = time.time() - victoria.export_duration(end-start, POOL_SIZE, exporter_name, ts) + output_duration.export(end-start, POOL_SIZE, exporter_name, ts) return result return wrap diff --git a/yearn/iearn.py b/yearn/iearn.py index 43f0b8da7..b3c57c3cd 100644 --- a/yearn/iearn.py +++ b/yearn/iearn.py @@ -43,7 +43,7 @@ def __repr__(self): return f"" def describe(self, block=None) -> dict: - vaults = self.active_vaults_at_block(block) + vaults = self.active_vaults_at(block) contracts = [vault.vault for vault in vaults] results = multicall_matrix(contracts, ["totalSupply", "pool", "getPricePerFullShare", "balance"], block=block) output = defaultdict(dict) @@ -67,12 +67,12 @@ def describe(self, block=None) -> dict: return dict(output) def total_value_at(self, block=None): - vaults = self.active_vaults_at_block(block) + vaults = self.active_vaults_at(block) prices = Parallel(8, "threading")(delayed(magic.get_price)(vault.token, block=block) for vault in vaults) results = fetch_multicall(*[[vault.vault, "pool"] for vault in vaults], block=block) return {vault.name: assets * price / vault.scale for vault, assets, price in zip(vaults, results, prices)} - def active_vaults_at_block(self, block=None): + def active_vaults_at(self, block=None): if block is None: return self.vaults return [vault for vault in self.vaults if contract_creation_block(str(vault.vault)) < block] diff --git a/yearn/outputs/describers/registry.py b/yearn/outputs/describers/registry.py index 7faa5861f..500b4496a 100644 --- a/yearn/outputs/describers/registry.py +++ b/yearn/outputs/describers/registry.py @@ -1,19 +1,21 @@ from collections import Counter -from typing import Union from joblib.parallel import Parallel, delayed from yearn.outputs.describers.vault import VaultWalletDescriber -from yearn.v1.registry import Registry as RegistryV1 -from yearn.v2.registry import Registry as RegistryV2 -from collections import Counter +from yearn.utils import contract class RegistryWalletDescriber: - def describe_wallets(self, registry: Union[RegistryV1, RegistryV2] , block=None): - vaults = registry.active_vaults_at(block=block) + def active_vaults_at(self, registry: tuple, block=None): + label, registry = registry + active = [vault for vault in registry.active_vaults_at(block=block) if vault.vault != contract("0xBa37B002AbaFDd8E89a1995dA52740bbC013D992")] # [yGov] Doesn't count for this context + return active + + def describe_wallets(self, registry: tuple, block=None): describer = VaultWalletDescriber() - data = Parallel(8,'threading')(delayed(describer.describe_wallets)(vault, block=block) for vault in vaults) - data = {vault.name: desc for vault,desc in zip(vaults,data)} + active_vaults = self.active_vaults_at(registry, block=block) + data = Parallel(8,'threading')(delayed(describer.describe_wallets)(vault.vault.address, block=block) for vault in active_vaults) + data = {vault.name: desc for vault,desc in zip(active_vaults,data)} wallet_balances = Counter() for vault, desc in data.items(): @@ -27,4 +29,4 @@ def describe_wallets(self, registry: Union[RegistryV1, RegistryV2] , block=None) "wallet balances usd": wallet_balances, } data.update(agg_stats) - return data \ No newline at end of file + return data diff --git a/yearn/outputs/describers/vault.py b/yearn/outputs/describers/vault.py index ac3c099fd..21c33c223 100644 --- a/yearn/outputs/describers/vault.py +++ b/yearn/outputs/describers/vault.py @@ -1,32 +1,45 @@ -from typing import Union -from yearn.outputs.postgres.postgres import PostgresInstance +import logging +from brownie import chain +from yearn.networks import Network + +from yearn.outputs.postgres.utils import fetch_balances from yearn.prices import magic -from yearn.v1.vaults import VaultV1 -from yearn.v2.vaults import Vault as VaultV2 +from yearn.exceptions import PriceError + +logger = logging.getLogger(__name__) class VaultWalletDescriber: def wallets(self, vault_address, block=None): return self.wallet_balances(vault_address, block=block).keys() def wallet_balances(self, vault_address, block=None): - return PostgresInstance().fetch_balances(vault_address, block=block) + return fetch_balances(vault_address, block=block) - def describe_wallets(self, vault: Union[VaultV1, VaultV2], block=None): - balances = self.wallet_balances(vault.vault.address, block=block) + def describe_wallets(self, vault_address, block=None): + balances = self.wallet_balances(vault_address, block=block) info = { 'total wallets': len(set(wallet for wallet, bal in balances.items())), - 'active wallets': sum(1 if balance > 50 else 0 for wallet, balance in balances.items()), 'wallet balances': { wallet: { "token balance": float(bal), - "usd balance": float(bal) * self.get_price(vault, block=block) + "usd balance": float(bal) * _get_price(vault_address, block=block) } for wallet, bal in balances.items() } - } + } + info['active wallets'] = sum(1 if balances['usd balance'] > 50 else 0 for wallet, balances in info['wallet balances'].items()) return info - def get_price(self, vault: Union[VaultV1, VaultV2], block=None): - if isinstance(vault, VaultV1) and vault.name == "aLINK": - return magic.get_price(self.vault.underlying(), block=block) - else: - return magic.get_price(self.token, block=block) \ No newline at end of file + +def _get_price(token_address, block=None): + try: + return magic.get_price(token_address, block=block) + except PriceError as e: + if chain.id == Network.Mainnet and block: + # crvAAVE vault state was broken due to an incident, return a post-fix price + # https://github.com/yearn/yearn-security/blob/master/disclosures/2021-05-13.md + if token_address == '0x03403154afc09Ce8e44C3B185C82C6aD5f86b9ab' and 12430455 <= block <= 12430661: + return 1.091553 + + logger.exception(e) + + return 0 diff --git a/yearn/outputs/postgres/postgres.py b/yearn/outputs/postgres/postgres.py deleted file mode 100644 index 9feea48fe..000000000 --- a/yearn/outputs/postgres/postgres.py +++ /dev/null @@ -1,130 +0,0 @@ -import os - -import pyodbc -from brownie import ZERO_ADDRESS, Contract, chain -from sqlalchemy import create_engine -from yearn.multicall2 import fetch_multicall -from yearn.outputs.postgres.tables import (addresses_table_query, - tokens_table_query, - treasury_table_query, - users_table_query) -from yearn.utils import is_contract, contract - -DATABASE = os.environ.get('POSTGRES_DB',os.environ.get('POSTGRES_USER','postgres')) -UID = os.environ.get('POSTGRES_USER','postgres') -PWD = os.environ.get('POSTGRES_PASSWORD','yearn-exporter') - -POSTRGES_CONN_STRING = "Driver={PostgreSQL Unicode};Server=postgres;Port:5432;"\ - f"Database={DATABASE};Uid={UID};Pwd={PWD};" - -SQLA_CONN_STRING = f"postgresql://{UID}:{PWD}@postgres/{DATABASE}?port=5432" - -chainid = chain.id - -class PostgresInstance: - def __init__(self): - self.conn = pyodbc.connect(POSTRGES_CONN_STRING) - self.cursor = self.conn.cursor() - self.sqla_engine = create_engine(SQLA_CONN_STRING) - if not self.table_exists('addresses'): - self.create_table(addresses_table_query) - print('addresses table created successfully...') - if not self.table_exists('tokens'): - self.create_table(tokens_table_query) - print('tokens table created successfully...') - if not self.table_exists('treasury_txs'): - self.create_table(treasury_table_query) - print('treasury_txs table created successfully...') - if not self.table_exists('user_txs'): - self.create_table(users_table_query) - print('user_txs table created successfully...') - - - # Table functions - - def table_exists(self, table_name): - response = self.cursor.execute(f"select exists(select tablename from pg_tables where tablename = '{table_name}')").fetchone()[0] - return True if response == '1' else False - - def create_table(self, table_create_query): - self.cursor.execute(table_create_query) - self.conn.commit() - - def last_recorded_block(self, table_name): - ''' - This will only work for tables with a `block` column - ''' - response = self.cursor.execute(f'SELECT max(block) from {table_name} where chainid = {chainid}').fetchone()[0] - self.conn.commit() - return response - - # Other functions - - def token_exists(self, token_address: str): - response = self.cursor.execute(f"select exists(select symbol from tokens where token_address = '{token_address}' and chainid = {chainid})").fetchone()[0] - return True if response == '1' else False - - def cache_token(self,token_address: str): - i = 0 - while i < 10: - try: - self.cache_address(token_address) - if not self.token_exists(token_address): - token = contract(token_address) - symbol, name, decimals = fetch_multicall([token,'symbol'],[token,'name'],[token,'decimals']) - self.cursor.execute(f"INSERT INTO TOKENS (CHAINID, TOKEN_ADDRESS, SYMBOL, NAME, DECIMALS)\ - VALUES ({chainid},'{token_address}','{symbol}','{name}',{decimals})") - self.conn.commit() - print(f'{symbol} added to postgres') - return - except: - i += 1 - - def address_exists(self, address: str): - response = self.cursor.execute(f"select exists(select address from addresses where address = '{address}' and chainid = {chainid})").fetchone()[0] - return True if response == '1' else False - - def cache_address(self,address: str): - i = 0 - while i < 10: - try: - if not self.address_exists(address): - self.cursor.execute(f"INSERT INTO ADDRESSES (CHAINID, ADDRESS, IS_CONTRACT)\ - VALUES ({chainid},'{address}',{is_contract(address)})") - self.conn.commit() - return - except: - i += 1 - - def fetch_balances(self,vault_address,block): - if block and block > self.last_recorded_block('user_txs'): - # NOTE: we use `postgres.` instead of `self.` so we can make use of parallelism - raise('this block has not yet been cached into postgres') - if block: - balances = self.cursor.execute(f""" - select a.wallet, coalesce(amount_in,0) - coalesce(amount_out,0) balance - from ( - select "to" wallet, sum(amount) amount_in - from user_txs where chainid = {chainid} and vault = '{vault_address}' and block <= {block} - group by "to" ) a - left join ( - select "from" wallet, sum(amount) amount_out - from user_txs where chainid = {chainid} and vault = '{vault_address}' and block <= {block} - group by "from") b on a.wallet = b.wallet - """).fetchall() - else: - balances = self.cursor.execute(f""" - select a.wallet, coalesce(amount_in,0) - coalesce(amount_out,0) balance - from ( - select "to" wallet, sum(amount) amount_in - from user_txs where chainid = {chainid} and vault = '{vault_address}' - group by "to" ) a - left join ( - select "from" wallet, sum(amount) amount_out - from user_txs where chainid = {chainid} and vault = '{vault_address}' - group by "from") b on a.wallet = b.wallet - """).fetchall() - self.conn.commit() - return {wallet: balance for wallet,balance in balances if wallet != ZERO_ADDRESS} - -postgres = PostgresInstance() diff --git a/yearn/outputs/postgres/tables.py b/yearn/outputs/postgres/tables.py index 1631896f7..ac77e2b23 100644 --- a/yearn/outputs/postgres/tables.py +++ b/yearn/outputs/postgres/tables.py @@ -1,3 +1,4 @@ +# NOTE: We don't use this anymore but keeping for reference until I implement treasury txs in postgres treasury_table_query = f'CREATE TABLE TREASURY_TXS(\ CHAINID SMALLINT NOT NULL,\ diff --git a/yearn/outputs/postgres/utils.py b/yearn/outputs/postgres/utils.py new file mode 100644 index 000000000..935675cb5 --- /dev/null +++ b/yearn/outputs/postgres/utils.py @@ -0,0 +1,70 @@ +from brownie import Contract, chain, ZERO_ADDRESS +from pony.orm import ObjectNotFound, db_session, TransactionIntegrityError, select +from yearn.cache import memory +import logging +from yearn.entities import Address, Token, UserTx, db +from yearn.multicall2 import fetch_multicall +from yearn.utils import is_contract + +@db_session +@memory.cache() +def cache_address(address: str) -> Address: + address_entity = Address.get(chainid=chain.id, address=address) + if not address_entity: + address_entity = Address(chainid=chain.id, address=address, is_contract=is_contract(address)) + return address_entity + +@db_session +@memory.cache() +def cache_token(address: str) -> Token: + address_entity = cache_address(address) + token = Token.get(address=address_entity) + if not token: + contract = Contract(address) + symbol, name, decimals = fetch_multicall([contract,'symbol'],[contract,'name'],[contract,'decimals']) + token = Token(address=address_entity, symbol=symbol, name=name, decimals=decimals) + print(f'token {symbol} added to postgres') + return token + +@db_session +def last_recorded_block(Entity: db.Entity) -> int: + ''' + Returns last block recorded for sql entity type `Entity` + ''' + TreasuryTx = 1 # NOTE: Get rid of this when treasury txs are implemented + if Entity == UserTx: + return select(max(e.block) for e in Entity if e.vault.address.chainid == chain.id).first() + elif Entity == TreasuryTx: + return select(max(e.block) for e in Entity if e.token.address.chainid == chain.id).first() + return select(max(e.block) for e in Entity if e.chainid == chain.id).first() + +@db_session +def fetch_balances(vault_address: str, block=None): + if block and block > last_recorded_block(UserTx): + # NOTE: we use `postgres.` instead of `self.` so we can make use of parallelism + raise Exception('this block has not yet been cached into postgres') + if block: + balances = db.select(f""" + a.wallet, coalesce(amount_in,0) - coalesce(amount_out,0) balance + from ( + select "to" wallet, sum(amount) amount_in + from user_txs where chainid = $(chain.id) and vault = $vault_address and block <= $block + group by "to" ) a + left join ( + select "from" wallet, sum(amount) amount_out + from user_txs where chainid = $(chain.id) and vault = $vault_address and block <= $block + group by "from") b on a.wallet = b.wallet + """) + else: + balances = db.select(f""" + a.wallet, coalesce(amount_in,0) - coalesce(amount_out,0) balance + from ( + select "to" wallet, sum(amount) amount_in + from user_txs where chainid = $(chain.id) and vault = $vault_address + group by "to" ) a + left join ( + select "from" wallet, sum(amount) amount_out + from user_txs where chainid = $(chain.id) and vault = $vault_address + group by "from") b on a.wallet = b.wallet + """) + return {wallet: balance for wallet,balance in balances if wallet != ZERO_ADDRESS} \ No newline at end of file diff --git a/yearn/outputs/victoria.py b/yearn/outputs/victoria.py deleted file mode 100644 index 025bfe28c..000000000 --- a/yearn/outputs/victoria.py +++ /dev/null @@ -1,259 +0,0 @@ -import requests -import os -import gzip -import math -import json -from typing import List, Dict -from brownie import Contract, chain -from yearn.treasury.buckets import get_token_bucket -from yearn.utils import contract -from yearn.networks import Network -from yearn.prices import constants - -mapping = { - "earn": { - "metric": "iearn", - "labels": ["vault", "param", "address", "version"], - "agg_stats": ["total wallets"] - }, - "ib": { - "metric": "ironbank", - "labels": ["vault", "param", "address", "version"], - "agg_stats": ["total wallets"] - }, - "v1": { - "metric": "yearn", - "labels": ["vault", "param", "address", "version"], - "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] - }, - "v2": { - "metric": "yearn_vault", - "labels": ["vault", "param", "address", "version", "experimental"], - "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] - }, - "v2_strategy": { - "metric": "yearn_strategy", - "labels": ["vault", "strategy", "param", "address", "version", "experimental"], - }, - "special": { - "metric": "yearn_vault", - "labels": ["vault", "param", "address", "version", "experimental"], - "agg_stats": ["total wallets"] - } -} - - -def export(block, timestamp, data): - metrics_to_export = [] - - if Network(chain.id) == Network.Fantom: - simple_products = ["ib"] - else: - simple_products = ["v1", "earn", "ib", "special"] - - for product in simple_products: - metric = mapping[product]["metric"] - for vault, params in data[product].items(): - - for key, value in params.items(): - if key in ["address", "version", "experimental"] or value is None: - continue - - has_experiments = product == "special" - - label_values = _get_label_values(params, [vault, key], has_experiments) - label_names = mapping[product]["labels"] - - if product == "ib" and key == 'tvl' and block >= constants.ib_snapshot_block: - # create one item with tvl=0 that will be used in existing dashboards - item_legacy = _build_item(metric, label_names, label_values, 0, timestamp) - metrics_to_export.append(item_legacy) - # create a second item to track ib tvl separately - item_own = _build_item(f'{metric}_own', label_names, label_values, value, timestamp) - metrics_to_export.append(item_own) - else: - item = _build_item(metric, label_names, label_values, value, timestamp) - metrics_to_export.append(item) - - - for vault, params in data["v2"].items(): - metric = mapping["v2"]["metric"] - for key, value in params.items(): - if key in ["address", "version", "experimental", "strategies"] or value is None: - continue - - label_values = _get_label_values(params, [vault, key], True) - label_names = mapping["v2"]["labels"] - - item = _build_item(metric, label_names, label_values, value, timestamp) - metrics_to_export.append(item) - - # strategies can have nested structs - metric = mapping["v2_strategy"]["metric"] - for strategy, strategy_params in data["v2"][vault]["strategies"].items(): - flat = flatten_dict(strategy_params) - for key, value in flat.items(): - if key in ["address", "version", "experimental"] or value is None: - continue - - label_values = _get_label_values(params, [vault, strategy, key], True) - label_names = mapping["v2_strategy"]["labels"] - - item = _build_item(metric, label_names, label_values, value or 0, timestamp) - metrics_to_export.append(item) - - # post all metrics for this timestamp at once - _post(metrics_to_export) - - -def export_wallets(timestamp, data): - metrics_to_export = [] - for key, value in data['agg_stats'].items(): - if key == 'wallet balances usd': - for wallet, usd_bal in value.items(): - label_names = ["param","wallet"] - label_values = ["balance usd",wallet] - item = _build_item("aggregate", label_names, label_values, usd_bal, timestamp) - metrics_to_export.append(item) - continue - label_names = ['param'] - label_values = [key] - item = _build_item("aggregate", label_names, label_values, value, timestamp) - metrics_to_export.append(item) - for product in ['v1','v2']: - metric = mapping[product]["metric"] - for key, value in data[product].items(): - if key in mapping[product]["agg_stats"]: - label_names = ['param'] - label_values = [key] - item = _build_item(metric, label_names, label_values, value, timestamp) - metrics_to_export.append(item) - continue - elif key == "wallet balances usd": - for wallet, usd_bal in value.items(): - label_names = ["param","wallet"] - label_values = ["balance usd",wallet] - item = _build_item(metric, label_names, label_values, usd_bal, timestamp) - metrics_to_export.append(item) - continue - - vault, params = key, value - for k, v in params.items(): - if k == 'wallet balances': - for wallet, bals in v.items(): - for denom, bal in bals.items(): - label_values = [wallet] + _get_label_values(params, [vault, denom], product in ['v2','special']) - label_names = ["wallet"] + mapping[product]["labels"] - item = _build_item(metric, label_names, label_values, bal, timestamp) - metrics_to_export.append(item) - continue - - label_values = _get_label_values(params, [vault, k], True) - label_names = mapping[product]["labels"] - - item = _build_item(metric, label_names, label_values, v, timestamp) - metrics_to_export.append(item) - - # post all wallet metrics for this timestamp at once - _post(metrics_to_export) - -def export_treasury(timestamp, data): - metrics_to_export = [] - for section, section_data in data.items(): - for wallet, wallet_data in section_data.items(): - for token, bals in wallet_data.items(): - symbol = 'ETH' if token == 'ETH' else contract(token).symbol() - for key, value in bals.items(): - if value != 0: - label_names = ['param','wallet','token_address','token','bucket'] - label_values = [key,wallet,token,symbol,get_token_bucket(token)] - item = _build_item(f"treasury_{section}",label_names,label_values,value,timestamp) - metrics_to_export.append(item) - - # post all metrics for this timestamp at once - _post(metrics_to_export) - - -def export_duration(duration_seconds, pool_size, direction, timestamp_seconds): - item = _build_item( - "export_duration", - [ "pool_size", "direction" ], - [ pool_size, direction ], - duration_seconds, - timestamp_seconds - ) - _post([item]) - - -def _build_item(metric, label_names, label_values, value, timestamp): - ts_millis = math.floor(timestamp) * 1000 - label_names.append("network") - label_values.append(Network.label(chain.id)) - meta = dict(zip(map(_sanitize, label_names), map(str, label_values))) - meta["__name__"] = metric - return {"metric": meta, "values": [_sanitize(value)], "timestamps": [ts_millis]} - - -def _to_jsonl_gz(metrics_to_export: List[Dict]): - lines = [] - for item in metrics_to_export: - lines.append(json.dumps(item)) - - jsonlines = "\n".join(lines) - return gzip.compress(bytes(jsonlines, "utf-8")) - - -def _post(metrics_to_export: List[Dict]): - data = _to_jsonl_gz(metrics_to_export) - base_url = os.environ.get('VM_URL', 'http://victoria-metrics:8428') - url = f'{base_url}/api/v1/import' - headers = { - 'Connection': 'close', - 'Content-Encoding': 'gzip' - } - with requests.Session() as session: - session.post( - url = url, - data = data, - headers = headers - ) - - -def _sanitize(value): - if isinstance(value, bool): - return int(value) - elif isinstance(value, str): - return value.replace('"', '') # e.g. '"yvrenBTC" 0.3.5 0x340832' - - return value - - -def flatten_dict(d): - def items(): - for key, value in d.items(): - if isinstance(value, dict): - for subkey, subvalue in flatten_dict(value).items(): - yield key + "." + subkey, subvalue - else: - yield key, value - - return dict(items()) - - -def _get_label_values(params, inital_labels, experimental = False): - address = _get_string_label(params, "address") - version = _get_string_label(params, "version") - label_values = inital_labels + [address, version] - if experimental: - experimental_label = _get_bool_label(params, "experimental") - label_values.append(experimental_label) - - return label_values - - -def _get_bool_label(a_dict, key): - return "true" if key in a_dict and a_dict[key] == True else "false" - - -def _get_string_label(a_dict, key): - return str(a_dict[key]) if key in a_dict else "n/a" diff --git a/yearn/outputs/victoria/output_base.py b/yearn/outputs/victoria/output_base.py new file mode 100644 index 000000000..e55c601a0 --- /dev/null +++ b/yearn/outputs/victoria/output_base.py @@ -0,0 +1,65 @@ +from yearn.outputs.victoria.output_helper import mapping, _get_label_values, _build_item, _post, _flatten_dict +from brownie import chain +from yearn.prices import constants +from yearn.networks import Network + +def export(block, timestamp, data): + metrics_to_export = [] + + if Network(chain.id) == Network.Fantom: + simple_products = ["ib"] + else: + simple_products = ["v1", "earn", "ib", "special"] + + for product in simple_products: + metric = mapping[product]["metric"] + for vault, params in data[product].items(): + + for key, value in params.items(): + if key in ["address", "version", "experimental"] or value is None: + continue + + has_experiments = product == "special" + + label_values = _get_label_values(params, [vault, key], has_experiments) + label_names = mapping[product]["labels"] + + if product == "ib" and key == 'tvl' and block >= constants.ib_snapshot_block: + # create one item with tvl=0 that will be used in existing dashboards + item_legacy = _build_item(metric, label_names, label_values, 0, timestamp) + metrics_to_export.append(item_legacy) + # create a second item to track ib tvl separately + item_own = _build_item(f'{metric}_own', label_names, label_values, value, timestamp) + metrics_to_export.append(item_own) + else: + item = _build_item(metric, label_names, label_values, value, timestamp) + metrics_to_export.append(item) + + for vault, params in data["v2"].items(): + metric = mapping["v2"]["metric"] + for key, value in params.items(): + if key in ["address", "version", "experimental", "strategies"] or value is None: + continue + + label_values = _get_label_values(params, [vault, key], True) + label_names = mapping["v2"]["labels"] + + item = _build_item(metric, label_names, label_values, value, timestamp) + metrics_to_export.append(item) + + # strategies can have nested structs + metric = mapping["v2_strategy"]["metric"] + for strategy, strategy_params in data["v2"][vault]["strategies"].items(): + flat = _flatten_dict(strategy_params) + for key, value in flat.items(): + if key in ["address", "version", "experimental"] or value is None: + continue + + label_values = _get_label_values(params, [vault, strategy, key], True) + label_names = mapping["v2_strategy"]["labels"] + + item = _build_item(metric, label_names, label_values, value or 0, timestamp) + metrics_to_export.append(item) + + # post all metrics for this timestamp at once + _post(metrics_to_export) diff --git a/yearn/outputs/victoria/output_duration.py b/yearn/outputs/victoria/output_duration.py new file mode 100644 index 000000000..3b0ad9d09 --- /dev/null +++ b/yearn/outputs/victoria/output_duration.py @@ -0,0 +1,11 @@ +from yearn.outputs.victoria.output_helper import _build_item, _post + +def export(duration_seconds, pool_size, direction, timestamp_seconds): + item = _build_item( + "export_duration", + [ "pool_size", "direction" ], + [ pool_size, direction ], + duration_seconds, + timestamp_seconds + ) + _post([item]) diff --git a/yearn/outputs/victoria/output_helper.py b/yearn/outputs/victoria/output_helper.py new file mode 100644 index 000000000..aac1af91d --- /dev/null +++ b/yearn/outputs/victoria/output_helper.py @@ -0,0 +1,114 @@ +import requests +import os +import gzip +import math +import json +from typing import List, Dict +from brownie import chain +from yearn.utils import contract +from yearn.networks import Network + +mapping = { + "earn": { + "metric": "iearn", + "labels": ["vault", "param", "address", "version"], + "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] + }, + "ib": { + "metric": "ironbank", + "labels": ["vault", "param", "address", "version"], + "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] + }, + "v1": { + "metric": "yearn", + "labels": ["vault", "param", "address", "version"], + "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] + }, + "v2": { + "metric": "yearn_vault", + "labels": ["vault", "param", "address", "version", "experimental"], + "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] + }, + "v2_strategy": { + "metric": "yearn_strategy", + "labels": ["vault", "strategy", "param", "address", "version", "experimental"], + }, + "special": { + "metric": "yearn_vault", + "labels": ["vault", "param", "address", "version", "experimental"], + "agg_stats": ["total wallets","active wallets","wallets > $5k","wallets > $50k"] + } +} + +def _build_item(metric, label_names, label_values, value, timestamp): + ts_millis = math.floor(timestamp) * 1000 + label_names.append("network") + label_values.append(Network.label(chain.id)) + meta = dict(zip(map(_sanitize, label_names), map(str, label_values))) + meta["__name__"] = metric + return {"metric": meta, "values": [_sanitize(value)], "timestamps": [ts_millis]} + + +def _to_jsonl_gz(metrics_to_export: List[Dict]): + lines = [] + for item in metrics_to_export: + lines.append(json.dumps(item)) + + jsonlines = "\n".join(lines) + return gzip.compress(bytes(jsonlines, "utf-8")) + + +def _post(metrics_to_export: List[Dict]): + data = _to_jsonl_gz(metrics_to_export) + base_url = os.environ.get('VM_URL', 'http://victoria-metrics:8428') + url = f'{base_url}/api/v1/import' + headers = { + 'Connection': 'close', + 'Content-Encoding': 'gzip' + } + with requests.Session() as session: + session.post( + url = url, + data = data, + headers = headers + ) + + +def _sanitize(value): + if isinstance(value, bool): + return int(value) + elif isinstance(value, str): + return value.replace('"', '') # e.g. '"yvrenBTC" 0.3.5 0x340832' + + return value + + +def _flatten_dict(d): + def items(): + for key, value in d.items(): + if isinstance(value, dict): + for subkey, subvalue in _flatten_dict(value).items(): + yield key + "." + subkey, subvalue + else: + yield key, value + + return dict(items()) + + +def _get_label_values(params, inital_labels, experimental = False): + address = _get_string_label(params, "address") + version = _get_string_label(params, "version") + label_values = inital_labels + [address, version] + if experimental: + experimental_label = _get_bool_label(params, "experimental") + label_values.append(experimental_label) + + return label_values + + +def _get_bool_label(a_dict, key): + return "true" if key in a_dict and a_dict[key] == True else "false" + + +def _get_string_label(a_dict, key): + return str(a_dict[key]) if key in a_dict else "n/a" diff --git a/yearn/outputs/victoria/output_treasury.py b/yearn/outputs/victoria/output_treasury.py new file mode 100644 index 000000000..e31dfd535 --- /dev/null +++ b/yearn/outputs/victoria/output_treasury.py @@ -0,0 +1,18 @@ +from yearn.outputs.victoria.output_helper import _build_item, _post +from yearn.treasury.buckets import get_token_bucket +from yearn.utils import contract + +def export(timestamp, data, label): + metrics_to_export = [] + for section, section_data in data.items(): + for wallet, wallet_data in section_data.items(): + for token, bals in wallet_data.items(): + symbol = 'ETH' if token == 'ETH' else contract(token).symbol() + for key, value in bals.items(): + label_names = ['param','wallet','token_address','token','bucket'] + label_values = [key,wallet,token,symbol,get_token_bucket(token)] + item = _build_item(f"{label}_{section}",label_names,label_values,value,timestamp) + metrics_to_export.append(item) + + # post all metrics for this timestamp at once + _post(metrics_to_export) diff --git a/yearn/outputs/victoria/output_wallets.py b/yearn/outputs/victoria/output_wallets.py new file mode 100644 index 000000000..cada3d426 --- /dev/null +++ b/yearn/outputs/victoria/output_wallets.py @@ -0,0 +1,55 @@ +from yearn.outputs.victoria.output_helper import mapping, _get_label_values, _build_item, _post + +def export(timestamp, data): + metrics_to_export = [] + for key, value in data['agg_stats'].items(): + if key == 'wallet balances usd': + for wallet, usd_bal in value.items(): + label_names = ["param","wallet"] + label_values = ["balance usd",wallet] + item = _build_item("aggregate", label_names, label_values, usd_bal, timestamp) + metrics_to_export.append(item) + continue + label_names = ['param'] + label_values = [key] + item = _build_item("aggregate", label_names, label_values, value, timestamp) + metrics_to_export.append(item) + for key in data.keys(): + if key == 'agg_stats': + continue + product = key + metric = mapping[product]["metric"] + for key, value in data[product].items(): + if key in mapping[product]["agg_stats"]: + label_names = ['param'] + label_values = [key] + item = _build_item(metric, label_names, label_values, value, timestamp) + metrics_to_export.append(item) + continue + elif key == "wallet balances usd": + for wallet, usd_bal in value.items(): + label_names = ["param","wallet"] + label_values = ["balance usd",wallet] + item = _build_item(metric, label_names, label_values, usd_bal, timestamp) + metrics_to_export.append(item) + continue + + vault, params = key, value + for k, v in params.items(): + if k == 'wallet balances': + for wallet, bals in v.items(): + for denom, bal in bals.items(): + label_values = [wallet] + _get_label_values(params, [vault, denom], product in ['v2','special']) + label_names = ["wallet"] + mapping[product]["labels"] + item = _build_item(metric, label_names, label_values, bal, timestamp) + metrics_to_export.append(item) + continue + + label_values = _get_label_values(params, [vault, k], True) + label_names = mapping[product]["labels"] + + item = _build_item(metric, label_names, label_values, v, timestamp) + metrics_to_export.append(item) + + # post all wallet metrics for this timestamp at once + _post(metrics_to_export) diff --git a/yearn/prices/band.py b/yearn/prices/band.py index 033c89780..92e31350d 100644 --- a/yearn/prices/band.py +++ b/yearn/prices/band.py @@ -13,6 +13,30 @@ Network.Fantom: '0x56E2898E0ceFF0D1222827759B56B28Ad812f92F' } +supported_assets = { + # https://docs.fantom.foundation/tutorials/band-protocol-standard-dataset#supported-tokens + Network.Fantom: [ + "0xaf319E5789945197e365E7f7fbFc56B130523B33", # FRAX + "0x04068DA6C83AFCFA0e13ba15A6696662335D5B75", # USDC + "0x0E1694483eBB3b74d3054E383840C6cf011e518e", # sUSD + "0x6a07A792ab2965C72a5B8088d3a069A7aC3a993B", # AAVE + "0x46E7628E8b4350b2716ab470eE0bA1fa9e76c6C5", # BAND + "0xD67de0e0a0Fd7b15dC8348Bb9BE742F3c5850454", # BNB + "0x321162Cd933E2Be498Cd2267a90534A804051b11", # BTC + "0xB01E8419d842beebf1b70A7b5f7142abbaf7159D", # COVER + "0x657A1861c15A3deD9AF0B6799a195a249ebdCbc6", # CREAM + "0x1E4F97b9f9F913c46F1632781732927B9019C68b", # CRV + "0x74b23882a30290451A17c44f4F05243b6b58C76d", # ETH + "0x44B26E839eB3572c5E959F994804A5De66600349", # HEGIC + "0x2A5062D22adCFaAfbd5C541d4dA82E4B450d4212", # KP3R + "0xb3654dc3D10Ea7645f8319668E8F54d2574FBdC8", # LINK + "0x924828a9Fb17d47D0eb64b57271D10706699Ff11", # SFI + "0x56ee926bD8c72B2d5fa1aF4d9E4Cbb515a1E3Adc", # SNX + "0xae75A438b2E0cB8Bb01Ec1E1e376De11D44477CC", # SUSHI + "0x29b0Da86e484E1C0029B56e817912d778aC0EC69", # YFI + ] +} + class Band(metaclass=Singleton): def __init__(self): if chain.id not in addresses: @@ -20,7 +44,7 @@ def __init__(self): self.oracle = contract(addresses[chain.id]) def __contains__(self, asset): - return chain.id in addresses + return chain.id in addresses and asset in supported_assets[chain.id] @ttl_cache(maxsize=None, ttl=600) def get_price(self, asset, block=None): diff --git a/yearn/prices/constants.py b/yearn/prices/constants.py index 9d6341a81..4ca82045e 100644 --- a/yearn/prices/constants.py +++ b/yearn/prices/constants.py @@ -36,11 +36,21 @@ "0x1c48f86ae57291F7686349F12601910BD8D470bb": "usdk", "0x056Fd409E1d7A124BD7017459dFEa2F387b6d5Cd": "gusd", "0x0E2EC54fC0B509F445631Bf4b91AB8168230C752": "linkusd", + "0x99D8a9C45b2ecA8864373A26D1459e3Dff1e17F3": "mim", + "0xa47c8bf37f92aBed4A126BDA807A7b7498661acD": "ust", + "0x196f4727526eA7FB1e17b2071B3d8eAA38486988": "rsv", + "0xdF574c24545E5FfEcb9a659c229253D4111d87e1": "husd", + "0x5BC25f649fc4e26069dDF4cF4010F9f706c23831": "dusd", + "0xe2f2a5C287993345a840Db3B0845fbC70f5935a5": "musd", + "0x739ca6D71365a08f584c8FC4e1029045Fa8ABC4B": "anydai", + "0xbbc4A8d076F4B1888fec42581B6fc58d242CF2D5": "anymin", }, Network.Fantom: { "0x04068DA6C83AFCFA0e13ba15A6696662335D5B75": "usdc", "0x8D11eC38a3EB5E956B052f67Da8Bdc9bef8Abf3E": "dai", - "0xe2D27f06F63d98b8e11b38b5b08A75D0c8dD62B9": "ust" + "0xe2D27f06F63d98b8e11b38b5b08A75D0c8dD62B9": "ust", + "0x82f0B8B456c1A451378467398982d4834b6829c1": "mim", + "0x049d68029688eAbF473097a2fC38ef61633A3C7A": "fusdt" }, Network.Arbitrum: { '0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8': 'usdc', diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index 0aa42057e..e9478554e 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -14,10 +14,11 @@ from yearn.prices.uniswap.v1 import uniswap_v1 from yearn.prices.uniswap.v2 import uniswap_v2 from yearn.prices.uniswap.v3 import uniswap_v3 +from yearn.prices.curve import curve from yearn.prices.yearn import yearn_lens from yearn.utils import contract -from yearn.prices import constants, curve +from yearn.prices import constants logger = logging.getLogger(__name__) @@ -26,7 +27,6 @@ def get_price(token, block=None): token = unwrap_token(token) return find_price(token, block) - def unwrap_token(token): token = str(token) logger.debug("unwrapping %s", token) @@ -49,7 +49,6 @@ def unwrap_token(token): def find_price(token, block): price = None - if token in constants.stablecoins: logger.debug("stablecoin -> %s", 1) return 1 @@ -71,10 +70,14 @@ def find_price(token, block): logger.debug('xcredit -> unwrap') wrapper = contract(token) price = get_price(wrapper.token(), block=block) * wrapper.getShareValue() / 1e18 + # no liquid market for yveCRV-DAO -> return CRV token price + elif chain.id == Network.Mainnet and token == '0xc5bDdf9843308380375a611c18B50Fb9341f502A' and block and block < 11786563: + if curve and curve.crv: + return get_price(curve.crv, block=block) markets = [ chainlink, - curve.curve, + curve, compound, fixed_forex, synthetix, diff --git a/yearn/prices/yearn.py b/yearn/prices/yearn.py index fa978e8df..8674caeb0 100644 --- a/yearn/prices/yearn.py +++ b/yearn/prices/yearn.py @@ -70,12 +70,14 @@ def get_price(self, token, block=None): [vault, 'decimals'], block=block, ) - return [share_price / 10 ** decimals, underlying] + if share_price and underlying and decimals: + return [share_price / 10 ** decimals, underlying] if hasattr(vault, 'getPricePerFullShare'): share_price, underlying = fetch_multicall( [vault, 'getPricePerFullShare'], [vault, 'token'], block=block ) - return [share_price / 1e18, underlying] + if share_price and underlying: + return [share_price / 1e18, underlying] yearn_lens = None diff --git a/yearn/treasury/treasury.py b/yearn/treasury/treasury.py index b6e31f5de..c0d6fa336 100644 --- a/yearn/treasury/treasury.py +++ b/yearn/treasury/treasury.py @@ -2,23 +2,26 @@ import threading import time -from brownie import Contract, chain, web3 +from brownie import ZERO_ADDRESS, Contract, chain, web3 from brownie.network.event import EventLookupError from eth_abi import encode_single from eth_utils import encode_hex from joblib import Parallel, delayed +from yearn.constants import (ERC20_TRANSFER_EVENT_HASH, + ERC677_TRANSFER_EVENT_HASH, STRATEGIST_MULTISIG, + TREASURY_WALLETS) from yearn.events import decode_logs +from yearn.exceptions import PriceError from yearn.multicall2 import fetch_multicall -from yearn.outputs import victoria +from yearn.outputs.victoria import output_treasury from yearn.partners.partners import partners from yearn.partners.snapshot import WildcardWrapper, Wrapper +from yearn.prices import compound from yearn.prices.constants import weth -from yearn.prices.magic import get_price, logger as logger_price_magic -from yearn.exceptions import PriceError +from yearn.prices.magic import get_price +from yearn.prices.magic import logger as logger_price_magic from yearn.utils import contract -from yearn.constants import TREASURY_WALLETS, ERC20_TRANSFER_EVENT_HASH, ERC677_TRANSFER_EVENT_HASH - logger = logging.getLogger(__name__) logger_price_magic.setLevel(logging.CRITICAL) @@ -44,17 +47,17 @@ def _get_price(token, block=None): try: price = get_price(token, block) except AttributeError: - logger.warn( + logger.error( f"AttributeError while getting price for {contract(token).symbol()} {token}" ) raise except PriceError: - logger.warn( + logger.error( f"PriceError while getting price for {contract(token).symbol()} {token}" ) price = 0 except ValueError: - logger.warn( + logger.error( f"ValueError while getting price for {contract(token).symbol()} {token}" ) price = 0 @@ -73,17 +76,22 @@ def get_token_from_event(event): return None except EventLookupError: logger.critical(event) + logger.critical( + f'One of your cached contracts has an incorrect definition: {event.address}. Please fix this manually' + ) raise + class Treasury: ''' - Used to export Yearn financial reports + Used to export financial reports ''' - def __init__(self, watch_events_forever=False): - self.addresses = list(TREASURY_WALLETS) + def __init__(self, label, wallets, watch_events_forever=False, start_block=0): + self.label = label + self.addresses = list(wallets) + self._start_block = start_block self._transfers = [] - self._start_block = 10502337 # Treasury didn't exist prior to block 10502337 # define transfer signatures for Transfer events from ERC-20 and ERC-677 contracts transfer_sigs = [ @@ -102,7 +110,6 @@ def __init__(self, watch_events_forever=False): treasury_addresses # Transfers out of Treasury wallets ] ] - self._watch_events_forever = watch_events_forever self._done = threading.Event() self._thread = threading.Thread(target=self.watch_transfers, daemon=True) @@ -174,11 +181,19 @@ def held_assets(self, block=None) -> dict: return balances def collateral(self, block=None) -> dict: - collateral = { - 'MakerDAO': self.maker_collateral(block=block), - } - if block is None or block >= 11315910: - collateral['Unit.xyz'] = self.unit_collateral(block=block) + collateral = {} + + maker_collateral = self.maker_collateral(block=block) + if maker_collateral: + for address, data in maker_collateral.items(): + collateral[f'{address} Maker CDP'] = data + + unit_collateral = self.unit_collateral(block=block) + if unit_collateral: + for address, data in unit_collateral.items(): + collateral[f'{address} Unit CDP'] = data + + collateral = {key: value for key, value in collateral.items() if len(value)} return collateral def maker_collateral(self, block=None) -> dict: @@ -187,90 +202,122 @@ def maker_collateral(self, block=None) -> dict: # ychad = contract('ychad.eth') ychad = contract('0xfeb4acf3df3cdea7399794d0869ef76a6efaff52') vat = contract('0x35D1b3F3D7966A1DFe207aa4514C12a259A0492B') - proxy = proxy_registry.proxies(ychad) - cdp = cdp_manager.first(proxy) - urn = cdp_manager.urns(cdp) - ilk = encode_single('bytes32', b'YFI-A') - ink = vat.urns(ilk, urn, block_identifier=block).dict()["ink"] yfi = "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e" - collateral = { - yfi: { - 'balance': ink / 10 ** 18, - 'usd value': ink / 10 ** 18 * get_price(yfi, block) if ink > 0 else 0, - } - } + ilk = encode_single('bytes32', b'YFI-A') + collateral = {} + for address in self.addresses: + proxy = proxy_registry.proxies(address) + cdp = cdp_manager.first(proxy) + urn = cdp_manager.urns(cdp) + ink = vat.urns(ilk, urn, block_identifier=block).dict()["ink"] + if ink: + collateral[address] = { + yfi: { + 'balance': ink / 10 ** 18, + 'usd value': ink / 10 ** 18 * get_price(yfi, block) if ink > 0 else 0, + } + } return collateral def unit_collateral(self, block=None) -> dict: + # NOTE: This only works for YFI collateral, must extend before using for other collaterals if block and block < 11315910: return - # ychad = contract('ychad.eth') - ychad = contract('0xfeb4acf3df3cdea7399794d0869ef76a6efaff52') unitVault = contract("0xb1cff81b9305166ff1efc49a129ad2afcd7bcf19") yfi = "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e" - bal = unitVault.collaterals(yfi, ychad, block_identifier=block) - collateral = { - yfi: { - 'balance': bal / 10 ** 18, - 'usd value': bal / 10 ** 18 * get_price(yfi, block), - } - } + collateral = {} + for address in self.addresses: + bal = unitVault.collaterals(yfi, address, block_identifier=block) + if bal: + collateral[address] = { + yfi: { + 'balance': bal / 10 ** 18, + 'usd value': bal / 10 ** 18 * get_price(yfi, block), + } + } return collateral - # def bonded_kp3r(self, block=None) -> dict: - # descriptive functions # debt def debt(self, block=None) -> dict: - debt = { - 'MakerDAO': self.maker_debt(block=block), - } - if not block or block >= 11315910: - debt['Unit.xyz'] = self.unit_debt(block=block) - # TODO: self.accounts_payable() + debt = {address: {} for address in self.addresses} + + maker_debt = self.maker_debt(block=block) + if maker_debt: + for address, data in maker_debt.items(): + debt[address].update(data) + + unit_debt = self.unit_debt(block=block) + if unit_debt: + for address, data in unit_debt.items(): + debt[address].update(data) + + compound_debt = self.compound_debt(block=block) + if compound_debt: + for address, data in compound_debt.items(): + debt[address].update(data) + + debt = {key: value for key, value in debt.items() if len(value)} return debt - def accounts_payable(self, block=None) -> dict: - for i, partner in enumerate(partners): - if i == 1: - flat_wrappers = [] - for wrapper in partner.wrappers: - if isinstance(wrapper, Wrapper): - flat_wrappers.append(wrapper) - elif isinstance(wrapper, WildcardWrapper): - flat_wrappers.extend(wrapper.unwrap()) - for wrapper in flat_wrappers: - print(wrapper.protocol_fees(block=block)) - def maker_debt(self, block=None) -> dict: proxy_registry = contract('0x4678f0a6958e4D2Bc4F1BAF7Bc52E8F3564f3fE4') cdp_manager = contract('0x5ef30b9986345249bc32d8928B7ee64DE9435E39') - # ychad = contract('ychad.eth') - ychad = contract('0xfeb4acf3df3cdea7399794d0869ef76a6efaff52') vat = contract('0x35D1b3F3D7966A1DFe207aa4514C12a259A0492B') - proxy = proxy_registry.proxies(ychad) - cdp = cdp_manager.first(proxy) - urn = cdp_manager.urns(cdp) ilk = encode_single('bytes32', b'YFI-A') - art = vat.urns(ilk, urn, block_identifier=block).dict()["art"] - rate = vat.ilks(ilk, block_identifier=block).dict()["rate"] - debt = art * rate / 1e45 dai = '0x6B175474E89094C44Da98b954EedeAC495271d0F' - debt = {dai: {'balance': debt, 'usd value': debt}} - return debt + maker_debt = {} + for address in self.addresses: + proxy = proxy_registry.proxies(address) + cdp = cdp_manager.first(proxy) + urn = cdp_manager.urns(cdp) + art = vat.urns(ilk, urn, block_identifier=block).dict()["art"] + rate = vat.ilks(ilk, block_identifier=block).dict()["rate"] + debt = art * rate / 1e45 + maker_debt[address] = {dai: {'balance': debt, 'usd value': debt}} + return maker_debt def unit_debt(self, block=None) -> dict: + # NOTE: This only works for YFI based debt, must extend before using for other collaterals if block and block < 11315910: return - # ychad = contract('ychad.eth') - ychad = contract('0xfeb4acf3df3cdea7399794d0869ef76a6efaff52') unitVault = contract("0xb1cff81b9305166ff1efc49a129ad2afcd7bcf19") yfi = "0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e" usdp = '0x1456688345527bE1f37E9e627DA0837D6f08C925' - debt = unitVault.getTotalDebt(yfi, ychad, block_identifier=block) / 10 ** 18 - debt = {usdp: {'balance': debt, 'usd value': debt}} - return debt + unit_debt = {} + for address in self.addresses: + debt = unitVault.getTotalDebt(yfi, address, block_identifier=block) / 10 ** 18 + unit_debt[address] = {usdp: {'balance': debt, 'usd value': debt}} + return unit_debt + + def compound_debt(self, block=None) -> dict: + markets = {market.ctoken for comp in compound.compound.compounds for market in comp.markets} + gas_token_markets = [market for market in markets if not hasattr(market,'underlying')] + other_markets = [market for market in markets if hasattr(market,'underlying')] + markets = gas_token_markets + other_markets + underlyings = [weth for market in gas_token_markets] + fetch_multicall(*[[market,'underlying'] for market in other_markets]) + + markets_zip = zip(markets,underlyings) + markets, underlyings = [], [] + for contract, underlying in markets_zip: + if underlying != ZERO_ADDRESS: + markets.append(contract) + underlyings.append(underlying) + + underlying_contracts = [Contract(underlying) for underlying in underlyings] + underlying_decimals = fetch_multicall(*[[underlying,'decimals'] for underlying in underlying_contracts]) + + compound_debt = {} + for address in self.addresses: + debts = fetch_multicall(*[[market,'borrowBalanceStored',address] for market in markets],block=block) + debts = [debt / 10 ** decimals if debt else None for debt, decimals in zip(debts,underlying_decimals)] + compound_debt[address] = {str(underlying): {'balance': debt, 'usd value': debt * get_price(underlying, block=block)} for underlying, debt in zip(underlyings,debts) if debt} + return compound_debt + + def aave_debt(self, block=None) -> dict: + # TODO: don't need this yet but I want to add anyway for completeness + pass # helper functions @@ -284,20 +331,23 @@ def watch_transfers(self): logger.info( 'pulling treasury transfer events, please wait patiently this takes a while...' ) + transfer_filters = [ + web3.eth.filter({"fromBlock": self._start_block, "topics": topics}) + for topics in self._topics + ] for block in chain.new_blocks(height_buffer=12): - for topics in self._topics: - topic_filter = web3.eth.filter({"fromBlock": self._start_block, "topics": topics}) - logs = topic_filter.get_new_entries() + for transfer_filter in transfer_filters: + logs = transfer_filter.get_new_entries() self.process_transfers(logs) if not self._done.is_set(): self._done.set() logger.info( - "loaded treasury transfer events in %.3fs", time.time() - start + f"loaded {self.label} transfer events in %.3fs", time.time() - start ) if not self._watch_events_forever: break - time.sleep(30) + time.sleep(5) def process_transfers(self, logs): for log in logs: @@ -329,5 +379,30 @@ def describe(self, block) -> dict: def export(self, block, ts): start = time.time() data = self.describe(block) - victoria.export_treasury(ts, data) + output_treasury.export(ts, data, self.label) logger.info('exported block=%d took=%.3fs', block, time.time() - start) + + +class YearnTreasury(Treasury): + def __init__(self,watch_events_forever=False): + super().__init__('treasury',TREASURY_WALLETS,watch_events_forever=watch_events_forever,start_block=10502337) + + def partners_debt(self, block=None) -> dict: + for i, partner in enumerate(partners): + if i == 1: + flat_wrappers = [] + for wrapper in partner.wrappers: + if isinstance(wrapper, Wrapper): + flat_wrappers.append(wrapper) + elif isinstance(wrapper, WildcardWrapper): + flat_wrappers.extend(wrapper.unwrap()) + for wrapper in flat_wrappers: + print(wrapper.protocol_fees(block=block)) + + # def bonded_kp3r(self, block=None) -> dict: + + # def debt - expends super().debt + +class StrategistMultisig(Treasury): + def __init__(self,watch_events_forever=False): + super().__init__('sms',STRATEGIST_MULTISIG,watch_events_forever=watch_events_forever,start_block=11507716) diff --git a/yearn/v1/vaults.py b/yearn/v1/vaults.py index f79b7e12d..dfe12d7bf 100644 --- a/yearn/v1/vaults.py +++ b/yearn/v1/vaults.py @@ -1,5 +1,4 @@ import logging - from dataclasses import dataclass from functools import cached_property from typing import Optional diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 7e6a64a4a..740b0c6b7 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -1,10 +1,11 @@ import logging +import os import re import threading import time from typing import List -from brownie import ZERO_ADDRESS, Contract, chain +from brownie import chain from eth_utils import encode_hex, event_abi_to_log_topic from joblib import Parallel, delayed from semantic_version.base import Version diff --git a/yearn/yearn.py b/yearn/yearn.py index 9efbe6a31..5565ba3ad 100644 --- a/yearn/yearn.py +++ b/yearn/yearn.py @@ -7,14 +7,14 @@ import yearn.iearn import yearn.ironbank -from yearn.outputs.describers.registry import RegistryWalletDescriber import yearn.special import yearn.v1.registry import yearn.v2.registry -from yearn.networks import Network -from yearn.outputs import victoria from yearn.exceptions import UnsupportedNetwork +from yearn.networks import Network +from yearn.outputs.victoria import output_base, output_wallets from yearn.prices import constants +from yearn.utils import contract logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class Yearn: Can describe all products. """ - def __init__(self, load_strategies=True, load_harvests=False, watch_events_forever=True, exclude_ib_tvl=True) -> None: + def __init__(self, load_strategies=True, load_harvests=False, load_transfers=False, watch_events_forever=True, exclude_ib_tvl=True) -> None: start = time() if chain.id == Network.Mainnet: self.registries = { @@ -36,25 +36,37 @@ def __init__(self, load_strategies=True, load_harvests=False, watch_events_forev } elif chain.id == Network.Fantom: self.registries = { - "v2": yearn.v2.registry.Registry(), + "v2": yearn.v2.registry.Registry(watch_events_forever=watch_events_forever), "ib": yearn.ironbank.Registry(exclude_ib_tvl=exclude_ib_tvl), } elif chain.id == Network.Arbitrum: self.registries = { - "v2": yearn.v2.registry.Registry(), + "v2": yearn.v2.registry.Registry(watch_events_forever=watch_events_forever), "ib": yearn.ironbank.Registry(exclude_ib_tvl=exclude_ib_tvl), } else: raise UnsupportedNetwork('yearn is not supported on this network') + self.exclude_ib_tvl = exclude_ib_tvl + if load_strategies: self.registries["v2"].load_strategies() if load_harvests: self.registries["v2"].load_harvests() - - self.exclude_ib_tvl = exclude_ib_tvl logger.info('loaded yearn in %.3fs', time() - start) + + def active_vaults_at(self, block=None): + active = [ + vault + for registry in self.registries.values() + for vault in registry.active_vaults_at(block=block) + # [yGov] Doesn't count for this context + if vault.vault != contract("0xBa37B002AbaFDd8E89a1995dA52740bbC013D992") + ] + return active + + def describe(self, block=None): desc = Parallel(4, "threading")( delayed(self.registries[key].describe)(block=block) @@ -62,11 +74,12 @@ def describe(self, block=None): ) return dict(zip(self.registries, desc)) + def describe_wallets(self, block=None): - registries = ['v1','v2'] # TODO: add other registries [earn, ib, special] + from yearn.outputs.describers.registry import RegistryWalletDescriber describer = RegistryWalletDescriber() - data = Parallel(4,'threading')(delayed(describer.describe_wallets)(self.registries[key], block=block) for key in registries) - data = {registry:desc for registry,desc in zip(registries,data)} + data = Parallel(4,'threading')(delayed(describer.describe_wallets)(registry, block=block) for registry in self.registries.items()) + data = {registry:desc for registry,desc in zip(self.registries,data)} wallet_balances = Counter() for registry, reg_desc in data.items(): @@ -81,6 +94,7 @@ def describe_wallets(self, block=None): "wallet balances usd": wallet_balances } } + data.update(agg_stats) return data @@ -96,7 +110,7 @@ def total_value_at(self, block=None): def export(self, block, ts): start = time() data = self.describe(block) - victoria.export(block, ts, data) + output_base.export(block, ts, data) products = list(data.keys()) if self.exclude_ib_tvl and block > constants.ib_snapshot_block: products.remove('ib') @@ -107,5 +121,5 @@ def export(self, block, ts): def export_wallets(self, block, ts): start = time() data = self.describe_wallets(block) - victoria.export_wallets(ts,data) + output_wallets.export(ts,data) logger.info('exported block=%d took=%.3fs', block, time() - start)