diff --git a/.circleci/config.yml b/.circleci/config.yml index c4a692dfa..47b2a93d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,7 @@ commands: name: Install python and other python dependencies command: | sudo apt update - sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-all-dev libffi-dev + sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev pip3 install -r misc/requirements.txt - run: diff --git a/Makefile b/Makefile index 92585f6e9..88d0e6aa3 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,8 @@ GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n') GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version) +COVERPKG := $(shell go list ./... | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' | paste -s -d, - ) + # Used for e2e test export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 ) @@ -45,7 +47,7 @@ fakepackage: go-algorand misc/release.py --host-only --outdir $(PKG_DIR) --fake-release test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer - go test ./... -coverprofile=coverage.txt -covermode=atomic + go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic lint: go-algorand golint -set_exit_status ./... diff --git a/README.md b/README.md index 3ce5ee47e..76ad1941a 100644 --- a/README.md +++ b/README.md @@ -178,20 +178,33 @@ If the maximum number of connections/active queries is reached, subsequent conne Settings can be provided from the command line, a configuration file, or an environment variable -| Command Line Flag (long) | (short) | Config File | Environment Variable | -| ------------------------ | ------- | -------------------------- | ---------------------------------- | -| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | -| pidfile | | pidfile | INDEXER_PIDFILE | -| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | -| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | -| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | -| genesis | g | genesis | INDEXER_GENESIS | -| server | S | server-address | INDEXER_SERVER_ADDRESS | -| no-algod | | no-algod | INDEXER_NO_ALGOD | -| token | t | api-token | INDEXER_API_TOKEN | -| dev-mode | | dev-mode | INDEXER_DEV_MODE | -| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | -| max-conn | | max-conn | INDEXER_MAX_CONN | +| Command Line Flag (long) | (short) | Config File | Environment Variable | +|-------------------------------|---------|-------------------------------|---------------------------------------| +| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | +| pidfile | | pidfile | INDEXER_PIDFILE | +| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | +| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | +| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | +| genesis | g | genesis | INDEXER_GENESIS | +| server | S | server-address | INDEXER_SERVER_ADDRESS | +| no-algod | | no-algod | INDEXER_NO_ALGOD | +| token | t | api-token | INDEXER_API_TOKEN | +| dev-mode | | dev-mode | INDEXER_DEV_MODE | +| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | +| max-conn | | max-conn | INDEXER_MAX_CONN | +| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT | +| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT | +| max-api-resources-per-account | | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT | +| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT | +| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT | +| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT | +| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT | +| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT | +| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT | +| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT | +| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | +| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | +| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | ## Command line diff --git a/accounting/eval_preload.go b/accounting/eval_preload.go new file mode 100644 index 000000000..72b5269a4 --- /dev/null +++ b/accounting/eval_preload.go @@ -0,0 +1,202 @@ +package accounting + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/protocol" +) + +// Add requests for asset and app creators to `assetsReq` and `appsReq` for the given +// transaction. +func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[basics.AssetIndex]struct{}, appsReq map[basics.AppIndex]struct{}) { + txn := &stxnad.Txn + + switch txn.Type { + case protocol.AssetConfigTx: + fields := &txn.AssetConfigTxnFields + if fields.ConfigAsset != 0 { + assetsReq[fields.ConfigAsset] = struct{}{} + } + case protocol.AssetTransferTx: + fields := &txn.AssetTransferTxnFields + if fields.XferAsset != 0 { + assetsReq[fields.XferAsset] = struct{}{} + } + case protocol.AssetFreezeTx: + fields := &txn.AssetFreezeTxnFields + if fields.FreezeAsset != 0 { + assetsReq[fields.FreezeAsset] = struct{}{} + } + case protocol.ApplicationCallTx: + fields := &txn.ApplicationCallTxnFields + if fields.ApplicationID != 0 { + appsReq[fields.ApplicationID] = struct{}{} + } + for _, index := range fields.ForeignApps { + appsReq[index] = struct{}{} + } + for _, index := range fields.ForeignAssets { + assetsReq[index] = struct{}{} + } + } + + for i := range stxnad.ApplyData.EvalDelta.InnerTxns { + addToCreatorsRequest(&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetsReq, appsReq) + } +} + +// MakePreloadCreatorsRequest makes a request for preloading creators in the batch mode. +func MakePreloadCreatorsRequest(payset transactions.Payset) (map[basics.AssetIndex]struct{}, map[basics.AppIndex]struct{}) { + assetsReq := make(map[basics.AssetIndex]struct{}, len(payset)) + appsReq := make(map[basics.AppIndex]struct{}, len(payset)) + + for i := range payset { + addToCreatorsRequest(&payset[i].SignedTxnWithAD, assetsReq, appsReq) + } + + return assetsReq, appsReq +} + +// Add requests for account data and account resources to `addressesReq` and +// `resourcesReq` respectively for the given transaction. +func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress, addressesReq map[basics.Address]struct{}, resourcesReq map[basics.Address]map[ledger.Creatable]struct{}) { + setResourcesReq := func(addr basics.Address, creatable ledger.Creatable) { + c, ok := resourcesReq[addr] + if !ok { + c = make(map[ledger.Creatable]struct{}) + resourcesReq[addr] = c + } + c[creatable] = struct{}{} + } + + txn := &stxnad.Txn + + addressesReq[txn.Sender] = struct{}{} + + switch txn.Type { + case protocol.PaymentTx: + fields := &txn.PaymentTxnFields + addressesReq[fields.Receiver] = struct{}{} + // Close address is optional. + if !fields.CloseRemainderTo.IsZero() { + addressesReq[fields.CloseRemainderTo] = struct{}{} + } + case protocol.AssetConfigTx: + fields := &txn.AssetConfigTxnFields + if fields.ConfigAsset == 0 { + if stxnad.ApplyData.ConfigAsset != 0 { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(stxnad.ApplyData.ConfigAsset), + Type: basics.AssetCreatable, + } + setResourcesReq(txn.Sender, creatable) + } + } else { + if creator := assetCreators[fields.ConfigAsset]; creator.Exists { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(fields.ConfigAsset), + Type: basics.AssetCreatable, + } + addressesReq[creator.Address] = struct{}{} + setResourcesReq(creator.Address, creatable) + } + } + case protocol.AssetTransferTx: + fields := &txn.AssetTransferTxnFields + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(fields.XferAsset), + Type: basics.AssetCreatable, + } + if creator := assetCreators[fields.XferAsset]; creator.Exists { + setResourcesReq(creator.Address, creatable) + } + source := txn.Sender + // If asset sender is non-zero, it is a clawback transaction. Otherwise, + // the transaction sender address is used. + if !fields.AssetSender.IsZero() { + source = fields.AssetSender + } + addressesReq[source] = struct{}{} + setResourcesReq(source, creatable) + addressesReq[fields.AssetReceiver] = struct{}{} + setResourcesReq(fields.AssetReceiver, creatable) + // Asset close address is optional. + if !fields.AssetCloseTo.IsZero() { + addressesReq[fields.AssetCloseTo] = struct{}{} + setResourcesReq(fields.AssetCloseTo, creatable) + } + case protocol.AssetFreezeTx: + fields := &txn.AssetFreezeTxnFields + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(fields.FreezeAsset), + Type: basics.AssetCreatable, + } + if creator := assetCreators[fields.FreezeAsset]; creator.Exists { + setResourcesReq(creator.Address, creatable) + } + setResourcesReq(fields.FreezeAccount, creatable) + case protocol.ApplicationCallTx: + fields := &txn.ApplicationCallTxnFields + if fields.ApplicationID == 0 { + if stxnad.ApplyData.ApplicationID != 0 { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(stxnad.ApplyData.ApplicationID), + Type: basics.AppCreatable, + } + setResourcesReq(txn.Sender, creatable) + } + } else { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(fields.ApplicationID), + Type: basics.AppCreatable, + } + if creator := appCreators[fields.ApplicationID]; creator.Exists { + addressesReq[creator.Address] = struct{}{} + setResourcesReq(creator.Address, creatable) + } + setResourcesReq(txn.Sender, creatable) + } + for _, address := range fields.Accounts { + addressesReq[address] = struct{}{} + } + for _, index := range fields.ForeignApps { + if creator := appCreators[index]; creator.Exists { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AppCreatable, + } + setResourcesReq(creator.Address, creatable) + } + } + for _, index := range fields.ForeignAssets { + if creator := assetCreators[index]; creator.Exists { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AssetCreatable, + } + setResourcesReq(creator.Address, creatable) + } + } + } + + for i := range stxnad.ApplyData.EvalDelta.InnerTxns { + addToAccountsResourcesRequest( + &stxnad.ApplyData.EvalDelta.InnerTxns[i], assetCreators, appCreators, + addressesReq, resourcesReq) + } +} + +// MakePreloadAccountsResourcesRequest makes a request for preloading account data and +// account resources in the batch mode. +func MakePreloadAccountsResourcesRequest(payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}) { + addressesReq := make(map[basics.Address]struct{}, len(payset)) + resourcesReq := make(map[basics.Address]map[ledger.Creatable]struct{}, len(payset)) + + for i := range payset { + addToAccountsResourcesRequest( + &payset[i].SignedTxnWithAD, assetCreators, appCreators, addressesReq, resourcesReq) + } + + return addressesReq, resourcesReq +} diff --git a/api/converter_utils.go b/api/converter_utils.go index 29bef855e..27451b191 100644 --- a/api/converter_utils.go +++ b/api/converter_utils.go @@ -525,7 +525,7 @@ func signedTxnWithAdToTransaction(stxn *transactions.SignedTxnWithAD, extra rowD return txn, nil } -func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) { +func (si *ServerImplementation) assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) { creator, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0)) if len(errorArr) != 0 { return idb.AssetsQuery{}, errors.New(errUnableToParseAddress) @@ -548,13 +548,37 @@ func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.Assets Unit: strOrDefault(params.Unit), Query: "", IncludeDeleted: boolOrDefault(params.IncludeAll), - Limit: min(uintOrDefaultValue(params.Limit, defaultAssetsLimit), maxAssetsLimit), + Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultAssetsLimit), si.opts.MaxAssetsLimit), } return query, nil } -func transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) { +func (si *ServerImplementation) appParamsToApplicationQuery(params generated.SearchForApplicationsParams) (idb.ApplicationQuery, error) { + addr, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0)) + if len(errorArr) != 0 { + return idb.ApplicationQuery{}, errors.New(errUnableToParseAddress) + } + + var appGreaterThan uint64 = 0 + if params.Next != nil { + agt, err := strconv.ParseUint(*params.Next, 10, 64) + if err != nil { + return idb.ApplicationQuery{}, fmt.Errorf("%s: %v", errUnableToParseNext, err) + } + appGreaterThan = agt + } + + return idb.ApplicationQuery{ + ApplicationID: uintOrDefault(params.ApplicationId), + ApplicationIDGreaterThan: appGreaterThan, + Address: addr, + IncludeDeleted: boolOrDefault(params.IncludeAll), + Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultApplicationsLimit), si.opts.MaxApplicationsLimit), + }, nil +} + +func (si *ServerImplementation) transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) { var errorArr = make([]string, 0) // Integer @@ -562,7 +586,7 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions filter.MinRound = uintOrDefault(params.MinRound) filter.AssetID = uintOrDefault(params.AssetId) filter.ApplicationID = uintOrDefault(params.ApplicationId) - filter.Limit = min(uintOrDefaultValue(params.Limit, defaultTransactionsLimit), maxTransactionsLimit) + filter.Limit = min(uintOrDefaultValue(params.Limit, si.opts.DefaultTransactionsLimit), si.opts.MaxTransactionsLimit) // filter Algos or Asset but not both. if filter.AssetID != 0 { @@ -610,3 +634,20 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions return } + +func (si *ServerImplementation) maxAccountsErrorToAccountsErrorResponse(maxErr idb.MaxAPIResourcesPerAccountError) generated.ErrorResponse { + addr := maxErr.Address.String() + max := uint64(si.opts.MaxAPIResourcesPerAccount) + extraData := map[string]interface{}{ + "max-results": max, + "address": addr, + "total-assets-opted-in": maxErr.TotalAssets, + "total-created-assets": maxErr.TotalAssetParams, + "total-apps-opted-in": maxErr.TotalAppLocalStates, + "total-created-apps": maxErr.TotalAppParams, + } + return generated.ErrorResponse{ + Message: "Result limit exceeded", + Data: &extraData, + } +} diff --git a/api/disabled_parameters.go b/api/disabled_parameters.go index 42fb31426..a5a385560 100644 --- a/api/disabled_parameters.go +++ b/api/disabled_parameters.go @@ -320,7 +320,7 @@ func GetDefaultDisabledMapConfigForPostgres() *DisabledMapConfig { get("/v2/accounts", []string{"currency-greater-than", "currency-less-than"}) get("/v2/accounts/{account-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "rekey-to"}) get("/v2/assets", []string{"name", "unit"}) - get("/v2/assets/{asset-id}/balances", []string{"round", "currency-greater-than", "currency-less-than"}) + get("/v2/assets/{asset-id}/balances", []string{"currency-greater-than", "currency-less-than"}) get("/v2/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to", "application-id"}) get("/v2/assets/{asset-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to"}) diff --git a/api/generated/common/routes.go b/api/generated/common/routes.go index 68471ddfc..f5a02a12e 100644 --- a/api/generated/common/routes.go +++ b/api/generated/common/routes.go @@ -71,154 +71,158 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9+2/cONLgv0L0fcAkuZadyewubgIsPmSTCTbYzGwQe2aBi3MYtlTdzbFEaknK7Z6c", - "//cPrCIlSqL6YTtOBtifErf4KLKKxXrz0yxXVa0kSGtmzz/Naq55BRY0/sXzXDXSZqJwfxVgci1qK5Sc", - "PQ/fmLFayNVsPhPu15rb9Ww+k7yCro3rP59p+HcjNBSz51Y3MJ+ZfA0VdwPbbe1a+5FubuYzXhQajBnP", - "+k9ZbpmQedkUwKzm0vDcfTJsI+ya2bUwzHdmQjIlgakls+teY7YUUBbmJAD97wb0NoLaTz4N4nx2nfFy", - "pTSXRbZUuuJ29nz2wve72fvZz5BpVcJ4jS9VtRASwoqgXVCLHGYVK2CJjdbcMgedW2doaBUzwHW+Zkul", - "9yyTgIjXCrKpZs8/zAzIAjRiLgdxhf9daoDfIbNcr8DOPs5TuFta0JkVVWJpbzzmNJimtIZhW1zjSlyB", - "ZK7XCfuxMZYtgHHJ3r9+yb777rvvGW2jhcIT3OSqutnjNbVYKLiF8PkQpL5//RLnP/MLPLQVr+tS5Nyt", - "O3l8XnTf2ZtXU4vpD5IgSCEtrEDTxhsD6bP6wn3ZMU3ouG+Cxq4zRzbTiPUn3rBcyaVYNRoKR42NATqb", - "pgZZCLlil7CdRGE7zec7gQtYKg0HUik1vlcyjef/onSaN1qDzLfZSgPHo7Pmcrwl7/1WmLVqyoKt+RWu", - "m1d4B/i+zPUlPF/xsnFbJHKtXpQrZRj3O1jAkjelZWFi1sjS8Sw3mqdDJgyrtboSBRRzx8Y3a5GvWc4N", - "DYHt2EaUpdv+xkAxtc3p1e0h87aTg+tW+4EL+no3o1vXnp2AazwIWV4qA5lVe+6qcP1wWbD4dukuLnPc", - "zcXO18BwcveBbm3cO+kIuiy3zCJeC8YN4yzcU3MmlmyrGrZB5JTiEvv71bhdq5jbNERO71J1ksnU9o02", - "I7F5C6VK4BI3z0spGS/LHfyyLJmwUBkv1DjWiBMULSudswJKwEV21wH+aqxWW1y8AddO1RaKTDXWE8Va", - "lW5AM0eM0LD0Obp8SpXz0lhuYVIgileyZ9GlqIQdL/dHfi2qpmKyqRagHcIDb7WKabCNlohsDSxHnC1Q", - "6hGuOy9ZzVdgGDjWK0iaw3nc0ZDKMg08X0/TPcG0h9Qrfp1p1cjiAKHFMqXjS8HUkIulgIK1o0zB0k2z", - "Dx4hj4OnE6UicMIgk+C0s+wBR8J1Aq3ueLoviKAIqyfsZ8+d8KtVlyBbJsYWW/xUa7gSqjFtpwkYcerd", - "6oJUFrJaw1Jcj4E889vhOAS18Sy08vd3rqTlQkLhuCsCrSwQt5mEKZrwWCFlwQ385U9TN3T3VcMlbJNM", - "d0gAtJxWK1q7L9R39yraGfYc6gPpcKmG9LeT9g6iO2yUEdtI3MLuq2cqaQ201/8AHTSem/Sf7E66KI0R", - "rreprRjM9PnEXiNWGY04OiVide7u4qUo8Z7+zR2OgNnGuHupj9twcxuxktw2Gp5fyCfuL5axM8tlwXXh", - "fqnopx+b0oozsXI/lfTTW7US+ZlYTW1KgDWpm2K3iv5x46V1UXvdLjc1RficmqHmruElbDW4OXi+xH+u", - "l0hIfKl/n5GWNzVzShF7q9RlU8c7mfcME4ste/NqikpwyF2MEJmGqZU0gOT6giSI9/4395PjdSCRlUdC", - "wOlvRqGQ241da1WDtgJiQ5D7739pWM6ez/7XaWc4OqVu5tRP2OkVduoOo5PLreddxLM8NyMpoKobS3d6", - "ii205/hDC9twzg4tavEb5JY2qA/GI6hqu33sAPawm/vbLfw/CndH7JsHmWvNt595H+lWz/B2Ho/8s5NB", - "HUuv+UpIXPicbdYgWcUvHTvgUtk1aOZwAcaG+534Hl35rQXLCwle0j6ZpU5MAqfmzkjtsPZWre4Ft3vs", - "OhcXH3hdi+L64uJjT84WsoDrNBo+K45LtcoKbvnhxNjbs1eua4Iuv17SGdrM7ouA7pd4jsDCw7LT+9qu", - "ez5s5jb0+x+GmjgVd2eqxoD9Gy+5zOE+sLzwQx2M4R+FFAjE38nA8R80BzS3W3kfKL6PA+zG2XtgsdHD", - "yow45X1skrmvXTqCwYX9+g/Nt7i8M8X/rVT55a1wuQtVOOqemX/QWul7oKIg5A1WPZ9VYAxfQdp0Fu9k", - "aHjI1gWAEe3gloAGhr8DL+365Ro+w2ZGY+/Z0vNOpb6Hjf2sxyrS/vetP1rVHqmtP+yRJyGaxnztu/f1", - "MKXelh/Oy3s4HXL0w3FsjkPyTbAixWaiRNiAD/ERkmyJTo3llnHvBSfr7oW8kK9gKSQ6a55fSMeHThfc", - "iNycNga0lxRPVoo9Z35Ip1VeyNl8eBFOmVrR0emhqZtFKXJ2CdsUFsgDm9bLy5VyWrlVlpeRKyryy3oH", - "QGdSGpMcTZA5ylCNzXw8Q6Zhw3WRAN207gccmRzEu2adMz82eUl8vIQfP30MeF2bDB15GXrypswS5cAo", - "Ycj7xxzKmLFKBx+IMAEaxO9Pynq/At8woi/WGDDs14rXH4S0H1l20Tx9+h2wF3X91o155uD41fsE3Hna", - "1uRZPdoEEQZLSTy4cMRnBtdW8ww9hcnlW+A1Yn8NzDQVOp3LkmG3nqGm1mqleeWdju0Cwn5MI4DgOOwu", - "i1aIizujXiGKJ70E/IQoxDZsDaX3pt0BX5EedWt07dHFdsQNXVx8wJCggJk2hGDFhTThVjBiJd0h8NEW", - "C2C5kwKgOGFvlgy52rzX3cf8eY7Zsg5hKECCnbs1om+M5Vxi4ERdYCCBkIzL7dAob8Da4AJ5D5ewPY9c", - "a0e6aLwfnu+5EovGDddeix2G2YYbVil0z+Qgbbn1rv0EaaaBaYS05GPMKXwic/Q7xTTw1EQRHO7gxCzE", - "jzEkxCiggdc1W5Vq4TlNS6LPWxoNfaaZyjsHgLkHhpJUnMI27Dh7NdeJjaCDOLEFt1ioG+9Ox3Dn8m5N", - "ckuhDYaNAPd3BI+PyC0oz8e0jEH51xpQKlMaYzv6JGXCkU4Rfeuyns9qrq3IRX2YqZVGf9fr4wbZd7Un", - "L3O1HN7Zoys1eYVQ42zBTfr6BvfFUWBjKN7JrTEwujATScu4ghOG/ml/VBclhkC14ZmEY64xNissm8IV", - "p0BLnwvQspOpAhj9HYmFtzU3IUwLo9kCizhIzJkg3nO3AUjA7txE1BvLrcLNW8IVn9r/adf4G1k43gGm", - "H7LWOr7DtTI8/vM2woTC0IODPHjFgyvc/euovSlLJpaskZdSbZxwfIyzez5zkl+TRpKSKPm5M7ei7aDG", - "gXw8wN+YCG0Oqn8ul6WQwDIm2j2wuAcUeKhyQdF33fn0c4BTDJ4wR4NugINHSBF3BHatVEkDs59UfGLl", - "6hggJQjkMTyMjcwm+hsOMDu1wRpe5dirGow5Sne05l04DKFxrM+1Lup3Q+aW1Np6rRg1WXgtJLrEUoTr", - "GFbu1H5pGgw+tSpX5clIXTNQAvL/rMdvM6eaJSU9QDI8C90iVY49EksneD2OGLyGlTAWtFfjEcI2oqgL", - "mNpacJBxa0G7if7fo/9+/uFF9n959vvT7Pv/ffrx059uHj8Z/fjs5q9//f/9n767+evj//6v2cTZgqzW", - "Si2nV2drvXTre69US7vYkWHH3jIffAVXykKG13h2xcsJz7Vr9NqgivEab/wkW+0hm1F8s5gw0OC0l7DN", - "ClE2aXr18/7jlZv2p1YrN83iErZ4eQLP12zBbb7G27U3vWuzY+qS713wW1rwW35v6z3sNLimbmLtyKU/", - "xx/kXAw44i52kCDAFHGMsTa5pTsYJGrUr6Ake/h03g0dzsI1PNllixodpiKMvUusjKCYvjtopORa+rEC", - "06vAwBKM8BY2Cmc3oxUdqgagjZTug2gap3X6ET67uB+vLhb5/Shpmd9/vMPyxsMfurz7igRC7B2jzZJa", - "PCIwPDh+sD3EFRnYxkGhVmkIRkI6LZFARTkfMl7b+Bh1WQeHISaIID4JQjXtVTqY5rMRIIzTI/zaU7TI", - "llpVePLG2l1EnGJCb+mRYHflDGb1WZxjenHME7OL9voZgJf/gO0vri1i1fWmfBEhDz0ynRqHPZmQVt0D", - "au5mMU1Rvh9xL+VTdNsU2WO+H5mteh6QI09AqVZpraxcodyhVl3ofEwOC3BaDVxD3tgua2JgdWkNQw8r", - "TQ4tTOlo58i5Rcmnu+UH3Cg/1h7UvWv55OfEHK9rra54mXmXwBSP1+rK83hsHjwIDyyOpY/Z+Q8v3r7z", - "4KPxGbjOWnVmclXYrv7DrMrJJUpPsNiQWrjmtrXUDu9/7xIQpudG2GBG2kBjdpKWJy5i0J2LKDq93q2w", - "DHL5kU4C782iJe7wakHdOrU6ayT5tPp+LH7FRRnMgAHa9KVCi+s8iUffK/EAd/aHRW7N7F5vitHpTp+O", - "PZwonmFH6llFCZCGKZ9i1uq5qNyiTREJtOJbRzfkjB2zJNlUmTt0mSlFnjYUy4VxJCHJx+kaM2w8oSa7", - "Ed1dnB6rEdFYrpk5ILpuAGQ0R3IzQ4zg1N4tlA/CaKT4dwNMFCCt+6TxLA6OpzuNIX361ipQwhNCadYP", - "qAThhMeoPz4d+E6La0e5jRLk9JrxpB5rfj0t7u6i/7ihpjQfBGK38hO7q0fgvmotpYGKWj87lz3P3hFR", - "L/GMIyljR8SKP3yeVTRSeK//LbCzvzpIULR82vhErsrUVfti+pp14x9xwXb3KQIW36SUyc5LoxLDNHLD", - "pQ358H63fG8DZNZ2vTZKG4sFFJJxXEdpinGe/Z30Q5Mttfod0vZRNCtvxtNHE1Pv9OAH63kDzjCh77WY", - "mSaUfcTYViq4K0itfeDOQA2lg9ap0xXHCbQfo2uSwUypKNFH1o8Nm7jEkNdEEQiojAf/GJfEXF5iuZ2e", - "dphmUXHQ4CmN37EoD/PYhsM3C55fpjUFB9OLLu6m58mzioXObTWKPr5OWBTC07b1hR1q0JWw/SuvO6i3", - "lfr/aOwoFxUv0+J/gbt/3hMoC7ESVFijMRCVhfADsVoJaYmKCmHqkm8psqnbmjdL9nQe8TePjUJcCSMW", - "JWCLb6nFghsUzDozXejilgfSrg02f3ZA83UjCw2FXfuKJUaxVjNDK1frUF+A3QBI9hTbffs9e4ShBEZc", - "wWO3i17cnj3/9nsspUF/PE1daL4Ezy72WyD/Dew/TccYS0FjOFHBj5rmx1REbZrT7zhN1PWQs4Qt/eWw", - "/yxVXPIVpAP0qj0wUV/EJnrsBvsiCyr6g4IlEzY9P1ju+FO25madloUIDJarqhK2cgfIKmZU5eipK0tA", - "k4bhqIIQ8foWrvAR4zZqlrZhPqw9jTL8U6vG6JqfeAX9bZ0zbphpHMydbdAzxBPmK3MUTMlyG1lvcW/c", - "XCiqOMEabexLVmshLVoHGrvM/g/L11zz3LG/kylws8Vf/jQG+W9YvoSBzJWbXx4H+IPvuwYD+iq99XqC", - "7IPQ5fuyR1LJrHIcpXjsuXz/VCYNqMryMh2nHDj6MEx999CHSl5ulGyS3JoeufGIU9+J8OSOAe9Iiu16", - "jqLHo1f24JTZ6DR58MZh6Of3b72UUSkNfSP3IqQO9OQVDVYLuMKQ6TSS3Jh3xIUuD8LCXaD/siEOnQbQ", - "imXhLKcUAUr/G2+H+zle9pQ5QanLS4BayNXpwvUhUZ1GHQrpK5BghJm+QFdrRznus7vyIusPDs0WUCq5", - "Mg9P6QHwCR/6CpAnvXm1D+rRwKHAWIZNpzfGtXNTvAsFyWho1/5L3EhtrO3exNL3vu10aKy7xii54qVP", - "haAIp763mda74egTAFmQWIfsb82FnIiXBSgmovwAZzxT2gqKswH4AjF7VlRgLK/q9DWLRnI6iXiqHaBt", - "F6eNGMiVLAwzQubAoFZmvS+DcyLz6FriZKUwdOXEpcJypalmE8oUVg2y6w6N/d+ZR9iHMdNK2SlAUfiI", - "E0CVsow3dg3StrG1gNUzhyuh7ADUOOhCIZbFfnQ8PlS74mW5nTNhv6FxtA+V5KwCfVkCsxqAbdbKACuB", - "X0FXKhVH+8aw82tRGCyEWsK1yNVK83otcqZ0AfqEvfaedNSCqJOf7+kJ83lRPjb4/Fri8goFpCLF66Rl", - "hhDv1m8Tr3hOF+jwZ6wvaqC8AnPCzjeKgDBdLqlxQkivx6KxlFNRiOUS8JziclB5wn7dhwgmLPqKpWfb", - "Yf2avsBpu5YZyscTSqQlS8W1fEmNmE9E6DvDBkejIo01EFQJxQr0nEyquO2igi532MluStvOYLMEis93", - "nE1Iq1XR5EAZq2c9eozAEiOQ2iqWUTQD0lCoudvBGYwtgac6hRwF3KckZknVXyHiDq5AswWAjAZ6REwn", - "gstYrjEMBKNC/FKheJxmzk290ryAw3y4yAR/ph5tpmUY4UodN8Avrv1QbOrJJr0bP31LR9Hw7paJeXmK", - "l02KXu+nEldeUylhDSXlDmAVWmw7HwlWS4DMCJm2fi4BkLfzPIfakXP8ygCAY1QkxCKrwFTHcLc6DEsr", - "roCyGnYIA1nOy7wpKfZ1x02/yXmp+y6jEpZWOQKLi093JkHh5lpg7C2Vb6X5tGOAUQ+s8XAFeutbkPYU", - "qqW6w6EHcQ7j7KGshCtI6zTAKYno72rDKi63LS7cFB0YczoveFRayElWQSc6Yftnr9hF4NNh8lS3G0iH", - "ionNLWI816CFKkTOhPwN/Glu2VKgGCq7rKQVssFq1Ro6uOmeYJgPNcx5GlOAnsrqdh/6gfMSNj1sF5E8", - "1w8zN5ZfAoEdMrf81XgoTjUYUTQTpkzN8z5kxxGjP7zvuYVT3aLW3BNdDjhUe8h3HbohLQ/IZoCt8S5N", - "8qke8z2EWfE2K4d5Rp2IvPXlIkLLCd1HWRUsTiFduh37CrTpx3RGNkC43jO2a9Ebn4poaEX2heNnyULI", - "jpmcb0vsuKO5IHxRviP2Bx8zktjBiQojLQBmI2y+zibSWFxbakFpQANNazwliRB4CmG5hNweAgPmQ1D1", - "8Uko6LOD4hXwAlPwutQWSmoZgvLoJ8Xc0CaSa6QRKIV2Yg2O8viIMoIthewj/l/UgbR/pfB/6CI94BgE", - "QcbjPm32pDaeeLp8T862YHBX2gjd6IzUyvAy7eEJkxZQ8u2uKbFBf9JWsA1OLrpzuLvD3IVCEcHpUOto", - "an/Odk3umgwX3B7P8amIqxsPMfnDFS8nMm7eQ63BOIGRcXb+w4u33pc3lXeTT6aJceuzWC1nk4nnN3NU", - "eNIsgkLj8Lt/lSNpx5wKh6NoOPd51Pt2QQZTBZqiDQ3RlWOA/hGC/1nNhXdUd0lH4531iWjj1MBDEgg6", - "BA8X4dO7cJDUSuKyXeNoCLbGz1TQg4Xy1WPgJ6ubFYusjW1N1a+fz3x1srgk096AdmGySqw0Mp30qNNV", - "1SJrXCJBkC67xEsqnrFM34aDfe8tfABxB16nSoWZUzgaVdRMIMqIqi7JyeqHcvdr3IsdlUTXxb19/jDK", - "+47Q+uwxVnBrB9/9h1bdFpb9CfO7w6j+KV+qqi5h+j6oyT1ODwrRzYklGqKnY4KpReV5ozsb3DBQ6hde", - "CnrTwGCZBqlUjXUZaiuk+w/mo6nG0v+Ba/cfKhrU/x9RVVS9wQ01Q7wIOfPlf1RjQ7j5zF3ZBSkMvm+q", - "usMtc1oPMh6P75oER9wZ6N674xEzJZm8u+B9dyrxywq/xDkCjADBYA0T/jKsAAu6crLrWm1Y1eRrDIvn", - "KwhR8hiBgobTwUS90UMwXT/bwzsfTc1zGogClEquV6CZjxlivqBuG3hUcTF4LGYYFoCqLE/dv/ti98eP", - "JKG0FEXwJ1IEAhiXsD0lYQB/vwXjmE4EmAAM0wE+I0h3yiqIE1P20OtlT46iCmC9XJ4W/HuUpxx8/qwd", - "KU+NU24OXR6uA49DY2C8zsOdTfHeJlhFt7ZDlYHx5k7L8HZxiAyfLuXjuqMSQRuC5bUYgsp+/fZXpmHp", - "36h78gQnePJk7pv++qz/2RHekydpDeyh1AfaIz+GnzdJMf0as8MX/JChGayG6J/Yy1VVKYmGprIcePlk", - "wTDuyeCbe5KBvIJS1ZBsTRscIR1zeTSsmpKTd0tICbrX6ZDAZSNWEgp7LSki4gz/PL+WqbbxVY+to+1I", - "1SCN3o+4XXHeQbE5CiCn91BvO2IX4t2NGJ7ivf2IrykOtR0Rh1qCvsuY536MA+o+rqSm3EUKxBYhLAmF", - "NMLw4FmtEKoU6kGGgOvWgwv/bnjpPdQS/cHnGHScX4KkUo/tS7RWMZCm0d4h7GDF8RwofhgVX/Cma3Lb", - "oo/ZrkJqGo3lrR3eh6FhAD11daJH4ZCjdheSc+2FXGU78opyTCzyDUPiKFq4dtb0c4M7ItQVFAcWDIj9", - "YZg8F/rvyC6iepTdIy7ptLLoWT85Lq/BHr159Zhh7ZypKibRK237lx0XiDwMIoptHMEyTCM8BoolwJQT", - "chC3wZYwYc/eVwJqedVVf8JWQ8PxXigPDET7OzdYzsk39w7zrzT6rAekf6JtPFSc9nx0iaD5bKVVkw5W", - "WlEq/iCMEhUDFLoohMas+Z+/fXb67M9/YYVYgbEn7F+YK0SX77g8Xh+bTHRl93rVPRkC1ubakjzk4ySi", - "OdceoaN4GOHjJXCYh8fwbSpTzGcol2T2OhXT9WYks7DaB5dgmmjEb3rG+vuI5BLSak7MN1PLZTJ1+p/4", - "e2dK0oEnaxhj/QCuTI8g3lIq+Ae9oHgzn+2pxVZetWXYbsd4SpiqnVpeJ47Pd8+y7gSdsLeuNwO5VNpp", - "2lVjnQyAjz4HW2dPSsVcG9vVkcY0G/k7aIWGBMmUzGF0B4poszE2hOcozxsf4ORgaHOk2yj0R2cozcwJ", - "yMekp46PGmukFST+uG38JdrF2l08Duh/rUWZoIJaue8mhmPOpGL0QkLckiL5upwxgtnHafcI6WGPeVwn", - "okjbyRwlFFRzpyuv1Fkp8jWXXcn3/cV4xjR5zGOPfd4/POb3WTRoB5xftmqQVBNBLdKXRnQKCmZvtRa1", - "hwW45tsKpL0l53tHvSlehl6m360B6AkNIPTeV0B66r1oN7b72GYPt6oW2k6J20ZrnE/oPW1kQCiW38mu", - "dIKciLBsMOYyClMNtlOv0rU2+EvYMh1MA3EV2u6x5CO1LLoWrUhlN52LCjq9hAS5lAgkDroSSb1M67UU", - "cE8s+5sdy+memN5JFWaCKsLT0rtoosXCEWR71vbpP6A8tqRta+iHD/TqY/fjZVHHP2Gv2jhm9LVQRF8X", - "3Ez2p6FHhrKB2+RsoYOdiutgc0anzcXFh5qiKRIH1zcgWca1GUs1vgnPl6v2lY2E4SY0u16C7tqljCeh", - "5VL/3jUc221Cs/EDLT3OM7+Pt6nTZ8ijOcMJErFxs77i2JPl2sPQUcseI+TO0qY+4gedNtHFdqyFMLZr", - "U4GD7oeXvCzPryXNlAhA6V5vTrkcqVqwz+VomaTjpN7rGAxH/oDGDhKe507KKrpY0QjObwwb1qSiCNJx", - "VareJX4kk0y8odOSG9eryXWjzWgsCYqccb1qKrLpf/717VnBZCVWUfg0snE5US810UlvNBRMaZ9AIpY+", - "O2iqHs6BNQLp7SF88b6Tzrrw1QlKnzv9A2pfrUHJLG8d4u6qckqeVeyCHMkXsxP2hoLNNfCCeKYWFlLV", - "6nrrx8zXDZQlmvSJorMWu1Et0hN3inrVAA1StgZ8YihRn/KPWv+Q16aZwNgUVyLBpo+kL4Chl26mru48", - "ISnnUir7B8LTkfUPB4+sReEfdd0WQixBhrf+SPTFYSfMpEqDWMldDyMtebgIzBBdyeugz6V8kluMeDO6", - "JVqJ+HZMFJ0fNBi9f8KLTMlym+KucULjgL22e7HzdaQ2xdF0IUPGrzKqpnPYEgObeRetEAkbteZ397u+", - "W5SrvHONysEAPa6xr28vLmrv8/f9ofdJZpGjcadkRqVdSrdw4k8asnB/Bo4lC6r60nRhVhfyBfsdtPL6", - "YjuUOxCdedqn/vus3JNEp7ZEkxl1G055ZAksWvwO6XCyjN7FxYdrPpIyEKY7yBe3q4i4F8evJ0oQxTgO", - "3ipfc+iOtcVoxh0bO/X258XFhyUvikE1ljj0iphMW02EdtvXYkJi4ZuJskc7sbncic0d4/dSNzZB4dvx", - "PlNQEClJZhN2nHqkwlGnQyu7anXjqQ85/K3//iDSCErvXYkjzLqDPHZUyeQV6mQv2gLIHjjVwnfCPAvx", - "vu7wuw6mlHIZuFlwjwUH7uCBLHr0nVW8vtcanHuZRwTxtNsfJp3+XUKUv5jDeFGtBxygiy4YPsN1t/f+", - "wuhpDOLXYRoMjwvBdE9/aqgwh6tTMRPI8QXkWrGwq+xHgRQY9xCHhptohnivGXvjRublhm9NMJV2hDU9", - "XNhVqhiTMNPFSZ5k303vjc7RMfYeclELfM20zwVbGp82ME68JkuGSsd0KPtMXLVGCx8bzruSjH3nV/B9", - "+eJyPLqg536bedm3FtDAwRjs2rwMY4cVtSiN7rMDXmJLlOpst3QPz/PeyZ3MzlsKj+Vx1IuYHE0zzd3k", - "8NGkCbeIdI0c0n7k+rJ3B3LTf8mRkiB6o/ZEjCh14RbPuHlnwrvunSoMxW5N+7+AJgfmey4LVbHXjSQq", - "ePTL+9eP/QvvgchC2QNHfB6Sr/SFt1ov/crPBu+6hUh08mmshLE6Ybf8el99W45ffUu8feZWd1/vvV0W", - "X+i9t3L03tvtV3r4S2/hxEy98/ZVEtAeTSI4OHdzT++LOZZ9+m7EP/1MtxMPSTqceO/ftvWuBhf/nYSs", - "3uu33LKNkz6Mr1naCVv9oM6uerBsYzMjP8LeoM/+eBPPung5CyfBooeJR1ONf4w33C3Rs+v0KhdVPS4j", - "4WfZyMIMtrB7aWSHB3Sn7ONFn9BmpzN1Sig4VBI4i12lfUjQFelTQdpHf4ePCWElWqo5iw8v05u/wzJS", - "3VbWWl2JIvXGR6lWIjdkgTnWZ/s29L2Zz6qmtOKW4/wY+pITOX0dipW/CmXBdcGgePbnP3/7fbfcr4xd", - "jTcpGWDjl+WNjNyKvC/Htqs7gIkFVJ6s1JhlTfra9KpzPbS+tTnWzu7i145zkSEg6fVGiw0hGost4xGp", - "Kye2l1Z0P83db2tu1h3rjOqfY116zjy/GsbdYdbPl3lMKjoU2Z1CIwbHY4pxdIfkazgbg7fWRH4wS/wx", - "4iTj8uB+iWR2dfQSUiFxr+sSnGzX8cDxucn1trbqNKCGrvww55kYP5kSj5fedWyA9U6Vk0SoQIITJjuJ", - "Cw0EHVS3iM8d7c9ZDFeqDONag3EQpeNp1vri4mNa2JyqGuCky3SnmyNxezbY0/6O075NSrj1JQHxwDrb", - "bhp4eJDGe36DIdtLlMZyJS3PUW6kAtyzF95gNvP1nmdra2vz/PR0s9mcBGvaSa6q0xWmnWRWNfn6NAxE", - "rz7FieC+i6+U6LhwubUiN+zFuzcoMwlbAkawF3CNVruWsmbPTp5S/QCQvBaz57PvTp6efEs7tkYiOKVa", - "HVRtGNfhSAQFozcF5glfQlztA+urYz0P7P7s6dOwDV5riJxVp78Zou/D/GfxNLjJ/Y14hN6Vx9H7DmMS", - "+VleSrWR7AetFZ0X01QV11tMU7WNloY9e/qUiaWvUYJ+Rcvdrf1hRimSs4+u3+nVs9Moamjwy+mn4LAX", - "xc2ez6eDYrKhbeRaTv96+qnv+Ls5sNmpDzQObYOLt/f36adgWbvZ8enU58rv6j6xPirSdfqJ4jdJU4um", - "SnfqCVqf7LWHDg1a2pH17PmHT4NzBde8qkvAIzW7+diisz2RHq038/aXUqnLpo5/McB1vp7dfLz5nwAA", - "AP//Ltl59z6zAAA=", + "H4sIAAAAAAAC/+x9+4/cNtLgv0L0fUBsX2vGcXYXFwOLD147xhrrZA2PkwXO40PYUnU3MxKpJamZ6fjm", + "f//AKlKiJErdPTN+LJCf7GnxUcUqFov14sdFrqpaSZDWLJ5+XNRc8wosaPyL57lqpM1E4f4qwORa1FYo", + "uXgavjFjtZCbxXIh3K81t9vFciF5BV0b13+50PDvRmgoFk+tbmC5MPkWKu4GtrvatfYj3dwsF7woNBgz", + "nvWfstwxIfOyKYBZzaXhuftk2JWwW2a3wjDfmQnJlASm1sxue43ZWkBZmJMA9L8b0LsIaj/5NIjLxXXG", + "y43SXBbZWumK28XTxTPf72bvZz9DplUJYxyfq2olJASMoEWoJQ6zihWwxkZbbpmDzuEZGlrFDHCdb9la", + "6T1oEhAxriCbavH0/cKALEAj5XIQl/jftQb4HTLL9Qbs4sMyRbu1BZ1ZUSVQe+Upp8E0pTUM2yKOG3EJ", + "krleJ+zHxli2AsYle/vyOfvuu+++Z7SMFgrPcJNYdbPHOLVUKLiF8PkQor59+RznP/MIHtqK13Upcu7w", + "Tm6fZ9139urFFDL9QRIMKaSFDWhaeGMgvVefuS8z04SO+yZo7DZzbDNNWL/jDcuVXItNo6Fw3NgYoL1p", + "apCFkBt2AbtJErbTfLoduIK10nAgl1Lje2XTeP4vyqd5ozXIfJdtNHDcOlsux0vy1i+F2aqmLNiWXyLe", + "vMIzwPdlri/R+ZKXjVsikWv1rNwow7hfwQLWvCktCxOzRpZOZrnRPB8yYVit1aUooFg6MX61FfmW5dzQ", + "ENiOXYmydMvfGCimljmN3R42bzs5uG61HojQ17sYHV57VgKucSOM0f/h2m/3ohDuJ14yYaEyzDT5lnHj", + "odqq0m12s2SRJGOlynnJCm45M1Y5CbFW2h/dJD6Wvn+njbAcCViw1W7YUha90ff3cesD13WpHGZrXhpI", + "r1fAPl4kxDI+JHlZLrzodRqDnzJrf+B1bTLEODOWW4jb1LVrIZWExEna/sC15jv3t7E7py6gjFh01Mny", + "UhnIrNqjSQTlABcsOvvjFTtKr2DvtsBwcveBdCrkbOnETVnumPUEcAzBghaxZGLNdqphV7h1SnGB/T02", + "jqcr5oiPJOupPE5vnGLu0WIkWHulVAlcImt7HTJz9Js+zcrA19TcHVw4QdEedEtWQAmIZMeE+KuxWu0Q", + "eccKS6ZqR3TV2PHmkIUflj4P9woyzqS6GmOyB+lSVMKO0f2RX4uqqZhsqhVoR/Bw8lnFNNhGSyS2BpYj", + "zVa9nV/zDRgG7mAUpGvjPE5wSWWZBp5vp6USwbRHEFX8OtOqkcUBKqVlSsdHtqkhF2sBBWtHmYKlm2Yf", + "PEIeB0+n6EbghEEmwWln2QOOhOsEWd32dF+QQBFVT9jP/uzAr1ZdgGyPGBKWwGoNl0I1pu00ASNOPX+Z", + "k8pCVmtYi+sxkGd+OZyEoDb+gKu8dpUrabmQULizD4FWFkjaTMIUTXisCrniBv7ypyn9qfuq4QJ2SaE7", + "ZABCp72zbt0X6juPRTvDnk19IB/SGRvz3yzvHcR32CgjsZHQkdxXL1TS9oFe/wMsBPHcdDvN7mQpoDHC", + "8Ta1FIOZPt2lxIhNRiOOdonYvHNn8VqUeE7/5jZHoGxj3LnUp204uY3YSG4bDU/P5SP3F8vYmeWy4Lpw", + "v1T0049NacWZ2LifSvrptdqI/ExsphYlwJq0HGC3iv5x46UtBfa6RTc1RficmqHmruEF7DS4OXi+xn+u", + "18hIfK1/J92rnJo5dU1+rdRFU8crmffMRqsde/ViiktwyDlBiELD1EoaQHZ9RhrEW/+b+8nJOpAoyiMl", + "4PQ3o/AK0o1da1WDtgJiM537739pWC+eLv7XaWfWO6Vu5tRP2N367NQZRjuXWy+7SGZ5aUZaQFU3ls70", + "lFho9/H7FrbhnB1Z1Oo3yC0tUB+MB1DVdvfQAexhN/e3Wqanzh+4bkOV/BOuI53qGZ7O45F/Nv7aVPON", + "kIj4kl1tQbKKXzhxwKWyW9DM0QKMDec7yT068lv7olcSvKZ9skjtmARNzZ2J2lHttdNzz1DPvQ8SDy5d", + "R9A6BdIflG8pP1rY+2SBzT3Rftbwen7+nte1KK7Pzz/0rlpCFnCdpscnJXapNlnBLb8dj25euK4JBv2a", + "eahv1L4vBrpf5jmCCp/3RL2v5brnzXYrGfuHZE3sirsLVWPA/o2XXOb3cpyu/FAHU/hHIQUC8Xeycf1B", + "5kDmdinvg8R+de9lI5O9+uAt/AdxU3u49QLcmbT3RdKDCPmZb4Q45X0s0pdi/D84/n45/m+lyi9uRcs5", + "UuGoe2b+QWul74GLgv4+wHq5qMAYvoG0YTxeydDwkKULACPZwaGA5sO/Ay/t9vkWPsFiRmPvWdJ3ncHs", + "Hhb2k26ryLa3D/8Iqz0KeX/YI3dCNI352lfv6xFKvSU/XJb3aDqU6IfT2BxH5JtgI46NwImQLR9eKSR5", + "CoSSjlLcRyCR7+ZcnssXsBYSXbFPz6WTQ6crbkRuThsD2l8CTjaKPWV+yBfc8nO5WA4PwilHCgaZeGjq", + "ZlWKnF3ALkUFin5Jm1zKjTo//8CssryMHM1RTIx373UG4zHL0QSZ4wzV2MzHkmUarrguEqCb1rmII1Nw", + "ztysS+bHJh+oj1Xz46e3wSjAY8LiVA7sTSYRByNkP1DF0fcnZb3XkF8x4i/WGDDs14rX74W0H1h23jx+", + "/B2wZ3XdGS1/7aJqHNDotrhXCygijvTM4NpqnmEcQBJ9C7xG6m+BmabCkJKyZNitH7yj1UbzyocUDMOC", + "ZghAcBx2lkUYInJn1OtmGSmDYwq6T0hCbMO2UI4Di46lV3SLujW59tzEZmI2z8/fYzhmoEwbILThQppw", + "KhixkW4T+Ei3FbDcaQFQnLBXa4ZSbdnr7uOtvcRsRYcwFJzG3jkc0fPNci4xaK0uMExISMblbuhyM2Bt", + "cHC+hQvYvYsc50c6YH2UDd9zJBaNG649FjsKsytuWKXQ+ZqDtOXOB+4kWDMNTCOkpQiCXhjYhNDAXRPF", + "Z7mNE4uQiQi3KFyJ1zXblGrlJU3Lok9bHg19poXKGweAuQeBkrw49SPm0gvBdWIhaCNOBfkdj6gb707b", + "cBa9W7PcWmiDQWHA/RnB4y1yC87zEWtjUP61BdTKlMbIrT5LmbClU0zfBqQsFzXXVuSiPsyKTqO/6fVx", + "g+w72pOHuVoPz+zRkZo8QqhxtuImfXyD++I4sDEUzehwDIIuzETaMmJwwjD6xG/VVYkBjm1oPNGYa4y8", + "DGhTqPgUaOl9AVp2OlUAo78isfK25SYEYWIkcRARB6k5E8z7zi0AMrDbNxH3xnqrcPOWcMmn1n868OWV", + "LJzsANMPSG3DWsKxMo4LDvFjlAIUwl9CzEsIdHH/Om5vypKJNWvkhVRXTjk+JpRluXCaX5MmkpKo+bk9", + "t6HloMaBfTzA35iIbA6qf67XpZDAMibaNbC4BhT0rXJBsbXd/vRzgLsYPGKOB90AB4+QYu4I7FqpkgZm", + "P6l4x8rNMUBKEChjeBgbhU30N6RveKjgoa5HgbRCprkxD3LBaZi9wxIBw0j9FYCkeFwm5JK5e94lL522", + "YhUpL+0g6bj1Bz1V26t55uGUHp+2PhBGeIodhROde7fBJlYWA9BpTXYG4nm9JUUCg+tFWkS3VjPR+Xun", + "ntAVptbqASJ+BwCGZs82FNBfefdeTccnWifal12wJYmRNLdPcUySLhMrNrZUtKFVb4bHdtIe0WvFqMnK", + "368j9Swlkt2uyJU0IE2DKS1W5ao8GRkiDJSAmk3W0ySyC9il7zCAAvYsdIuMFOyBWLsrxcNIddGwEcZC", + "L+2kjYTtAn13mKpRc2tBu4n+34P/fvr+WfZ/efb74+z7/3364eOfbh4+Gv345Oavf/3//Z++u/nrw//+", + "r8XEqQFZrZVaT2Nna712+L1VqpXK2JFhxx6anx2DS2UhQwU1u+TlRLiNa/TS4OX5JeqySYWhR2xGWVNi", + "wvSI017ALitE2aT51c/7jxdu2p9ae5NpVhewQ7UQeL5lK27zLeqNveldm5mpS74X4deE8Gt+b/gethtc", + "UzexduzSn+M/ZF8MZO2cOEgwYIo5xlSbXNIZAYlH/QsoydMznc1Lm7NwDU/mrKyjzVSEsecuTBEU06cS", + "jZTEpR/gNI0FRsNhZpKwURqWGWF06AUXrf90HkTTXPH2Bv/JL7IxdvFl1o+Svs36j3dAbzz8oejdV/gi", + "Uu8YOw1pSiMGw43jB9vDXJHpeJzM4JTkYP6m3RJdFShXUca4jbdRly13GGGCCuKT91TTHqWDaT4ZA0Li", + "KkG4p3iRrbWqcOeNldKIOcXEjbzHgt2RM5jV14YY84sTnpizvNeDBrz8B+x+cW2Rqq53UEwP3TKdgSLc", + "Yfy15W6kuZsvIMX5fsS9nE8huVNsj1UEyCDb8+0duQNKtUnbG8oN6h1q06V8xeywAnf3g2vIG9tl+w3s", + "ia3J8/Nqk0PbaTpLJ3LbUkmLef0BF8qPtYd0b1o5+Skpx+taq0teZt7ZNSXjtbr0Mh6bB9/YZ1bH0tvs", + "3Q/PXr/x4KNbBbjO2uvMJFbYrv6PwcrpJUpPiNiQEr/ltrUkDM9/7+wSpucgu8JM6sGN2WlanrlIQHfO", + "z2j3eofZOujlR7q/vJ+WUJzx10Ldums7Ozt5a/seWn7JRRkM3AHa9KFCyHU+8qPPlXiAO3t6I4d9dq8n", + "xWh3p3fHHkkUzzCTMl1R4r5hyqdGt/dcvNyitRwZtOI7xzdknhyLJNlUmdt0mSlFnnaByJVxLCHJe+8a", + "M2w8cU12I7qzOD1WI6KxXDNzgNFtAGQ0R3IxQ/Tr1NqtlA8vaqT4dwNMFCCt+6RxLw62p9uNoSjLra9A", + "CR8fFW/5jJcgnPCY648vY3En5NpRbnMJcvea8aSeah6flnZ3uf90NuKx/odAzF9+4kCMEbgvWktp4KLW", + "7s5lz2d9RDxXPONIy5iJxfKbz4uKRgrvBbgFdfbXHAsXLV/uJC0ujrpHxdVT7nR7Mtlaq98hbT1Eo+vV", + "ePpoYuqdHvzgW9Bg30zchsSgpNItSNXWn7krSO3t+c5ADc/O1pnSFaTriDS56abU9tjp048EnBDsuP+i", + "eBO8oAZvKJe04Z5jYbvejSm9beMQ0VMav9u2HuaxXYNfrXh+kdaeHUzPuiirnt/WKhY6t5WF+lQ6YVHA", + "VtvWF+mpQVfC9o+B7mJ2W02Ypj1YB+5UXuSqWNn1db5KoxLDNPKKSxtKLXmB5nsbIM+T63WltLFYOS2J", + "ZQG5qHiZVokLXP13PSWrEBtBRZIaA1GJHz8Qq5WQlrioEKYu+Y7i2LqlebVmj5eRVPPUKMSlMGJVArb4", + "llqsuEFlpTNdhS4OPZB2a7D5kwOabxtZaCjs1lefMoq1txW0/LThEyuwVwCSPcZ2337PHmDgiBGX8NCt", + "oldBF0+//R7LItEfj9NCHovdzQndAqVuEPppPsbIGRrDHZ9+1LQUpnKl0/J9ZjdR10P2Erb0R8L+vVRx", + "yTeQDses9sBEfZGa6MUarIssqIAbKltM2PT8YLmTT9mWm21aPyAwWK6qStjKBxIYVTl+6krM0KRhOKoG", + "RxK+hSt8xCidmqXtep/XxkTVWlJYYyzVT7yC/rIuGTfMNA7mzl7mBeIJ81WWCqZkuYssmrg2bi5UUJyy", + "iXbnNau1kBZvzI1dZ/+H5Vuuee7E38kUuNnqL38ag/w3LEXFQObKzS+PA/yzr7sGA/oyvfR6gu2DquX7", + "sgdSyaxyEqV46KV8f1dOBg6lo9KDRB8mJcwPfai+5UbJJtmt6bEbjyT1nRhPzgx4R1Zs8TmKH4/G7LNz", + "ZqPT7MEbR6Gf3772WkalNPQNv6uQKNLTVzRYLeASA+TTRHJj3pEWujyICneB/su6/YPKGallYS+nLgKU", + "7DleDvdzjPbUFVupiwuAWsjN6cr1IVWdRh0q6RuQYISZPkA3W8c57rM78iKLCA7NVlAquTGfn9MD4BN+", + "5Q2gTHr1Yh/Uo4FDscgMm04vjGvnpngTikvS0K79lziR2sjqvWnEb33b6UBod4xRKs1zn/hCUT99Dyzh", + "e8XRTg6yILUOxd+WCzkRHQ1QTES+Ac54prQVFHsC8AXi2KyowFhe1eljFg3HtBNxVztA2y7uNmIgV7Iw", + "zAiZA4Name2+fN2JPLNriZOVwtCRE5d9zJWm+nuoU1g1yKU8NNNjNmu0D2OmlbJTgKLyEaf7KmUZb+wW", + "pG0jqQErIQ8xoVwQvHHQgUIii/3oZHyoXMjLcrdkwn5D42gfPshZBfqiBGY1ALvaKgOsBH4JXVFyHO0b", + "w95di8JgyfESrkWuNprXW5EzpQvQJ+yl9y7jLYg6+fkenzCfBecjwd9dS0SvUEBXpBhPQjME9Le+jBjj", + "JR2gw5+xVrSB8hLMCXt3pQgI02UOG6eE9HqsGksZNIVYrwH3KaKDlyfs132IYMLy6hhs3Q7rcfoCu+1a", + "ZqgfT1wiLVkqruVzasR82knfQTTYGhXdWANDlVBsQC/JkIrLLiroMsWd7qa07Qw2a6BsDCfZhLRaFU0O", + "lJ981uPHCCwxAqmtSBx5+JGHQnX7Ds5gbAky1V3IUcF9TGqWVH0MkXZwCZqi5buBHpDQieAylmsMjcBI", + "CY8qFA/TwrmpN5oXcJhfE4Xgz9SjzasNI1yq4wb4xbUfqk093aR34qdP6Sj23J0ysSxPybJJ1evtVJrS", + "SyoLr6GkTBGsKI5tlyPFag2QGSHT1s81AMp2nudQO3aO3/MBcIKKlFgUFZjYGs5WR2FpxSVQDsuMMpDl", + "vMybkuJBZ076q5yXuu9GKWFtlWOw+JmHziQo3FwrjEelUtw0n3YCMOqBFT0uQe98C7o9hcrXbnPoge9/", + "nCuWlXAJ6TsNcEoZ+7u6YhWXu5YWbooOjGWUWNJCTroKOpaJ2j/7i10EPm0mz3XzQDpSTCxuEdO5Bi1U", + "IXIm5G/gd3MrlgLHUAl9Ja2QDb48oKGDm84Jhtlvwwy3MQfoqRx+96EfTC7hqkftItLn+qHXxvILILBD", + "np4/Gg+lqQYjimbClKl53ofsOGb0m/ctt3CqW9Kae+LLgYRqN/ncphvy8oBtBtQar9KknOoJ30OEFW8z", + "VZgX1IloVF8cJLScuPsoq4LFKSTHt2Nfgjb9OMfIBgjXe8Z2LXrjU8kUrci+cPwsWQhjMZPz7UgcdzwX", + "lC/KbsX+4OMoEis4UU+mBcBcCZtvs4nUDteWWlBqzOCmNZ6SVAjchbBeQ24PgQFzBOgliUko6LOD4gXw", + "AhMuu3QPSvQYgvLgJ8Xc0CbSa6QRqIV2ag2O8vCIeqAth+xj/l/Ugbx/qfB/6CI9YBsERcbTPm32pDae", + "ebrsXs52YHBV2qjVaI/UyvAy7eEJkxZQ8t3clNigP2mr2AYnF5053J1h7kChKNl0+HE0td9nc5O7JkOE", + "2+053hVxpfohJX+45OVEFspbqDUYpzAyzt798Oy19+VN5aLkk6lT3PqcZcvZZJmBmyVeeNIigsLF8Lt/", + "/yppx5wKEaMIMfd51Pt2oQVT5biiBQ0Rh2OA/hEC4lnNhXdUd4k445X1yVnjdLlDguo7Ag+R8ClPOEgK", + "k7hI2zgagm3xM5VvYeEpgjHwk7XsilXWxnum3iJZLnwturgA194gb2GySmw0Cp30qNM19CJrXCJpjg67", + "xKtYXrBMn4aDde8hPoC4A6+7SoWZUzQalcZNEMqIqi7JyeqHGiVvH5VY1sWCffrQwvuOy/rkkVVwawff", + "/QdU3RaW/enp82FU/5TPVVWXMH0e1OQep8fh6OTEghzRM2DB1KLyvNGdDW4YKPULLwW9T2OwKIdUqsYq", + "HLUV0v0Hc7RUY+n/wLX7D5WI6v+PuCqq1eGGWiBdMC0+DBRCsBfuyC7owuD7pmp53DLP8yDj8fisSUjE", + "2eDv3hmPlCnJ5N0FtLtdiV82+CWOm2cECAZrmPCXYQVY0JXTXbfqilVNvsVQcb6BEDmOEShoOB1M1Bs9", + "BNP1MyC889HUPKeBKECp5HoDmvmYIeYrY7eBRxUXg4e/hmEBeJXlqfN3Xzz7+ME71JaiqPZE2HwA4wJ2", + "p6QM4O+3EBzTwfETgGGI/CcE6U6R9nGyxh5+vejpUVTvrZff0oJ/j/qUg8/vtSP1qXEayqHoIR64HRoD", + "YzwPdzbFa5sQFR1uh14Gxos7rcPb1SE6fLpwk+uOlwhaECymxhBU9uu3vzINa//e6KNHOMGjR0vf9Ncn", + "/c+O8R49St/APtf1gdbIj+HnTXJMv6Lw8DVWFGgGK9P451JzVVVKoqGpLAdePlkwjHsy+H6qZCAvoVQ1", + "JFvTAkdEx/wWDZum5OTdElKC7nU6JHDZiI2Ewl5Liog4wz/fXctU2/iox9bRcqQqzkYPwdyuFPOgtCCF", + "jdPL47cdsQvx7kYMj97ffsSXFIfajohDrUHfZcx3fowDqnxupKZ8PgrEFiEsCZU0ovDgicQQqhSqf4aA", + "69aDC/9ueOk91BL9we8w6Di/AEmFPds3361iIE2jvUPYwYrjOVD8MCo+4E3X5LYlPrO5snkajeWtHd6H", + "oWEAPXV1qkfhiKPmi1K59kJusplcmxyTbXzDkEyJFq7ZCo5ucMeEuoLiwCT62B+GCWWh/8TwXbWo7jWm", + "dKpV9ESrHJecYA9evXjIsJ7MVGWP6MXN/WjHBasOg4hiG0ewDFPrjoFiDTDlhBzEbbA1TNiz95VFWl92", + "FZGw1dBwvBfKAwPR/s4Nljjyzb3D/CuNPusB6Z/bHA8VpwIfXTZnudho1aSDlTaUnj4Io8SLASpdFEJj", + "tvzP3z45ffLnv7BCbMDYE/YvzBWiw3dcDLFPTSa6Iou9Wq4MAWvzT0kf8nES0ZxbT9BRPIzw8RI4zOen", + "8G2qNSwXqJdk9joV0/VqpLOw2geXYOpkJG96xvr7iOQS0mpOwjdT63Uynfif+HtnStJBJmsYU/0AqUwP", + "2t5SK/gHvYZ7s1zsqU9WXralyW4neEqYqpRbXie2z3dPsm4HnbDXrjcDuVba3bSrxjodAB/wD7bOnpaK", + "uTa2qxqOaTbyd9AKDQmSKZnD6AwU0WJjbAjPUZ83PsDJwdDmDbdR6A/OUJtZEpAP6Z463mqskVaQ+uOW", + "8ZdoFWt38Dig/7UVZYILauW+mxiOJZOK0XsYcUuK5OtyxghmH6fdY6TPu83j2glF2k7mOKGgOjRdyaHO", + "SpFvuewK/O8vUDPmyWMe7u3L/uE2v89COjNwftlKOlJNBLVIXy7QXVAwe6u1qH1egGu+q0DaW0q+N9Sb", + "4mWwwLWevwHoiRtA6L2vXPjU2/9ubPexzR5ur1poOyVpG+G4nLj3tJEB4WmETnelHeRUhHWDMZdRmGqw", + "nforXWuDv4Ad08E0EFdm7R6+P/KWRceiFanspneigu5eQopcSgUSBx2JdL1M32sp4J5E9jcz6LTDzHOF", + "meAK6jvPEy0VjmDbs7ZP/zH8sSVtV0M/fKBXDb0fL4t3/BP2oo1jRl8LRfR1wc1kfxp6ZCgbuE3OFjrY", + "qbgONmd02pyfv68pmiKxcX0D0mVcm7FW45vwfL1p31RJGG5Cs+s16K5dyngSWq71713Dsd0mNBs/x9OT", + "PJ1Lqea7RVDLFsuFA9j94wBy/6717wt8gaYcu5LSe8iTOcMJErFxi/7FsafLtZuh45Y9RsjZcp8+4ged", + "NtHBdqyFMLZrU4GD7ofnvCzfXUuaKRGA0r3En3I5UgVdn8vRCkknSb3XMRiO/AaNHSQ8z52WVXSxohGc", + "3xg2rNNEEaTjSk29Q/xIIZl4MallN643k3ijzWisCYqccb1pKrLpf3r89mAwWZ1UFD6NbFxi02tNtNMb", + "DQVT2ieQiLXPDpqqEXNg3Tx6aeq12oi808668NUJTl+6+wfUvlqDklneOsTdUeUueVaxc3Ikny9O2CsK", + "NtfAC5KZWlhIVXDr4Y+Zr1eAlekDR2ctdaP6nCduF/Uq5BnkbA34oFSiZuN/ak1AXptmgmJTUokUmz6R", + "vgCFnruZulrsRKScS6nsfxCdjqwJOHhSLwr/qOu2OGAJMrzsSKovDjthJlUaxEbOPYO15uEgMENyJY+D", + "vpTySW4x4c3olGg14tsJUXR+0GD02g0vMiXLXUq6xgmNA/HarsXsW1htiqPpQoaMxzKqpnMYikHMvIkw", + "RMbGW/Ob+8XvFiUc71y3cTBAT2rs69uLi5p5sZ/yq/pD79PMIkfjrGZGpV1KhzjJJw1ZOD+DxJIFVX1p", + "ujCrc/mM/Q5a+ftiO5TbEJ152qf++6zck0SntkSTGXUbTnlkCSxCfkY7nCwtd37+/pqPtAyE6Q76xe2q", + "BO6l8cuJEkQxjYO3ytccumNtMZpxZmGnXno9P3+/5kUxqMYSh16RkGmridBq+1pMyCz8aqLs0Sw117PU", + "nBm/l7pxFS58M69xhQsiJclchRWnHqlw1OnQyq5G3XjqQzZ/678/iDXCpfeuzBFmnWGPmcqRvMI72bO2", + "KLAHTrXwnTAvQryvO/yugymlXAdpFtxjwYE7eA6NnvhnFa/vtS7lXuERQTzt9odJp3+XEBUe6vLjRbUe", + "cIAuumD46NrdXncMo6cpiF+HaTA8LgTTPfSqocIcru6KmSCOLyDXqoVdZT8KpMC4hzg03EQzxGvN2Cs3", + "Mi+v+M4EU2nHWNPDhVWlijEJM12c5En23fTa6BwdY28hF7XAt2v7UrDl8WkD48TbwWSodEKHss/EZWu0", + "8LHhvCvJ2Hd+Bd+XLy7HowN66ZeZl31rAQ0cjMGuzfMwdsCoJWl0nu1PhEgV6GyXdI/M897JWWHnLYXH", + "yjjqRUKOppmWbnL4kNCEW0S6Ro5oP3J90TsDuem/20lJEL1ReypGlLpwi6fNvDPhTfd2E4Zit6b9X0CT", + "A/Mtl4Wq2MtGEhc8+OXty4f+Pf/AZKHsgWM+D8lX+upZrdce87PBW2chEp18GhthrE7YLb/el9DW45fQ", + "Eu+BOezu6w20i+ILvYFWjt5Auz2mh79+FnbM1NtnXyUD7blJBAfnvPT0vphjxafvRvLTz3Q79ZC0wy55", + "ISog4OgZ6l0NDv47KVm9t465ZVdO+zDx46eJoM6uerBsYzMjP8LeoM/+eBNPnXg9CyfBooeJJ3KNf3o5", + "nC3RI/v0UhVVPS4j5WfdyMIMlrB7fWPGAzqr+3jVJ7SZdaZOKQWHagJnsau0Dwm6In0qSPvE8/CBHaxE", + "SzVn8ZlteuF5WEaqW8paq0tRpN69KNVG5IYsMMf6bF+HvjfLRdWUVtxynB9DX3Iip49DsfFHoSy4LhgU", + "T/7852+/79D9ysTVeJGSATYeLW9k5FbkfT22xe4AIRZIebJRY5E16WvTm8710PrWllg7u4tfO85FhoCk", + "8Y2QDSEaqx3jEasrp7aXVnQ/Ld1vW262neiM6p9jXXrOvLwaxt1h1s+XeWAp2hTZnUIjBttjSnB0m+Rr", + "2BuD98dEfrBI/DGSJOPy4B5FMrs6fgmpkLjWdQlOt+tk4Hjf5HpXW3UaSENHfpjzTIyfEYnHS686NsB6", + "p8ppIlQgwSmTncaFBoIOqlvE547W5yyGK1WGcavBOIjS8TRbfX7+Ia1sTlUNcNplutPNkbQ9G6xpf8Vp", + "3SY13PqCgPjMd7Z5Hvj8II3X/AZDtteojeVKWp6j3kgFuBfPvMFs4es9L7bW1ubp6enV1dVJsKad5Ko6", + "3WDaSWZVk29Pw0D0ElKcCO67+EqJTgqXOytyw569eYU6k7AlYAR7AddotWs5a/Hk5DHVDwDJa7F4uvju", + "5PHJt7RiW2SCU6rVQdWGEQ/HIqgYvSowT/gC4mofWF8d63lg9yePH4dl8LeGyFl1+psh/j7MfxZPg4vc", + "X4gH6F15GL3vMGaRn+WFVFeS/aC1ov1imqrieodpqrbR0rAnjx8zsfY1StCvaLk7td8vKEVy8cH1O718", + "chpFDQ1+Of0YHPaiuNnz+RQf14/ciXvbB5/sbKtEWtXhfQ6aYVAEN7RNzxf9evqx77C8ObDZqQ+QDm2H", + "QOLfpx+DRfBm5tOpz/Gf6z6BHxUXO/1Icad0w4ymSnfqKYgf7bWHDg1x2m3HxdP3HwfyAK55VZeAomBx", + "86Flw1aSeHa8Wba/lEpdNHX8iwGu8+3i5sPN/wQAAP//Pao01mC7AAA=", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/api/generated/common/types.go b/api/generated/common/types.go index fa13b5bff..7216e86ac 100644 --- a/api/generated/common/types.go +++ b/api/generated/common/types.go @@ -84,6 +84,18 @@ type Account struct { // * Online - indicates that the associated account used as part of the delegation pool. // * NotParticipating - indicates that the associated account is neither a delegator nor a delegate. Status string `json:"status"` + + // The count of all applications that have been opted in, equivalent to the count of application local data (AppLocalState objects) stored in this account. + TotalAppsOptedIn uint64 `json:"total-apps-opted-in"` + + // The count of all assets that have been opted in, equivalent to the count of AssetHolding objects held by this account. + TotalAssetsOptedIn uint64 `json:"total-assets-opted-in"` + + // The count of all apps (AppParams objects) created by this account. + TotalCreatedApps uint64 `json:"total-created-apps"` + + // The count of all assets (AssetParams objects) created by this account. + TotalCreatedAssets uint64 `json:"total-created-assets"` } // AccountParticipation defines model for AccountParticipation. @@ -235,9 +247,6 @@ type AssetHolding struct { // Asset ID of the holding. AssetId uint64 `json:"asset-id"` - // Address that created this asset. This is the address where the parameters for this asset can be found, and also the address where unwanted asset units can be sent in the worst case. - Creator string `json:"creator"` - // Whether or not the asset holding is currently deleted from its account. Deleted *bool `json:"deleted,omitempty"` @@ -847,6 +856,9 @@ type CurrencyGreaterThan uint64 // CurrencyLessThan defines model for currency-less-than. type CurrencyLessThan uint64 +// Exclude defines model for exclude. +type Exclude []string + // ExcludeCloseTo defines model for exclude-close-to. type ExcludeCloseTo bool @@ -913,6 +925,17 @@ type AccountsResponse struct { NextToken *string `json:"next-token,omitempty"` } +// ApplicationLocalStatesResponse defines model for ApplicationLocalStatesResponse. +type ApplicationLocalStatesResponse struct { + AppsLocalStates []ApplicationLocalState `json:"apps-local-states"` + + // Round at which the results were computed. + CurrentRound uint64 `json:"current-round"` + + // Used for pagination, when making another request provide this token with the next parameter. + NextToken *string `json:"next-token,omitempty"` +} + // ApplicationLogsResponse defines model for ApplicationLogsResponse. type ApplicationLogsResponse struct { @@ -959,6 +982,17 @@ type AssetBalancesResponse struct { NextToken *string `json:"next-token,omitempty"` } +// AssetHoldingsResponse defines model for AssetHoldingsResponse. +type AssetHoldingsResponse struct { + Assets []AssetHolding `json:"assets"` + + // Round at which the results were computed. + CurrentRound uint64 `json:"current-round"` + + // Used for pagination, when making another request provide this token with the next parameter. + NextToken *string `json:"next-token,omitempty"` +} + // AssetResponse defines model for AssetResponse. type AssetResponse struct { diff --git a/api/generated/v2/routes.go b/api/generated/v2/routes.go index b8fa5c63e..86e16bfb0 100644 --- a/api/generated/v2/routes.go +++ b/api/generated/v2/routes.go @@ -24,6 +24,18 @@ type ServerInterface interface { // (GET /v2/accounts/{account-id}) LookupAccountByID(ctx echo.Context, accountId string, params LookupAccountByIDParams) error + // (GET /v2/accounts/{account-id}/apps-local-state) + LookupAccountAppLocalStates(ctx echo.Context, accountId string, params LookupAccountAppLocalStatesParams) error + + // (GET /v2/accounts/{account-id}/assets) + LookupAccountAssets(ctx echo.Context, accountId string, params LookupAccountAssetsParams) error + + // (GET /v2/accounts/{account-id}/created-applications) + LookupAccountCreatedApplications(ctx echo.Context, accountId string, params LookupAccountCreatedApplicationsParams) error + + // (GET /v2/accounts/{account-id}/created-assets) + LookupAccountCreatedAssets(ctx echo.Context, accountId string, params LookupAccountCreatedAssetsParams) error + // (GET /v2/accounts/{account-id}/transactions) LookupAccountTransactions(ctx echo.Context, accountId string, params LookupAccountTransactionsParams) error @@ -73,6 +85,7 @@ func (w *ServerInterfaceWrapper) SearchForAccounts(ctx echo.Context) error { "next": true, "currency-greater-than": true, "include-all": true, + "exclude": true, "currency-less-than": true, "auth-addr": true, "round": true, @@ -140,6 +153,16 @@ func (w *ServerInterfaceWrapper) SearchForAccounts(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) } + // ------------- Optional query parameter "exclude" ------------- + if paramValue := ctx.QueryParam("exclude"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", false, false, "exclude", ctx.QueryParams(), ¶ms.Exclude) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter exclude: %s", err)) + } + // ------------- Optional query parameter "currency-less-than" ------------- if paramValue := ctx.QueryParam("currency-less-than"); paramValue != "" { @@ -192,6 +215,7 @@ func (w *ServerInterfaceWrapper) LookupAccountByID(ctx echo.Context) error { "pretty": true, "round": true, "include-all": true, + "exclude": true, } // Check for unknown query parameters. @@ -232,11 +256,317 @@ func (w *ServerInterfaceWrapper) LookupAccountByID(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) } + // ------------- Optional query parameter "exclude" ------------- + if paramValue := ctx.QueryParam("exclude"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", false, false, "exclude", ctx.QueryParams(), ¶ms.Exclude) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter exclude: %s", err)) + } + // Invoke the callback with all the unmarshalled arguments err = w.Handler.LookupAccountByID(ctx, accountId, params) return err } +// LookupAccountAppLocalStates converts echo context to params. +func (w *ServerInterfaceWrapper) LookupAccountAppLocalStates(ctx echo.Context) error { + + validQueryParams := map[string]bool{ + "pretty": true, + "application-id": true, + "include-all": true, + "limit": true, + "next": true, + } + + // Check for unknown query parameters. + for name, _ := range ctx.QueryParams() { + if _, ok := validQueryParams[name]; !ok { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unknown parameter detected: %s", name)) + } + } + + var err error + // ------------- Path parameter "account-id" ------------- + var accountId string + + err = runtime.BindStyledParameter("simple", false, "account-id", ctx.Param("account-id"), &accountId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter account-id: %s", err)) + } + + // Parameter object where we will unmarshal all parameters from the context + var params LookupAccountAppLocalStatesParams + // ------------- Optional query parameter "application-id" ------------- + if paramValue := ctx.QueryParam("application-id"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "application-id", ctx.QueryParams(), ¶ms.ApplicationId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter application-id: %s", err)) + } + + // ------------- Optional query parameter "include-all" ------------- + if paramValue := ctx.QueryParam("include-all"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "include-all", ctx.QueryParams(), ¶ms.IncludeAll) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + if paramValue := ctx.QueryParam("limit"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // ------------- Optional query parameter "next" ------------- + if paramValue := ctx.QueryParam("next"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "next", ctx.QueryParams(), ¶ms.Next) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter next: %s", err)) + } + + // Invoke the callback with all the unmarshalled arguments + err = w.Handler.LookupAccountAppLocalStates(ctx, accountId, params) + return err +} + +// LookupAccountAssets converts echo context to params. +func (w *ServerInterfaceWrapper) LookupAccountAssets(ctx echo.Context) error { + + validQueryParams := map[string]bool{ + "pretty": true, + "asset-id": true, + "include-all": true, + "limit": true, + "next": true, + } + + // Check for unknown query parameters. + for name, _ := range ctx.QueryParams() { + if _, ok := validQueryParams[name]; !ok { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unknown parameter detected: %s", name)) + } + } + + var err error + // ------------- Path parameter "account-id" ------------- + var accountId string + + err = runtime.BindStyledParameter("simple", false, "account-id", ctx.Param("account-id"), &accountId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter account-id: %s", err)) + } + + // Parameter object where we will unmarshal all parameters from the context + var params LookupAccountAssetsParams + // ------------- Optional query parameter "asset-id" ------------- + if paramValue := ctx.QueryParam("asset-id"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "asset-id", ctx.QueryParams(), ¶ms.AssetId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter asset-id: %s", err)) + } + + // ------------- Optional query parameter "include-all" ------------- + if paramValue := ctx.QueryParam("include-all"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "include-all", ctx.QueryParams(), ¶ms.IncludeAll) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + if paramValue := ctx.QueryParam("limit"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // ------------- Optional query parameter "next" ------------- + if paramValue := ctx.QueryParam("next"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "next", ctx.QueryParams(), ¶ms.Next) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter next: %s", err)) + } + + // Invoke the callback with all the unmarshalled arguments + err = w.Handler.LookupAccountAssets(ctx, accountId, params) + return err +} + +// LookupAccountCreatedApplications converts echo context to params. +func (w *ServerInterfaceWrapper) LookupAccountCreatedApplications(ctx echo.Context) error { + + validQueryParams := map[string]bool{ + "pretty": true, + "application-id": true, + "include-all": true, + "limit": true, + "next": true, + } + + // Check for unknown query parameters. + for name, _ := range ctx.QueryParams() { + if _, ok := validQueryParams[name]; !ok { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unknown parameter detected: %s", name)) + } + } + + var err error + // ------------- Path parameter "account-id" ------------- + var accountId string + + err = runtime.BindStyledParameter("simple", false, "account-id", ctx.Param("account-id"), &accountId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter account-id: %s", err)) + } + + // Parameter object where we will unmarshal all parameters from the context + var params LookupAccountCreatedApplicationsParams + // ------------- Optional query parameter "application-id" ------------- + if paramValue := ctx.QueryParam("application-id"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "application-id", ctx.QueryParams(), ¶ms.ApplicationId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter application-id: %s", err)) + } + + // ------------- Optional query parameter "include-all" ------------- + if paramValue := ctx.QueryParam("include-all"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "include-all", ctx.QueryParams(), ¶ms.IncludeAll) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + if paramValue := ctx.QueryParam("limit"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // ------------- Optional query parameter "next" ------------- + if paramValue := ctx.QueryParam("next"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "next", ctx.QueryParams(), ¶ms.Next) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter next: %s", err)) + } + + // Invoke the callback with all the unmarshalled arguments + err = w.Handler.LookupAccountCreatedApplications(ctx, accountId, params) + return err +} + +// LookupAccountCreatedAssets converts echo context to params. +func (w *ServerInterfaceWrapper) LookupAccountCreatedAssets(ctx echo.Context) error { + + validQueryParams := map[string]bool{ + "pretty": true, + "asset-id": true, + "include-all": true, + "limit": true, + "next": true, + } + + // Check for unknown query parameters. + for name, _ := range ctx.QueryParams() { + if _, ok := validQueryParams[name]; !ok { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unknown parameter detected: %s", name)) + } + } + + var err error + // ------------- Path parameter "account-id" ------------- + var accountId string + + err = runtime.BindStyledParameter("simple", false, "account-id", ctx.Param("account-id"), &accountId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter account-id: %s", err)) + } + + // Parameter object where we will unmarshal all parameters from the context + var params LookupAccountCreatedAssetsParams + // ------------- Optional query parameter "asset-id" ------------- + if paramValue := ctx.QueryParam("asset-id"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "asset-id", ctx.QueryParams(), ¶ms.AssetId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter asset-id: %s", err)) + } + + // ------------- Optional query parameter "include-all" ------------- + if paramValue := ctx.QueryParam("include-all"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "include-all", ctx.QueryParams(), ¶ms.IncludeAll) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter include-all: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + if paramValue := ctx.QueryParam("limit"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // ------------- Optional query parameter "next" ------------- + if paramValue := ctx.QueryParam("next"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "next", ctx.QueryParams(), ¶ms.Next) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter next: %s", err)) + } + + // Invoke the callback with all the unmarshalled arguments + err = w.Handler.LookupAccountCreatedAssets(ctx, accountId, params) + return err +} + // LookupAccountTransactions converts echo context to params. func (w *ServerInterfaceWrapper) LookupAccountTransactions(ctx echo.Context) error { @@ -438,6 +768,7 @@ func (w *ServerInterfaceWrapper) SearchForApplications(ctx echo.Context) error { validQueryParams := map[string]bool{ "pretty": true, "application-id": true, + "creator": true, "include-all": true, "limit": true, "next": true, @@ -464,6 +795,16 @@ func (w *ServerInterfaceWrapper) SearchForApplications(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter application-id: %s", err)) } + // ------------- Optional query parameter "creator" ------------- + if paramValue := ctx.QueryParam("creator"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "creator", ctx.QueryParams(), ¶ms.Creator) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter creator: %s", err)) + } + // ------------- Optional query parameter "include-all" ------------- if paramValue := ctx.QueryParam("include-all"); paramValue != "" { @@ -785,7 +1126,6 @@ func (w *ServerInterfaceWrapper) LookupAssetBalances(ctx echo.Context) error { "include-all": true, "limit": true, "next": true, - "round": true, "currency-greater-than": true, "currency-less-than": true, } @@ -838,16 +1178,6 @@ func (w *ServerInterfaceWrapper) LookupAssetBalances(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter next: %s", err)) } - // ------------- Optional query parameter "round" ------------- - if paramValue := ctx.QueryParam("round"); paramValue != "" { - - } - - err = runtime.BindQueryParameter("form", true, false, "round", ctx.QueryParams(), ¶ms.Round) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter round: %s", err)) - } - // ------------- Optional query parameter "currency-greater-than" ------------- if paramValue := ctx.QueryParam("currency-greater-than"); paramValue != "" { @@ -1397,6 +1727,10 @@ func RegisterHandlers(router interface { router.GET("/v2/accounts", wrapper.SearchForAccounts, m...) router.GET("/v2/accounts/:account-id", wrapper.LookupAccountByID, m...) + router.GET("/v2/accounts/:account-id/apps-local-state", wrapper.LookupAccountAppLocalStates, m...) + router.GET("/v2/accounts/:account-id/assets", wrapper.LookupAccountAssets, m...) + router.GET("/v2/accounts/:account-id/created-applications", wrapper.LookupAccountCreatedApplications, m...) + router.GET("/v2/accounts/:account-id/created-assets", wrapper.LookupAccountCreatedAssets, m...) router.GET("/v2/accounts/:account-id/transactions", wrapper.LookupAccountTransactions, m...) router.GET("/v2/applications", wrapper.SearchForApplications, m...) router.GET("/v2/applications/:application-id", wrapper.LookupApplicationByID, m...) @@ -1414,181 +1748,191 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9f2/cNrboVyHmXWCTvpGdJt3Fa4DFRTbZYINNdoM47QIv7kM5EmeGtUSqJGV7mpfv", - "fsFzSIqSqBmNPXaSdv5KPOKPQ/Lw8Pw+H2e5rGopmDB69vTjrKaKVswwBX/RPJeNMBkv7F8F07niteFS", - "zJ76b0QbxcVqNp9x+2tNzXo2nwlasbaN7T+fKfZrwxUrZk+Nath8pvM1q6gd2Gxq29qN9OnTfEaLQjGt", - "h7P+W5QbwkVeNgUjRlGhaW4/aXLFzZqYNdfEdSZcECkYkUti1p3GZMlZWegTD/SvDVObCGo3+TiI89l1", - "RsuVVFQU2VKqiprZ09kz1+/Tzs9uhkzJkg3X+FxWCy6YXxELCwqHQ4wkBVtCozU1xEJn1+kbGkk0oypf", - "k6VUO5aJQMRrZaKpZk8/zDQTBVNwcjnjl/DfpWLsN5YZqlbMzH6ap85uaZjKDK8SS3vlTk4x3ZRGE2gL", - "a1zxSyaI7XVC3jTakAUjVJB3L5+TJ0+efE9wGw0rHMKNrqqdPV5TOIWCGuY/TznUdy+fw/xnboFTW9G6", - "LnlO7bqT1+dZ+528ejG2mO4gCYTkwrAVU7jxWrP0XX1mv2yZxnfcNUFj1plFm/GDdTdek1yKJV81ihUW", - "GxvN8G7qmomCixW5YJvRIwzT3N0NXLClVGwilmLjg6JpPP9nxdO8UYqJfJOtFKNwddZUDLfkndsKvZZN", - "WZA1vYR10wreANeX2L54zpe0bOwW8VzJZ+VKakLdDhZsSZvSED8xaURpaZYdzeEh4ZrUSl7yghVzS8av", - "1jxfk5xqHALakStelnb7G82KsW1Or24HmodOFq4b7Qcs6MvdjHZdO3aCXcNFyPJSapYZueOt8s8PFQWJ", - "X5f24dL7vVzk/ZoRmNx+wFcb9k5YhC7LDTFwrgWhmlDi36k54UuykQ25gsMp+QX0d6uxu1YRu2lwOJ1H", - "1XImY9s32IzE5i2kLBkVsHmOS8loWW6hl2VJuGGVdkyNJY0wQRFI6ZwUrGSwyPY5gF+1UXIDi9fMtpO1", - "YUUmG+OQYi1LO6Cew4ngsPg5enxKmdNSG2rYKEMUr2THoktecTNc7ht6zaumIqKpFkzZA/e01UiimGmU", - "gMNWjORwZgvgerjtTktS0xXThFnSy5Gbg3ns1RDSEMVovh7He4RpB6pX9DpTshHFBKbFEKniR0HXLOdL", - "zgoSRhmDpZ1mFzxc7AdPy0pF4PhBRsEJs+wAR7DrxLHa62m/wAFFp3pCfnDUCb4aecFEIGJksYFPtWKX", - "XDY6dBqBEabeLi4IaVhWK7bk10Mgz9x2WAqBbRwJrdz7nUthKBessNQVgJaGIbUZhSmacF8mZUE1+8t3", - "Yy90+1WxC7ZJEt0+AuByglS0tl+w7/ZVhBl2XOqJeLiUffzbinuT8A4aZUg2Eq+w/eqISloC7fSfIIPG", - "c6P8k91KFsUx/PM2thW9me6O7dV8leGIg1vCV+/tW7zkJbzTv9jL4U+20fZd6p6tf7k1XwlqGsWenotv", - "7F8kI2eGioKqwv5S4U9vmtLwM76yP5X402u54vkZX41tioc1KZtCtwr/seOlZVFzHZabmsJ/Ts1QU9vw", - "gm0Us3PQfAn/XC8BkehS/TZDKW9s5pQg9lrKi6aOdzLvKCYWG/LqxRiWwJDbCCEQDV1LoRmg6zPkIN65", - "3+xPltYxAaQ8YgJOf9ESmNx27FrJminDWawIsv/9L8WWs6ez/3XaKo5OsZs+dRO2coUZe8Pw5lLjaBfS", - "LEfNkAuo6sbgm54iC+Eefwiw9edsj0UufmG5wQ3qgvGAVbXZPLQAO9j14XYL/g/M3R775kCmStHNHe8j", - "vuoZvM7DkX+wPKgl6TVdcQELn5OrNROkoheWHFAhzZopYs+CaePfd6R7+OQHDZZjEhynfTJL3ZjEmepb", - "H2p7aq/l6iBnu0Ovc37+gdY1L67Pz3/q8NlcFOw6fQx3esalXGUFNXQ6Mnb27IXtmsDLLxd1+jqzQyHQ", - "YZFnj1O4X3J6qO068GXTN8HfI0FN3IrbE1WtmfkbLanI2SFOeeGGmnzCb7jgAMQ/UMFxPGZ/zGErD3HE", - "h7jAdpydFxYa3S/PCFMeYpP0oXZpDwLn9+uI8+Esb43xfytlfnGjs9x2VDDqjpn/rpRUB8Aiz+T1Vj2f", - "VUxrumJp1Vm8k77hlK3zAMOxM7sEUDD8g9HSrJ+v2R1sZjT2ji1934rUB9jYO71WkfS/a/3RqnZwbd1h", - "97wJ0TT6S9+9L4codbZ8Oi3vnGmfok8/Y73fIX/yWqRYTZRwG3AuPlygLtGKsdQQ6qzgqN09F+fiBVty", - "Acaap+fC0qHTBdU816eNZspxiicrSZ4SN6SVKs/FbN5/CMdUrWDodNDUzaLkOblgm9QpoAU2LZeXK2ml", - "ciMNLSNTVGSXdQaAVqU0RDmcILOYIRuTOX+GTLErqooE6DqYH2BkNBBvm3VO3NhoJXH+Em789DWgda0z", - "MORlYMkbU0uUPaWERusfsUdGtJHK20C49tDA+f5LGmdXoFcE8Ys0mmnyc0XrD1yYn0h23jx69ISRZ3X9", - "2o55ZuH42dkE7H3a1GhZ3VsF4QdLcTywcDjPjF0bRTOwFCaXbxit4fTXjOimAqNzWRLo1lHU1EquFK2c", - "0TEswO/H+AEgHNPesmiFsLgz7OW9eNJLgE9whNCGrFnprGm3OK9Ijrrxce2Qxbb4DZ2ffwCXIH8ywYVg", - "RbnQ/lXQfCXsJXDeFgtGcssFsOKEvFoSoGrzTnfn8+coZiAdXKODBHlv1wi2MZJTAY4TdQGOBFwQKjZ9", - "pbxmxngTyDt2wTbvI9PaniYaZ4enO57EorHDhWexPWFyRTWpJJhnciZMuXGm/QRqpoFpuDBoY8zRfSKz", - "+DtGNODWRB4c9uLEJMSN0UfEyKGB1jVZlXLhKE1A0acBR32fcaLy1gKgD0BQkoKT34Ytd6+mKrEReBFH", - "tuAGC7Xj3eoabl3ejVFuyZUGtxFG3RtB4ytyA8xzPi1DUP6zZsCVSQW+HV2U0v5Kp5A+mKzns5oqw3Ne", - "T1O14uhvO33sILue9uRjLpf9N3vwpCafEGycLahOP9/MfrEY2Gj0d7Jr9ITOz4TcMqzghIB92l3VRQku", - "UME9E8+YKvDN8stGd8Ux0NL3ginR8lQejO6OxMzbmmrvpgXebJ5ETGJzRpD3vd0AQGB7byLsjflWbuct", - "2SUd2/9x0/grUVjawXTXZS0Yvv2z0r/+8+Bhgm7o3kDureLeFG7/tdjelCXhS9KICyGvLHO8j7F7PrOc", - "X5M+JCmA87N3boXbgY09+jiA/6SjY7NQ/Xu5LLlgJCM87IGBPUDHQ5lz9L5r76ebg1nB4BticdAOMHmE", - "FHJHYNdSljgw+ZeMb6xY7QOkYBxoDPVjA7GJ/mYT1E7BWcOJHDtFgyFFaa/WvHWHwWMcynPBRP22T9yS", - "UlunFcEmCyeFRI9YCnEtwcqt2C90A86nRuayPBmIa5qVDOh/1qG3mRXNkpweAzQ8890iUY484EvLeD2M", - "CLxiK64NU06MBwiDR1HrMLUxzEJGjWHKTvT/Hvz30w/Psv9Ls98eZd//79OfPn736eE3gx8ff/rrX/9/", - "96cnn/768L//azZyt1hWKymX46sztVra9b2TMuAudCTQsbPMe1/BpTQsg2c8u6TliOXaNnqpQcR4CS9+", - "kqx2DpugfzMfUdDAtBdskxW8bNL46ub95ws77b+CVK6bxQXbwOPJaL4mC2ryNbyuneltmy1Tl3Tngl/j", - "gl/Tg6132m2wTe3EyqJLd46v5F70KOI2cpBAwBRyDE9tdEu3EEiQqF+wEvXh43E3eDkL2/Bkmy5qcJkK", - "P/Y2tjKCYvztwJGSa+n6CoyvAhxLwMObm8idXQ9WNFUMAB0pvgfRNFbqdCPcObsfry5m+d0oaZ7ffbzF", - "8obDT13eoTyB4PT2kWZRLB4gGFwcN9gO5IoUbEOnUCMV80pCvC0RQ4UxHyJe2/AatVEH0w7GsyAuCEI2", - "4SntTXNnCMiG4RFu7SlcJEslK7h5Q+kuQk4+Ird0ULB9cnqzuijOIb5Y4gnRRTvtDIyW/2SbH21bOFXb", - "G+NFuJh6ZVoxDnoSLow8wNHcTmOawnw34k7MR++2MbSHeD9UW3UsIHvegFKu0lJZuQK+Q65a1/kYHRbM", - "SjXsmuWNaaMmelqXoBi6X26yr2FKeztHxi0MPt3OP8BGubF2HN3bQCfv8uRoXSt5ScvMmQTGaLySl47G", - "Q3NvQbhndix9zd7//dnrtw58UD4zqrIgzoyuCtrVX82qLF8i1QiJ9aGFa2qCprb//juTANcdM8IVRKT1", - "JGbLaTnkQgLdmoii2+vMCkvPl+9pJHDWLFziFqsWq4NRq9VGok2ra8eil5SXXg3ooU0/Kri41pK497sS", - "D3Bre1hk1swO+lIMbnf6duygRPEMW0LPKgyA1ES6ELMg54JwCzpFQNCKbizeoDF2SJJEU2X20mW65Hla", - "USwW2qKEQBunbUyg8YiYbEe0b3F6rIZHY9lmeoJ3XQ/IaI7kZnofwbG9W0jnhNEI/mvDCC+YMPaTgrvY", - "u572Nvrw6RuLQAlLCIZZ36MQBBPuI/64cOBbLS6MchMhyMo1w0ndqbn1hLO7jfxjhxqTfACI7cJPbK4e", - "gPsiaEo9FgU7OxUdy94eXi/xjAMuY4vHirt8jlQ0gjur/w1OZ3d2EC9oubDxkViVsaf22fgza8ff44Ft", - "31MALH5JMZKdllomhmnEFRXGx8O73XK9NUO1tu11JZU2kEAh6ce1l6QYx9nfSj7U2VLJ31haPwpq5avh", - "9NHE2Ds9+GQ5r0cZRuS9cDLjiLILGUOmgtuCFPQDtwaqzx0Eo06bHMfjfnxcowRmTESJPpKub9jIIwa0", - "JvJAAGHc28eoQOLyHNLtdKTDNImKnQZPcfyWRDmYhzocerWg+UVaUrAwPWv9bjqWPCOJ7xyyUXTP64RE", - "LjyhrUvsUDNVcdN98tqLelOu/2sjRzmvaJlm/wvY/fcdhrLgK46JNRrNorQQbiBSSy4MYlHBdV3SDXo2", - "tVvzakkezSP65k6j4Jdc80XJoMW32GJBNTBmrZrOd7HLY8KsNTR/PKH5uhGFYoVZu4wlWpIgmYGWKxjU", - "F8xcMSbII2j37ffkAbgSaH7JHtpddOz27Om330MqDfzjUepBcyl4tpHfAuivJ/9pPAZfChzDsgpu1DQ9", - "xiRq45R+y23CrlPuErR0j8Puu1RRQVcs7aBX7YAJ+8JpgsWuty+iwKQ/wFgSbtLzM0MtfcrWVK/TvBCC", - "QXJZVdxU9gIZSbSsLD61aQlwUj8cZhBCWh/g8h/Bb6MmaR3m/erTMMI/tWrwrvkXrVh3W+eEaqIbC3Or", - "G3QE8YS4zBwFkaLcRNpb2Bs7F7AqlrEGHfuS1IoLA9qBxiyz/0PyNVU0t+TvZAzcbPGX74Yg/w3SlxAm", - "cmnnF/sBfu/7rphm6jK99WoE7T3T5fqSB0KKrLIUpXjoqHz3ViYVqNLQMu2n7Cl63019+9BTOS87SjaK", - "bk0H3WhEqW+FeGLLgLdExbCevfBx75XdO2Y2Ko0etLEn9MO7147LqKRiXSX3wocOdPgVxYzi7BJcptOH", - "ZMe85VmoctIp3Ab6z+vi0EoAgS3zdzklCGD433A77M/xssfUCVJeXDBWc7E6Xdg+yKrjqH0mfcUE01yP", - "P6CrtcUc+9k+eZH2B4YmC1ZKsdL3j+ke8BEb+ooBTXr1YhfUg4F9grEMmo5vjG1np3jrE5Lh0Lb953iR", - "gq/tzsDSd67tuGusfcYwuOK5C4VAD6eutRnXe0XBJsBEgWwdkL815WLEX5axYsTLj8GMZ1IZjn42jH0G", - "nz3DK6YNrer0MwtKcryJcKstoKGLlUY0y6UoNNFc5IywWur1rgjOkcijawGTlVzjkxOnCsulwpxNwFMY", - "2Yuum+r7vzWOsAtjpqQ0Y4AC8xEHgEppCG3MmgkTfGsZZM/srwSjA0DiwAcFSRZ5Y2m8z3ZFy3IzJ9z8", - "CcdRzlWSkoqpi5IRoxgjV2upGSkZvWRtqlQY7U+avL/mhYZEqCW75rlcKVqveU6kKpg6IS+dJR2kIOzk", - "5nt0QlxclPMNfn8tYHmFZCgixevEZXoX72C3iVc8xwe0/zPkF9WsvGT6hLy/kgiEbmNJtWVCOj0WjcGY", - "ioIvlwzuKSwHhCfo136IYIKkr5B6Ngzr1vQZbtu1yIA/HhEiDWoqrsVzbERcIELXGNa7GhVKrB6hSlas", - "mJqjShW2nVesjR22vJtUplXYLBn651vKxoVRsmhyhhGrZx18jMDiA5BCFsvImwFwyOfcbeH0yhZPU61A", - "DgzuI2SzhOyuEM6OXTJFFoyJaKAHSHQiuLShCtxAwCvELZUVD9PEualXihZsmg0XiOAP2CNEWvoRLuV+", - "A/xo2/fZpg5v0nnx06905A1vX5mYlqdo2Sjr9W4scOUlphJWrMTYAchCC23nA8ZqyVimuUhrP5eMAW2n", - "ec5qi85xlQHGLKFCJhZIBYQ6+rfVnrAw/JJhVMMWZiDLaZk3Jfq+bnnpr3Jaqq7JqGRLIy2CxcmnW5Ug", - "t3MtwPcW07fifMoSwKgH5Hi4ZGrjWqD05LOl2suhen4Ow+ihrGSXLC3TMIpBRP+QV6SiYhPOwk7RgjHH", - "+wJXJUCOvAoY0fG0f3CCXQQ+XiaHdduBtEcxsrlFfM41U1wWPCdc/MLcbQ5kyWMMpl2WwnDRQLZqxVq4", - "8Z0gEA/Vj3kaYoAai+q2H7qO84JddU67iPi5rpu5NvSCIdg+css9jVPPVDHNi2ZElalo3oVsP2R0l/cd", - "NexUhaPVB8LLHoUKl3zbpevjcg9teqc13KVROtUhvlOIFQ1ROcQR6oTnrUsX4VuOyD7SSK9x8uHSYexL", - "pnTXpzPSAbLrHWPbFp3xMYmGkqhf2H+WzLvs6NH5NkiOW5zzzBfGO0J/5nxGEjs4kmEkAKCvuMnX2UgY", - "i22LLTAMqCdpDadEFgJuIVsuWW6mwADxEJh9fBQK/GyheMFoASF4bWgLBrX0QXnwL0ns0Dria4TmwIW2", - "bA2M8nCPNIIBQ3Yh/49yIu5fSvgfmEgnXAPPyLizT6s9sY1Dnjbek5IN07ArwUM3uiO11LRMW3j8pAUr", - "6WbblNCgO2lgbL2RC98cat8w+6CgR3Da1Tqa2t2zbZPbJv0Fh+s5vBVxduP+Sf79kpYjETfvWK2Ytgwj", - "oeT935+9dra8sbibfDRMjBoXxWooGQ08/zQHgSdNItA1Dr67qhxJPeaYOxx6w9nPg943czIYS9AUbaj3", - "rhwC9E/v/E9qyp2hug06Gu6sC0QbhgZOCSBoD7i/CBfeBYOkVhKn7Rp6Q5A1fMaEHsSnrx4CP5rdrFhk", - "wbc1lb9+PnPZyeKUTDsd2rnOKr5SQHTSo45nVYu0cYkAQXzsEpVUHGEZfw17+95ZeA/iFrxWlPIzp85o", - "kFEzcVCaV3WJRlY3lH1f415kryC61u/t7t0oD+2hdec+VuzGBr7Du1bdFJbdAfPb3aj+LZ7Lqi7Z+HtQ", - "o3kcCwrhywkpGqLSMV7VIvO8Ua0Oru8o9SMtOdY00JCmQUhZQ16G2nBh/wPxaLIx+H9Glf0PJg3q/g+x", - "KsreYIeawblwMXPpf2RjvLv5zD7ZBQoMrm8qu8MNY1onKY+Hb02CIm51dO+88XAyJaq8W+d9eyvhywq+", - "xDECBAEBZw3t/9KkYIapyvKua3lFqiZfg1s8XTHvJQ8eKKA47U3UGd0703WjPZzxUdc0x4HQQamkasUU", - "cT5DxCXUDY5HFeW9YjF9twAQZWnq/d3luz8skgTcUuTBnwgR8GBcsM0pMgPw+w0Ix3ggwAhgEA5whyDd", - "KqogDkzZga8XHT4KM4B1YnkC+Afkpyx87q7tyU8NQ26mLg/WAdeh0Wy4zunGpnhvE6SiXdtUYWC4ueM8", - "vFlM4eHTqXxsdxAicEMgvRYBUMnP3/5MFFu6GnXffAMTfPPN3DX9+XH3s0W8b75JS2D3JT7gHrkx3LxJ", - "jOnmmO1X8AOCpiEboiuxl8uqkgIUTWXZs/KJgoDfk4aae4IwcclKWbNka9zg6NAhlkexVVNStG5xIZjq", - "dJriuKz5SrDCXAv0iDiDP99fi1Tb+KmH1tF2pHKQRvUjbpact5dsDh3IsR7qTUdsXbzbEX0p3puP+BL9", - "UMOIMNSSqduM+d6NMSHv40oojF1ER2zu3ZKAScMT7pXV8q5KPh+kd7gOFlz2a0NLZ6EWYA9+D07H+QUT", - "mOoxVKI1kjChG+UMwhZWGM+C4oaR8QOv2yY3TfqYbUukpkBZHvTwzg0NHOixq2U9Cns4cnsiOduei1W2", - "Ja4oh8Ai19AHjoKGa2tOPzu4RUJVsWJiwoDYHgbBc77/lugizEfZFnFJh5VFZf3EML0GefDqxUMCuXPG", - "sphEVdp2LztOEDkNIvRtHMDSDyPcB4olY2NGyJ7fBlmyEX32rhRQy8s2+xO06iuOd0I50RHtH1RDOifX", - "3BnMv1Dvsw6QrkTbcKg47HnvFEHz2UrJJu2stMJQ/J4bJQgGwHShC41e0z9/+/j08Z//Qgq+YtqckP9A", - "rBA+vsP0eN3TJLxNu9fJ7kkAsBBri/yQ85OI5ly7Ax34w3DnLwHD3P8J3yQzxXwGfElmrlM+Xa8GPAup", - "nXMJhIlG9KajrD+EJxcXRlEkvplcLpOh0/+G31tVkvI0WbHhqU+gylgE8YZcwT+xguKn+WxHLrbyMqRh", - "uxnhKdlY7tTyOnF9njzO2ht0Ql7b3oSJpVRW0q4aY3kAKPrsdZ0dLhVibUybRxrCbMRvTElQJAgiRc4G", - "byCPNht8Q2gO/Lx2Dk4WhhAjHbzQH5wBNzNHIB+inDq8aqQRhiP7Y7fxx2gXa/vwWKD/s+ZlAgtqab/r", - "GI45EZJghYS4JXrytTFjCLPz0+4g0v1e8zhPRJHWk1lMKDDnTpteqdVS5Gsq2pTvu5PxDHFyn2KPXdrf", - "v+aHTBq0Bc7PmzVIyBGnFuFSI1oBBaK3gkbtfgGu6aZiwtyQ8r3F3ugvg5Xpt0sAakQC8L13JZAeqxdt", - "x7YfQ/RwELVAd4rUNlrjfETuCZ4BPll+y7viDbIswrIBn8vITdXrTp1IF3TwF2xDlFcNxFlo22LJe0pZ", - "+Cwanopues8r1solyMilWCA+6UlE8TIt16LDPZLsP21ZTltieitW6BGs8KWlt+FEOIU90PYs9OkWUB5q", - "0jY167oPdPJjd/1lQcY/IS+CHzPYWtCjr3VuRv1T3yKD0cAhOJsrr6eiyuucwWhzfv6hRm+KxMV1DZCX", - "sW2GXI1rQvPlKlTZSChufLPrJVNtu5TyxLdcqt/ahkO9jW82LNDSoTzzQ9SmTt8hd8wZTJDwjZt1BccO", - "LxcuQ4stO5SQW1ObOo8fMNpED9u+GsJYr40JDtofntOyfH8tcKaEA0pbvTllcsRswS6WIxBJS0md1dEr", - "jtwFjQ0kNM8tl1W0vqIRnH/SpJ+TCj1Ih1mpOo/4nkQyUUMnoBtVq9F1g85oyAnynFC1airU6d/9+nas", - "YDQTKy9cGNkwnajjmvCmN4oVRCoXQMKXLjpoLB/OxByBWHsIKt633FnrvjqC6XMrf7DaZWuQIsuDQdw+", - "VVbIM5KcoyH5fHZCXqGzuWK0QJqpuGGpbHWd9UPk6xUrS1DpI0Zn4XSjXKQn9hZ1sgFqwGzFoMRQIj/l", - "15r/kNa6GTmxMaqEjE33kD7DCT23M7V55/GQciqENF/ROe2Z/7BXZC1y/6jrkAixZMLX+kPWF4YdUZNK", - "xfhKbCuMtKT+IdD940o+B10q5YLc4oPXg1cicMQ3I6Jg/MDBsP4JLTIpyk2KusYBjT3yGvZia3WkEOKo", - "W5ch7VYZZdOZtkRPZt5GKwTEBqn57WHXd4N0lbfOUdkboEM1dvXt+EXtLH/fHXoXZxYZGrdyZpjapbQL", - "R/qkWObfT0+xRIFZX5rWzepcPCO/MSWdvBiGsheiVU+70H8XlXuS6BRSNOlBt/6Ue6bAwsVv4Q5H0+id", - "n3+4pgMuA2C6BX9xs4yIO8/45UgKoviMvbXK5Ry6ZW4xnHHLxo7V/jw//7CkRdHLxhK7XiGRCdlEcLdd", - "LiZAFno1kvZo62kut57mlvE7oRtXXuDbUp/JC4gYJHPldxx7pNxRx10r22x1w6mnXP5gv5+EGl7ovS1y", - "+Fm3oMeWLJm0ApnsWUiA7ICTAb4T4kiIs3X735VXpZRLT828ecwbcHsFsrDoO6lofdAcnDuJRwTxuNmf", - "jRr924Ao9zD78aJcDzBA613QL8N1u3p/fvT0CcLXfhgMjRPBtKU/FasghqsVMROH4xLIBbawzeyHjhTg", - "9xC7hutohnivCXllR6blFd1oryptEWt8OL+rmDEmoaaLgzxRv5veG5WDYewdy3nNoZpplwoGHB9XMI5U", - "k0VFpSU6GH3GL4PSwvmG0zYlY9f45W1fLrkcjR7oudtmWna1BTiwVwbbNs/92H5F4Uij92xCJbZEqs6w", - "pTtonrNObiV2TlO4L43DXkjkcJpx6ib6RZNGzCLCNrKH9oaqi84bSHW3kiMGQXRG7bAYUejCDcq4OWPC", - "27ZOFbhiB9X+j0yhAfMdFYWsyMtGIBY8+PHdy4euwrtHMp/2wCKfg+QLrfBWq6Vb+Vmvrpv3REebxopr", - "oxJ6yy+36ttyWPUtUfvMru5Q9d4uis9U760c1Hu7+UqnV3rzN2asztsXiUA7JAlv4NxOPZ0tZl/y6boh", - "/XQz3Yw9RO5wpN6/Cfmueg//rZisTvVbasiV5T60y1naMltdp842e7AIvpmRHWGn02d3vJGyLo7Pgkkg", - "6WGiaKp2xXj92xKVXceqXJj1uIyYn2UjCt3bwrbSyBYL6Fbex7E+vs1WY+oYUzCVEziLTaVdSMAU6UJB", - "QtHffjEhyESLOWeh8DLW/O2nkWq3slbykhepGh+lXPFcowZmX5vta9/303xWNaXhNxznje+LRuT0c8hX", - "7ikUBVUFYcXjP//52+/b5X5h5Gq4SUkHG7csp2SkhuddPjasbgIR80d5spJDkjVqa1Or1vQQbGtzyJ3d", - "+q/tZyIDQNLrjRbrXTQWG0IjVJeWbS8Nb3+a29/WVK9b0hnlP4e89JQ4etX3u4Oon89TTCq6FNmtXCN6", - "12OMcLSX5Eu4G71aazyfTBLfRJRkmB7cLRHVrhZffCgk7HVdMsvbtTRweG9ytamNPPVHg0++n/OMD0um", - "xOOldx0aQL5TaTkRTJBgmcmW4wIFQQvVDfxzB/tzFsOVSsO4VkxbiNL+NGt1fv5Tmtkcyxpguct0p097", - "nu1Zb0+7O477Nsrh1hcIxD3LbNtx4P5BGu75J3DZXgI3lkthaA58Iybgnj1zCrOZy/c8WxtT66enp1dX", - "Vydem3aSy+p0BWEnmZFNvj71A2HVpzgQ3HVxmRItFS43hueaPHv7CngmbkoGHuwFuwatXcCs2eOTR5g/", - "gAla89nT2ZOTRyff4o6tAQlOMVfH7OnHT/PZ6eXj09hVZpUs4sWoytcoCLi2JxALz1C6eVWERi+leuaH", - "c2YPLGP89MNYwSJ7Ze3fvzZMbWY+i36sBmqNccPrsTvKGdUUGl0wTaMwblwxknsmLrI0Y5k9dskE4cj2", - "lbzioXiGskKte7UTMEPbPQFuU2rRFYvgPSE/aBblrZQXEASC7KZ3KfdpF0OnEcDsECm4WpQfhvjirjlW", - "Fzz8qPCa9xWEPYHRRESuoyednHBOVeuKaLiUHfmGNKK0/IU3P4DVUIelQbpAzGaRU7cDLt7K+63q8RPw", - "k2QOwsxCuOeJuMzqIBvBY+I8bUHL5UQnh+PzkH4k9huYtwXYnKJ+TkJCj56Gee7s/r6+8rBsMXoVjC3Y", - "OQFntCxTy4xsTfudcOnK7nyhx2unuNXZeo+/yJzrau3AeiHHpz3wC7YZA6YNjB2/WTv9+LZ/HgPf0zRv", - "RW8rp2CqREigXDMFQ4ocFPkaMNNryZAue0eOgmu6KKGYAIrAHS+AUeQL+V33OIE4zck48e/7P2yZ4Sco", - "AAIJp+AJe/zokX+nnVorGu30F40MWDvguN/oPoEgKUbRZ8/bGmQbEh+jvQXP9Qrfp6puzLhN+tpk8CoM", - "R/5BOy+3mq64cJ4coCyq6AXohASGBDlHKn87fUy1fWqCFcA9Tg5jJuhs2ve/uwE/JfmqLuQPwKHioV3g", - "d7c6x9FMY+MZv3rr8A2ngP3OISA6g2Kmsk/z2Z+/9iVYpKYry73NNPB3s58+9bjG04/ek5EXn0ZZyNdS", - "XjR10EXGVTMGnCS2dffqbxsgEls5yaDh9DQXSIpleCOKEoCcxXtkVMP24oumUuADUszfJz9yJ2R7D2J9", - "h8Q5TRCP9HD23aPvjiT9yyHpJRDaHST9tF8ZZQp975s7txD4uE7JLkJ/FO4PI9xHOU7sLEt+7cic9+vJ", - "ZS8/noDsydwVrU5CAZZ0GGxveQhtImPiUPj6MTmxD1eLJz1AiF1q2/jq/aa2+1CCF/wvdrc8BjatpTew", - "Aj6IMqg4IcBR8xXJgsHN/lLhT6DEPeMr+1OJP4H5CJXnqbVrvhpfvIZuFf5jx5u0SHeTo4V0LWeLjcuc", - "kj6LtLT2RfJTfkpqiFRRCYN26oq7ogFj04cGBwEB8733YaDXO2DwDfYVxO9E/9pfWbQmrIhleMVOyBtH", - "aKgg714+J0+ePPneFXi1zCuiy9iCcUgMEI+BCwSjoCZ8nkJ+3r18DgCcBZvEpFY7DzVg1KFWDiN+eQv/", - "A+uK/5BK1M8pb+OqnZTpxDLMmLGdPQl5Ne5RKP2D6P+GxRNvnyJrpA6Jf+c6Ex71ir8rITRSXE2ySMft", - "x43S3VbbDdOHtlD8US2MR3H9RuL6gTWjvfs0zajVTX58NGz1grJvbdz6XVuGon06/dgllLstRN1M9knF", - "YdskbR1KsaN9cr2TJT0aZA5FdvYkNvdnmLmlOeZoy/hK2MgBETr1iTknUiJi208gR6/lSn8eknRktQ5j", - "GfnM2u8/qCoaQq2CTmeQjQs98lz8XBs6ljSIYCrONpXG3Tjm3dlbOZ75rubFdS+vJBYmGAklvEsWvZSr", - "zJP/feWJ13L1gqbTJX8NnD+S6ltwDtverJCaa6fSA1puc8LHoXZoOo56iOPjuMdr9RLs3mj29pnh/JVC", - "E1fIk7KdSrtmh57djj66Wtozmh1gvkZwMzaf/bbffAexxx74YQoEaRqVt82P+qLwangafHSD/h0ru+CQ", - "Tz/667lbweUyku12gLYNp0uTcdako2rrTlVb2tXrmUQL79HPGKY8kpujZu7L1sz1KeapK8+8UyOHrHcv", - "Zf/VWgJBiQvFb6WofrKjbHSUjQ4nG30GP9aj293v3e3uYHzeYRmgmF5PEgzfcMGB+P4D6d1RRgzFv9vX", - "6Cgl/pF4nn2iqjoWkThz91bR8RhYdQysOgZWHQOrjoFV92zNPoZAHUOgjrLY7zsEaorHik+kykWcRjgm", - "+a5Q6hiq37ETy2BRz2W14IK10oxfQZsvykh7UNAorhLrG0IVFu+lsGNdmZLlyPvqK5GGrM/zma+3SpXl", - "lKe8t53VeAAh53U0f1zUbK+1QcENUNgRH3qGuCzsPpflhhi4UgXU+gvJr+eWQd7IhlzBZSn5BfSHAvUY", - "z1ZhucFumi6oVdKMGrdd9yyUZ9mlA7x7A9IxXu8Yr3eM1/sDqDagBrs+/YhV21GBsNMIDp3GtBd/sx93", - "aSzwMuJ06QjkGKD71ZZuu0W4uGNowVeM8ZNUd5Gz5vZkSMFl86ivO+rrjvq6o77uqK87JkI6agGPWsCj", - "FvCoBTxqAY9awLvTAn5Ozd3XVhfgqBs86gaPmpI9g2M6VRQ/Wplod3gMseJjOShUn1IUxlg3JUbGCWXT", - "sxF+RSQk2q69Luv0y3mMJDmSly9FEQv17tWlv+vdsnvsmlZ1yaDiHqRqcP1Dwb5cVhU8VOEXN3L0iyNl", - "n3769D8BAAD//9X7UAtdHwEA", + "H4sIAAAAAAAC/+x9+2/cuNXov0LM/YAme0d29tHiboDiQ5ps0KBJG8TZLXDjXJQjcWa41pAqSdmezfX/", + "/oHnkBQlURqNPXaS7vyUeMTHIXl43ufw0yyXm0oKJoyePf00q6iiG2aYgr9onstamIwX9q+C6VzxynAp", + "Zk/9N6KN4mI1m8+4/bWiZj2bzwTdsKaN7T+fKfbvmitWzJ4aVbP5TOdrtqF2YLOtbGs30s3NfEaLQjGt", + "+7P+Q5RbwkVe1gUjRlGhaW4/aXLFzZqYNdfEdSZcECkYkUti1q3GZMlZWegTD/S/a6a2EdRu8mEQ57Pr", + "jJYrqagosqVUG2pmT2fPXL+bnZ/dDJmSJeuv8bncLLhgfkUsLCgcDjGSFGwJjdbUEAudXadvaCTRjKp8", + "TZZS7VgmAhGvlYl6M3v6YaaZKJiCk8sZv4T/LhVjv7HMULViZvZxnjq7pWEqM3yTWNord3KK6bo0mkBb", + "WOOKXzJBbK8T8qbWhiwYoYK8e/mcfP/99z8S3EbDCodwg6tqZo/XFE6hoIb5z1MO9d3L5zD/mVvg1Fa0", + "qkqeU7vu5PV51nwnr14MLaY9SAIhuTBsxRRuvNYsfVef2S8j0/iOuyaozTqzaDN8sO7Ga5JLseSrWrHC", + "YmOtGd5NXTFRcLEiF2w7eIRhmvu7gQu2lIpNxFJsfFA0jef/rHia10oxkW+zlWIUrs6aiv6WvHNbodey", + "LguyppewbroBHuD6EtsXz/mSlrXdIp4r+axcSU2o28GCLWldGuInJrUoLc2yozk8JFyTSslLXrBibsn4", + "1Zrna5JTjUNAO3LFy9Juf61ZMbTN6dXtQPPQycJ1q/2ABX25m9Gsa8dOsGu4CP3l/3TtrntRcPsTLQk3", + "bKOJrvM1odpBtZalvex6TiJKRkqZ05IU1FCijbQUYimVY91IPuaufyONkBwOsCCLbbelKFqj7+5j94dd", + "V6W0K1vSUrP0fvnVx5sEq4yZJC3LmSO9VmJwU2bhB1pVOoMVZ9pQw+I2VWVbCClYgpOGH6hSdGv/1mZr", + "xQWgEbPmdLK8lJplRu6QJLxwABsW8f54x/aSK8j7NSMwuf2AMhVgtrDkpiy3xLgDsAhBvBQxJ3xJtrIm", + "V3B1Sn4B/d1qLE5viD18OLKWyGPlxiHk7m1GArUXUpaMCkBtJ0Nm9vyGuVnp8RqbW8YFExSB0c1JwUoG", + "i2yQEH7VRsktLN6iwpzIyh66rE3/cojCDYufu3cFEGdQXI1XsmPRJd9w01/uG3rNN/WGiHqzYMoeuOd8", + "RhLFTK0EHLZiJIczW7RufkVXTBNmGSNHWRvmsYRLSEMUo/l6mCohTDsI0YZeZ0rWopggUhoiVcyydcVy", + "vuSsIGGUIViaaXbBw8V+8DSCbgSOH2QQnDDLDnAEu04cq72e9gscUHSqJ+Rnxzvgq5EXTAQWg8SSkUqx", + "Sy5rHToNwAhTjytzQhqWVYot+XUfyDO3HZZCYBvH4DZOusqlMJQLVljeB0BLw5DaDMIUTbivCLmgmv3p", + "hyH5qfmq2AXbJoluFwFwOUFnXdsv2Hd8FWGGHZd6Ih4ij43xbxT3JuEdNMqQbCRkJPvVEZW0faDVf4KF", + "IJ4btdPsTpYCHMOzt6Gt6Mx0f0qJ5qsMR+zdEr56b3nxkpfAp3+1l8OfbK0tX2qfrefcmq8ENbViT8/F", + "N/YvkpEzQ0VBVWF/2eBPb+rS8DO+sj+V+NNrueL5GV8NbYqHNWk5gG4b/MeOl7YUmOuw3NQU/nNqhora", + "hhdsq5idg+ZL+Od6CYhEl+o3lL3KoZlTavJrKS/qKt7JvGU2WmzJqxdDWAJDjhFCIBq6kkIzQNdnKEG8", + "c7/ZnyytYwJIeSQEnP6qJaggzdiVkhVThrPYTGf/+1+KLWdPZ//rtDHrnWI3feombLQ+M8TD8OZS42gX", + "0ixHzVAK2FS1QZ6eIgvhHn8IsHXnbI5FLn5lucENaoPxiG0qs31sAXaw68Ptlm6J8xP3rSuS3+M+IlfP", + "gDv3R/5ZO7WpoisuYOFzcrVmgmzohSUHVEizZorYs2DaeP6OdA9ZfrAvOiHBSdons9SNSZypvvOhNqf2", + "2sq5ZyDnHuKIO0rXHmedAul48uHkext7SBRYHejsRw2v5+cfaFXx4vr8/GNL1eKiYNfp87jXwy7lKiuo", + "obfD0dUL2zWBoF8yDrWN2odCoMMizx6n8LAc9VDbdeDLdisae6SsiVtxd6KqNTN/oSUV+UHY6cINNfmE", + "33DBAYi/oo3reMz+mMNWHuKI3e4e5CKjvXryFT4ebuoOBy/AnY/2UEc66SAfWCOEKQ+xSZ8L8Y8Yf1iM", + "/0sp84tbneXYUcGoO2b+SSmpDoBFXn7vrHo+2zCt6YqlDePxTvqGU7bOAwzHzuwSwHz4V0ZLs36+Zvew", + "mdHYO7b0fWMwO8DG3uu1imx7u9YfrWqHQN4eds+bEE2jv/Td+3KIUmvLp9Py1pl2Kfr0M9b7HfKNtxHH", + "RuBEyJYLr+QCPQVcCntS1EUgoe/mXJyLF2zJBbhin54LS4dOF1TzXJ/WmimnBJysJHlK3JAvqKHnYjbv", + "MsIhRwoEmThoqnpR8pxcsG3qFDD6JW1yKVfy/PwjMdLQMnI0RzExzr3XGIz7KIcTZBYzZG0yF0uWKXZF", + "VZEAXQfnIoyMwTljs86JGxt9oC5WzY2fvga9AI8Bi1PZsTfpRBwMF+1AFXu+f5fGeQ3pFUH8IrVmmvxr", + "Q6sPXJiPJDuvnzz5npFnVdUYLf/VRNVYoMFtcVALKCwczjNj10bRDOIAkss3jFZw+mtGdL2BkJKyJNCt", + "Hbyj5ErRjQsp6IYFjRwAwjGNl0UrhMWdYa+beSQM9k/QfoIjhDZkzcp+YNG+5xVpUbc+rh2a2EjM5vn5", + "BwjH9CcTAoRWlAvtuYLmK2EvgYt0WzCSWymAFSfk1ZIAVZu3urt4a0cxA+ngGoPTyHu7RvB8k5wKCFqr", + "CggT4oJQse263DQzxjs437ELtn0fOc73dMC6KBu6gyUWtR0usMXmhMkV1WQjwfmaM2HKrQvcSaBmGpia", + "C4MRBK0wsAGiAbcmis+yFycmIQMRblG4Eq0qsirlwlGagKJPA476PsNE5a0FQB+AoCQVp3bEXHojqEps", + "BF7EoSC//Rdqx7vTNRxd3q1RbsmVhqAwRh2PoPEVuQXmuYi1Pij/XDOQyqSCyK02Sml/pVNIHwJS5rOK", + "KsNzXk2zouPob1t97CC7WHuSmctll2f3WGqShWDjbEF1mn0z+8ViYK0xmtGu0RM6PxNKy7CCEwLRJ+6q", + "LkoIcAyh8XjGVEHkpV82hooPgZa+F0yJRqbyYLR3JBbe1lT7IEyIJPYkYpKYM4C87+0GAALbexNhbyy3", + "cjtvyS7p0P4PB768EoWlHUy3A1JDWItnK/24YB8/hilAPvzFx7z4QBf7r8X2uiwJX5JaXAh5ZYXjfUJZ", + "5jMr+dXpQ5ICJD9751a4HdjYo48D+A86OjYL1T+Wy5ILRjLCwx4Y2AMM+pY5x9ja5n66OZhVDL4hFgft", + "AJNHSCF3BHYlZYkDk7/L+MaK1T5ACsaBxlA/NhCb6G+W1vBAwANZDwNpuUhjY+7pgpUwW8wSAINI/QVj", + "AuNxCRdzYvW8S1paacVIFF7CIOm49UctUduJefrxkByftj7gioCL7bUm5Hu3WU0sLHqg05LsCMTjckvq", + "CDTsF0oRzV6NROfvnHpAVhjaq0ew8DsA0DV7hlBAp/LuVE37HK0h7fMm2BLJSBrbhzAmeS4DO9a3VITQ", + "qrddtp20R7RaEWyycPp1JJ6lSLK9FbkUmgldQ0qLkbksT3qGCM1KBpJN1pIksgu2TeswDAjsme8WGSnI", + "I760KsXjSHRRbMW1Ya20kxAJ2wT6biFVo6LGMGUn+n+P/vvph2fZ/6XZb0+yH//36cdPP9w8/qb343c3", + "f/7z/2//9P3Nnx//93/NBrgGyyol5XJ4daZSS7u+d1IGqgwdCXRsLfPBV3ApDctAQM0uaTkQbmMbvdSg", + "PL8EWTYpMLQOm2DWFB8wPcK0F2ybFbys0/jq5v3bCzvt34O9SdeLC7YFsZDRfE0W1ORrkBtb09s2I1OX", + "dOeCX+OCX9ODrXfabbBN7cTKokt7jq/kXnRo7Rg5SCBgCjn6pza4pSMEElj9C1aip2c4mxcvZ2EbnoxZ", + "WXuXqfBjjylMERTDXAlHSq6lHeA0vAqIhoPMJG6iNCzdW9FUBRes/8gPommuaNDg712RjVcXK7NulLQ2", + "6z7eYXn94acu71Dhi3B6+9hpUFLqIRhcHDfYDuSKTMf9ZAYrJHvzN96WSFXAXEURr61/jZpsuWkH40UQ", + "l7wn68BKO9PcGwKyhCqBa0/hIlkquYGb1xdKI+TkAxp5CwUbltOZ1dWG6OOLJZ6Qs7zTg8Zo+Te2/cW2", + "hVO1vb1gOvXKNAYKr8M4teVuR3M3X0AK892IOzEfQ3KH0B6qCKBBtuXb2/MGlHKVtjeUK5A75KpJ+YrR", + "YcGs7seuWV6bJtuvY08MJs+HlSa7ttN0lk7ktsWSFuPyA2yUG2vH0b0NdPI+T45WlZKXtMycs2uIxit5", + "6Wg8NPe+sQcWx9LX7P1Pz16/deCDW4VRlQV1ZnBV0K76alZl5RKpBkisT4lfUxMsCV3+75xdXLccZFeQ", + "Sd3RmK2k5ZALCXTj/Ixur3OYLb1cvqf7y/lpcYkj/lpWBXdtY2dHb23bQ0svKS+9gdtDm2YquLjGR743", + "X4kHuLOnN3LYZwflFL3bnb4dOyhRPMNIyvQGE/c1kS41Oui5oNyCtRwQdEO3Fm/QPNknSaLeZPbSZbrk", + "edoFIhbaooRA771tTKDxgJpsR7S8OD1WzaOxbDM9wejWATKaI7mZPvp1aO8W0oUX1YL/u2aEF0wY+0nB", + "XexcT3sbfVGWW6tACR8fFm95QCUIJtxH/XFlLO60uDDKbZQgq9f0J3Wn5tYTzu4u+k9jI+7LfwDEuPIT", + "B2L0wH0RLKUei4LdnYqWz3qPeK54xp6UMRKL5S6fIxW14M4LcIvT2V1zzCtartxJmlzspUfF1VPupD3p", + "bKnkbyxtPQSj61V/+mhi7J0efLIW1Lk3A9oQ75RUusVRhfozdwUpaM93BqrLO4MzpSlI1xzS4KUbEttj", + "p087EnCAsMP9i+JNQEH13lAq8MI9h8J2LY0pfW3jENFTHL+5tg7mvl2DXi1ofpGWni1Mz5ooq5bf1kji", + "O4fKQu1TOiFRwFZo64r0VExtuGmzgUYxu60kjNNOloEbkRewKhZ2XZ2vUsvEMLW4osL4UkuOoLnemqHn", + "yfa6kkobqJyWXGXBcr6hZVokLmD337eErIKvOBZJqjWLSvy4gUgluTCIRQXXVUm3GMfWbM2rJXkyj6ia", + "O42CX3LNFyWDFt9iiwXVIKw0pivfxS6PCbPW0Py7Cc3XtSgUK8zaVZ/SkgRtBSw/IXxiwcwVY4I8gXbf", + "/kgeQeCI5pfssd1FJ4LOnn77I5RFwj+epIk8FLsbI7oFUF1P9NN4DJEzOIZln27UNBXGcqXD9H3kNmHX", + "KXcJWjqWsPsubaigK5YOx9zsgAn7wmmCF6uzL6LAAm4gbBFu0vMzQy19ytZUr9PyAYJBcrnZcLNxgQRa", + "biw+NSVmcFI/HFaDQwof4PIfIUqnImm73sPamLBaS2rVEEv1d7ph7W2dE6qJri3Mjb3MEcQT4qosFUSK", + "chtZNGFv7FwgoFhhE+zOS1IpLgxozLVZZv+H5GuqaG7J38kQuNniTz/0Qf4LlKIiTOTSzi/2A/zB910x", + "zdRleuvVANp7Ucv1JY+EFNnGUpTisaPy7Vs5GDiUjkr3FL2blDA+9FR5y46SDaJb3UI3GlHqOyGeGBnw", + "jqgY1rMXPu69sgfHzFql0YPW9oR+fvfaSRkbqVjb8LvwiSIteUUxozi7hAD59CHZMe94FqqcdAp3gf7z", + "uv29yBmJZf4upxQBTPbsb4f9OV72kIot5cUFYxUXq9OF7YOiOo7aFdJXTDDN9TADXa0t5tjPluVFFhEY", + "mixYKcVKPzyme8AH/MorBjTp1YtdUPcG9sUiM2g6vDG2nZ3irS8uiUPb9p+DI4XI6p1pxO9c2+FAaMvG", + "MJXmuUt8waiftgcW13tFwU7ORIFiHZC/NeViIDqasWIg8o3BjGdSGY6xJ4x9hjg2wzdMG7qp0mwWDMd4", + "E+FWW0BDF6uNaJZLUWiiucgZYZXU6135ugN5ZtcCJiu5RpYTl33MpcL6eyBTGNnJpZya6TGaNdqGMVNS", + "miFAQfiI032lNITWZs2ECZHUDCohd1eCuSCgcSBDQZJF3lga7ysX0rLczgk3f8BxlAsfpGTD1EXJiFGM", + "kau11IyUjF6ypig5jPYHTd5f80JDyfGSXfNcrhSt1jwnUhVMnZCXzrsMWhB2cvM9OSEuC85Fgr+/FrC8", + "QjJUkeJ14jJ9QH/wZcQrniMD7f4MtaI1Ky+ZPiHvryQCoZvMYW2FkFaPRW0wg6bgyyWDewrLAeUJ+jUf", + "IpigvDoEW4dh3Zo+w227FhnIxwNKpEFLxbV4jo2ISztpO4g6V2ODGqtHqJIVK6bmaEiFbecb1mSKW9lN", + "KtMYbJYMszEsZePCKFnUOcP85LMWPkZg8R5IoSJx5OEHHPLV7Rs4vbHF01SrkIOA+wTFLCHbK4SzY5dM", + "YbR8M9AjJDoRXNpQBaERECnhlsqKx2niXFcrRQs2za8JRPBn7BHyav0Il3K/AX6x7btiU0s2aXH8NJeO", + "Ys8tl4lpeYqWDYpe74bSlF5iWXjFSswUgYri0HbeE6yWjGWai7T1c8kY0Haa56yy6By/58OYJVQoxAKp", + "gMRWz1vtCQvDLxnmsIwIA1lOy7wuMR50hNNf5bRUbTdKyZZGWgSLn3loTILczrWAeFQsxY3zKUsAox5Q", + "0eOSqa1rgdqTr3xtL4fq+P77uWJZyS5ZWqdhFFPG/iqvyIaKbTgLO0UDxjxKLAmQo6wCjmU87Z+dYheB", + "j5fJYd04kPYoBja3iM+5YorLgueEi1+Zu82BLHmMwRL6Uhguanh5QLEGbuQTBLLfuhlufQxQQzn89kM7", + "mFywq9ZpF5E81w691oZeMATb5+k51jj1TBXTvKgHTJmK5m3I9kNGd3nfUcNOVThafSC87FCocMnHLl0X", + "lzto0zmt/i4N0qkW8Z1CrGjIVCGOUCeiUV1xEN9yQPeRRnqLk0+OD2NfMqXbcY6RDZBd7xjbtmiNjyVT", + "lET7wv6zZD6MRQ/Ot0Vy3OCcF74wuxX6MxdHkdjBgXoyAQB9xU2+zgZSO2xbbIGpMR1Nqz8lihBwC9ly", + "yXIzBQbIEcCXJAahwM8WiheMFpBw2aR7YKJHF5RHf5fEDq0juUZoDlJoI9bAKI/3qAcaMGQX8v8iJ+L+", + "pYT/gYt0wjXwgow7+7TZE9s45GmyeynZMg27EqJWoztSSU3LtIfHT1qwkm7HpoQG7UmDYOudXMhzqOVh", + "lqFglGw6/Dia2t2zscltk+6Cw/Xs34q4Un33JH+6pOVAFso7VimmrcBIKHn/07PXzpc3lIuSD6ZOUeNy", + "lg0lg2UGbuag8KRJBIaLwXf3/lXSjjkUIoYRYvZzr/ftQguGynFFG+ojDvsA/c0HxJOKcueobhJx+jvr", + "krP66XJTguqbA+4uwqU8wSCplcRF2vrREGQNn7F8C/FPEfSBH6xlVyyyEO+ZeotkPnO16OICXDuDvLnO", + "NnylgOikRx2uoRdZ4xJJc8jsEq9iOcIyzA07+95aeAfiBrxGlfIzp86oVxo3cVCab6oSnaxuqF7y9l6J", + "ZU0s2P2HFh46LuveI6vYrR18hw+oui0su9PTx8Oo/iGey01VsmF+UKF7HB+HQ84JBTmiZ8C8qUXmea0a", + "G1w3UOoXWnJ8n0ZDUQ4hZQVVOCrDhf0P5GjJ2uD/GVX2P1giqv0/xKqoVocdagbnAmnxfiAfgj2zLLtA", + "hcH1TdXyuGWe5yTjcZ/XJCjiaPB3i8fDyZRo8m4C2u2thC8r+BLHzRMEBII1tP9Lk4IZpjZWdl3LK7Kp", + "8zWEitMV85HjEIEChtPORK3RfTBdOwPCOR91RXMcCAOUSqpWTBEXM0RcZewQeLShvPPwVzcsAFRZmuK/", + "u+LZ+w/egbQURbUnwuY9GBdse4rCAPx+C8IxHBw/ABiEyN8jSHeKtI+TNXbg60VLjsJ6b638lgD+AeUp", + "C5+7a3vKU/00lKnLg3XAdag1669zurMp3tsEqWjWNlUZ6G/usAxvFlNk+HThJtsdlAjcECimRgBU8q9v", + "/0UUW7r3Rr/5Bib45pu5a/qv79qfLeJ9801aA3so9QH3yI3h5k1iTLuicPc1ViBoGirTuOdSc7nZSAGG", + "prLsePlEQSDuScP7qYIwcclKWbFka9zg6NAhv0WxVV1S9G5xIZhqdZoSuKz5SrDCXAuMiDiDP99fi1Tb", + "mNVD62g7UhVno4dgbleKuVNaEMPG8eXx247YhHg3I/pH728/4kuMQw0jwlBLpu4y5ns3xoQqnyuhMJ8P", + "A7G5D0sCIQ1PuPNEog9V8tU/fcB18OCyf9e0dB5qAf7g9xB0nF8wgYU9w5vvRhImdK2cQ9jCCuNZUNww", + "Mmbwumly2xKf2VjZPAXG8mCHd2FoEECPXa3oUdjDkeNFqWx7LlbZSK5NDsk2rqFPpgQL12gFRzu4RUK1", + "YcXEJPrYHwYJZb7/wPBNtajmNaZ0qlX0RKvol5wgj169eEygnsxQZY/oxc3dy44LVk2DCGMbe7B0U+v2", + "gWLJ2JATshO3QZZswJ69qyzS8rKpiAStuobjnVBODET7K9VQ4sg1dw7zLzT6rAWke26zP1ScCrx32Zz5", + "bKVknQ5WWmF6eieMEhQDELowhEav6R+//e70uz/+iRR8xbQ5If+EXCFkvv1iiO3TJLwpstiq5UoAsJB/", + "ivKQi5OI5ly7A+3Fw3AXLwHDPPwJ36Zaw3wGcklmrlMxXa96MgupXHAJpE5G9KZlrD9EJBcXRlEkvplc", + "LpPpxP+A3xtTkvI0WbH+qU+gyvig7S2lgr/ha7g389mO+mTlZShNdjvCU7KhSrnldeL6fP9d1tygE/La", + "9iZMLKWymvamNlYGgAf8va2zJaVCro1pqoZDmo34jSkJhgRBpMhZjwfyaLMhNoTmIM9rF+BkYQh5wyEK", + "/dEZSDNzBPIx6qn9q0ZqYTiKP3Ybf4l2sbKMxwL9zzUvE1hQSftdx3DMiZAE38OIW2IkX5MzhjC7OO0W", + "Ij3sNY9rJxRpO5nFhALr0DQlhxorRb6moinwv7tATR8n93m4t037u9f8kIV0RuD8vJV0hBwIahGuXKBV", + "UCB7K1jUHhbgim43TJhbUr632BvjZaDAtRrXANSABuB77yoXPvT2vx3bfgzZw0HVAtspUttojfMBvSdE", + "BvinERrZFW+QFRGWNcRcRmGq3nbqVLpgg79gW6K8aSCuzNo8fL+nloVs0fBUdtN7vmGNXoKCXEoE4pNY", + "IqqXab0WA+6RZP9hZDlhmHGs0ANYgX3HcSKcwh5oexb6tB/D71vSthVrhw+0qqG342VBxz8hL0IcM/ha", + "MKKvCW5G+1PXI4PZwCE5mytvp6LK25zBaXN+/qHCaIrExXUNUJaxbfpSjWtC8+UqvKmSMNz4ZtdLppp2", + "KeOJb7lUvzUN+3Yb36z/HE+L8jQupYpuZ14sm81nFmD7jwXI/rtUv83gBZqy70pK3yF3zBlMkIiNm7UV", + "x5YsFy5Dgy07jJCj5T5dxA84bSLGtq+FMLZrY4GD5ofntCzfXwucKRGA0rzEn3I5YgVdl8sRiKSlpM7r", + "6A1H7oLGDhKa51bKKppY0QjOP2jSrdOEEaT9Sk0tJr4nkUy8mBTQjarV4LrBZtSXBHlOqFrVG7Tp3//6", + "dqxgsDopL1waWb/EppOa8KbXihVEKpdAwpcuO2ioRszEunn40tRrueJ5I5014asDmD63+gerXLUGKbI8", + "OMQtq7JKnpHkHB3J57MT8gqDzRWjBdJMxQ1LVXBrrR8yX68YVKb3GJ2F043qc57YW9SqkKcBsxWDB6US", + "NRu/1pqAtNL1wIkNUSUUbNqH9BlO6LmdqanFjoeUUyGk+YrOac+agJ0n9aLwj6oKxQFLJvzLjij6wrAD", + "ZlKpGF+JsWewltQzAt09riQ7aFMpl+QWH7zucYkgEd+OiILzAwfD125okUlRblPUNU5o7JDXsBejb2GF", + "FEfdhAxpt8qoms60JXoy8zZaISA2aM1vD7u+W5RwvHPdxs4ALaqxq28rLmrkxX7Mr2oPvUsyixyNo5IZ", + "lnYp7cKRPimWef7pKZYosOpL3YRZnYtn5DempNMXw1D2QjTmaZf677JyTxKdQokm3evWnXLPEli4+BHp", + "cLC03Pn5h2vakzIApjvIF7erErjzjF8OlCCKz9h7q1zNoTvWFsMZRzZ26KXX8/MPS1oUnWoscegVEplQ", + "TQR329ViAmShVwNlj0ZPczl6miPjt1I3rrzCN/Ial1cQMUnmyu849kiFow6HVjY16vpTT7n8wX8/CTW8", + "0ntX5PCzjqDHSOVIugGd7FkoCuyAkwG+E+JIiPN1+9+VN6WUS0/NvHvMO3A7z6HhE/9kQ6uD1qXcSTwi", + "iIfd/mzQ6d8kRPmHutx4Ua0HGKCJLug+una31x396OkThK/dNBgaF4JpHnpVbAM5XI2KmTgcV0AuiIVN", + "ZT8MpIC4hzg0XEczxHtNyCs7Mi2v6FZ7U2mDWMPD+V3FijEJM12c5In23fTeqBwcY+9YzisOb9e2qWDA", + "8WED48DbwWiotEQHs8/4ZTBauNhw2pRkbDu/vO/LFZejEYOeu22mZdtagAN7Y7Bt89yP7VcUjjTiZ7sT", + "IVIFOsOW7qB5zjs5SuycpXBfGoe9kMjhNMPUTXQfEhpwiwjbyB7aG6ouWjyQ6va7nZgE0Rq1JWJEqQu3", + "eNrMORPeNm83QSh2MO3/whQ6MN9RUcgNeVkLxIJHv7x7+di95++RzJc9sMjnIPlCXz2r1NKt/Kzz1pmP", + "REefxoproxJ2yy/3JbRl/yW0xHtgdnWHegPtovhMb6CVvTfQbr/S6a+f+Rsz9PbZF4lAOzQJ7+Acp57O", + "F7Mv+XTdkH66mW4nHqJ02CQvRAUE7Hn6elcdxn8nIav11jE15MpKHzp+/DQR1NlUDxYhNjPyI+wM+myP", + "N/DUiZOzYBIoeph4Ile7p5c9b4ke2ceXqrDqcRkJP8taFLqzhc3rGyMe0FHZx4k+vs2oM3VIKJgqCZzF", + "rtI2JOCKdKkg4Ynn7gM7UIkWa87CM9v4wnO3jFSzlZWSl7xIvXtRyhXPNVpg9vXZvvZ9b+azTV0afstx", + "3vi+6EROs0O+cqxQFFQVhBXf/fGP3/7YLPcLI1f9TUoG2LhlOSMjNTxvy7FhdROImD/Kk5Xsk6xBX5ta", + "Na6H4FubQ+3sJn5tPxcZAJJeb7RYH6Kx2BIaobq0YntpePPT3P62pnrdkM6o/jnUpafE0atu3B1k/Xye", + "B5aiS5HdKTSicz2GCEdzSb6Eu9F5f4znk0nim4iS9MuDuyWi2dXii0+FhL2uSmZlu4YG9u9NrraVkaf+", + "aJDl+znPeP8ZkXi89K5DA6h3Kq0kggUSrDDZSFxgIGigukV8bm9/zmK4UmUY14ppC1E6nmatzs8/poXN", + "oaoBVrpMd7rZ82zPOnva3nHct0EJt7pAIB5YZxvHgYcHqb/nNxCyvQRpLJfC0BzkRizAPXvmDGYzV+95", + "tjam0k9PT6+urk68Ne0kl5vTFaSdZEbW+frUD4QvIcWJ4K6Lq5RoqXC5NTzX5NnbVyAzcVMyiGAv2DVY", + "7QJmzb47eYL1A5igFZ89nX1/8uTkW9yxNSDBKdbqmD39dDOfnV5+dxqHyqySD1sxqvI1KgKu7QnkwjPU", + "bl4VodFLqZ754ZzbA5/2ffph6BEfe2Xt3/+umdrOfBX92AzUOOP612N3ljOaKTSGYJpaYd64gtf8UYiL", + "PM349By7ZIJwFPtKvuHh8QxllVrHtRMwQ9s9AW5KatEVi+A9IT9rFtWtlBeQBILipg8p92UXQ6cBwOwQ", + "KbgalO+n+OKuOVEXIvyo8Jb3FaQ9gdNERKGjJ62acM5U6x7RcCU78i2pRWnlC+9+AK+hDkuDcoFYzSKn", + "bgdcvpWPW9XDJ+AnyRyEmYVwzxNxldVBNwJm4iJtwcrlVCeH4/NQfiSOG5g3j5I5Q/2chIIeHQvz3Pn9", + "/ZvD/ad8MapgaMEuCDijZZlaZuRr6i7zp2u3zAb7cbW6ztcQodIFtPfIMLy/4coFNA/s4N7MXf8oasAn", + "y4VogdBStDZwQh+7Hey6KmXBZk+XtNQsvT0MF9namiAg+EBM3DsXGNFJE9QYg6mzKDpg1kpxtC2EFOmC", + "H13ZQZstkG7Lz2b73rrSPYX0hV45O8Wd7puPwoxc7O79I1gv1F21l/CCbYeAaZKVh6ndztjK8c9D4Hs+", + "4yMbmtdssHwlFLWumIIhRQ7OFQ3UwlsuEed9cE3BNV2U8MADmiVakRmDBCHU3N3jBOLSM8MMuRuTMjLD", + "R3iUBYqAwU377skTLzs5U2M02umvGoXiZsDhWN59knNSF9BXNBxNfA7FqNEHhud6hTLDpqrNcJzAtcmA", + "U/dH/lk7IlnRFRcuugYMeBt6AXY6gWlaLrjN306f527Zf/DMOIHBYcwEO1ojk7U34GNS1m1D/giCXB7b", + "Bf5wp3McrP42XIWtsw7fcArY7xwCYoAuVo+7mc/++LUvwSI1XVmJeqZB5p59vOlI8qeffHQpL24GxfrX", + "Ul7UVbAPxy+Z9KR7bOvu1V+2QCRGpftgdfY0F0iKVUIiihKAnMV7ZFTN9pJVp1LgA1LMo4x4lBEfRka8", + "F1a6BwO9R4aZZlJHHjX74ckPRzb75bDZEpjfDjZ72qMAu/iuiGLeunRUVkhuy63zv4c0ESxbMsKdn1UV", + "ZMZDcIv+kvj0wdWM3ytbPho5b2XkPDAr7dz3PdTTZpbmph6V1Sj5pbOxR4ngKBF8jRJBSLX7LHKAV02+", + "HP5/Lx6/I88/8vwH4/nhRk9j9PFjDEf+7vl7MKIcmfqRqX9tTD1R3HY/Fu+tlWlj5p1Y/nMc+lkM2lH/", + "P8oCR1ngfvT/FgHYV/U/CgSJahdHseAoFnzdYsH+On8QCDq+0IOIAkcjwJHxHxn/ZzcCHJn9Ufs/svmv", + "n83HeaFTA+u6uf8jXPt9PPwOpn3kBYfJdIke/LGzLPm1o7i+yE0uO49FCnhKnLNycHsElJWAwfYORMcE", + "4aE49PD1U3JiX7s5nvQA9aZT28ZX77eV3YcSgvB+tbvlMbBuyh6EGExfUTzk+0G1b81XJAvZ5/aXDf4E", + "GY1nfGV/KvEnyKXGTNLU2jVfDS9eQ7cN/mPHm7RId5OjhbTTyBdbJ4mnzyItxn6Rgax+SmqI1TSWmN0V", + "T73hIhudPjQ4CAgLtpQunSWCgV7vgME32DcD4l61Er+yaE0rbimv4Rt2Qt44QkMFeffyOfn+++9/JHjh", + "rZaC6DK0YBwSX0uIgQsEo6AmfJ5Cft69fA4AnIX41Emtdh5qwKhDrRxG/PIW/jtOnPxdZq99zkQHXLUz", + "JzgNEZ+PGRdPwiMzo9aHw2rNvxNtdz7rqgh3fy+uo/W0d7Iz4TGh6z9KCZ3iZI7LM7Q9KUMVGvbwD9+/", + "z/YlKBCoP7Sq74dLhxJDqMHaFA9LEnRsdjvB+2g+PpoMjn7j36Pf+D86LTjap9NPbWK9Oz04eoJryHjZ", + "NEmnBqdE4i7L2CkW/+68f/dGdvYkNg+XAXpHl9DRn/KViLI9InTqX8qdSImIbT+BHL2WK/15SNJR1DqM", + "d+YzW+B/p+ZwqH0c7Eq95/GwHJMraD2ujrm3cZu3be6nKtO98crhpygrXlx3HnolXBTseqC2932K6KVc", + "ZZ7875+CunpB0++Xfw2SP5LqO0gOYzxrPJgvNrxAy7GqmJMC8Y52iCNz3INbtUxn7qnGhzOa7Z7djj64", + "Wtpx3B1gvlpwMzSf/TZ7+EjVY+jhMfTwqGc+pLELDvn0k7+euw1c7onA3dXvbMPp2mT8jNnRtHWvpi0g", + "c1Np4QMWNIMpj+TmaJn7si1zXYp5uqAlFTnbaZFD0Vvja62+evPVWgJBcbUYgcCMUlQ/2VE3OupGxwcM", + "jnF4U+PwDiZ0HVYaiYnnJC3tDRf8WCkmxfUWDWs4qmy/JwFknzSrlnsiftd+VI87ZlodM62OmVbHTKtj", + "ptUDu5aPOVHHnKijLvafnRM1JXzEPzPMRfzIdkzyge8Pih/3HVHSW9RzuVlwwRptxq+gebnLSHtQ0Ahe", + "YXd82Dc0kugQMrBjXZmS5QB/hYia+E30+WypGPuNZYYqKylP4bet1XgA4Y2VaP74kZW91malYbSeEZ+L", + "hrgs7D6X5ZaYUPSI0PA0/NwKyFtZkyu4LCW/gP7ugRa76RtikbjzYJqRxOr4Qzvqumf4uv6urLf5Q3hz", + "jgl8xwS+YwLf78C0sShlfqFPP8FRZ2hA2OmRhk5D1ou/2I+7LBZ4GXG6dEpyDNDDWkvHbhEu7hjn/xVj", + "/CTTXRQ5OV4dKcRPHu11R3vd0V53tNcd7XXHykhHK+DRCni0Ah6tgEcr4NEKeH9WwM9pubv/p1OOtsGj", + "bfBoKfmsmSrx0Z5+sjrR7lwVYtXHssUhhwyFMdZNSVhxStn08oRfEQmJtmuvyzr9ch7TOo7k5UsxxN7M", + "Z5qpS3/Xa1XOns7WxlT66ekpu6abqmQnudycQt0E1/9TkPvlZgOMKvziRo5+caTs5uPN/wQAAP//N7du", + "Y+VJAQA=", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/api/generated/v2/types.go b/api/generated/v2/types.go index 6b6a120fe..298972d96 100644 --- a/api/generated/v2/types.go +++ b/api/generated/v2/types.go @@ -84,6 +84,18 @@ type Account struct { // * Online - indicates that the associated account used as part of the delegation pool. // * NotParticipating - indicates that the associated account is neither a delegator nor a delegate. Status string `json:"status"` + + // The count of all applications that have been opted in, equivalent to the count of application local data (AppLocalState objects) stored in this account. + TotalAppsOptedIn uint64 `json:"total-apps-opted-in"` + + // The count of all assets that have been opted in, equivalent to the count of AssetHolding objects held by this account. + TotalAssetsOptedIn uint64 `json:"total-assets-opted-in"` + + // The count of all apps (AppParams objects) created by this account. + TotalCreatedApps uint64 `json:"total-created-apps"` + + // The count of all assets (AssetParams objects) created by this account. + TotalCreatedAssets uint64 `json:"total-created-assets"` } // AccountParticipation defines model for AccountParticipation. @@ -235,9 +247,6 @@ type AssetHolding struct { // Asset ID of the holding. AssetId uint64 `json:"asset-id"` - // Address that created this asset. This is the address where the parameters for this asset can be found, and also the address where unwanted asset units can be sent in the worst case. - Creator string `json:"creator"` - // Whether or not the asset holding is currently deleted from its account. Deleted *bool `json:"deleted,omitempty"` @@ -847,6 +856,9 @@ type CurrencyGreaterThan uint64 // CurrencyLessThan defines model for currency-less-than. type CurrencyLessThan uint64 +// Exclude defines model for exclude. +type Exclude []string + // ExcludeCloseTo defines model for exclude-close-to. type ExcludeCloseTo bool @@ -913,6 +925,17 @@ type AccountsResponse struct { NextToken *string `json:"next-token,omitempty"` } +// ApplicationLocalStatesResponse defines model for ApplicationLocalStatesResponse. +type ApplicationLocalStatesResponse struct { + AppsLocalStates []ApplicationLocalState `json:"apps-local-states"` + + // Round at which the results were computed. + CurrentRound uint64 `json:"current-round"` + + // Used for pagination, when making another request provide this token with the next parameter. + NextToken *string `json:"next-token,omitempty"` +} + // ApplicationLogsResponse defines model for ApplicationLogsResponse. type ApplicationLogsResponse struct { @@ -959,6 +982,17 @@ type AssetBalancesResponse struct { NextToken *string `json:"next-token,omitempty"` } +// AssetHoldingsResponse defines model for AssetHoldingsResponse. +type AssetHoldingsResponse struct { + Assets []AssetHolding `json:"assets"` + + // Round at which the results were computed. + CurrentRound uint64 `json:"current-round"` + + // Used for pagination, when making another request provide this token with the next parameter. + NextToken *string `json:"next-token,omitempty"` +} + // AssetResponse defines model for AssetResponse. type AssetResponse struct { @@ -1035,6 +1069,9 @@ type SearchForAccountsParams struct { // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. IncludeAll *bool `json:"include-all,omitempty"` + // Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account. + Exclude *[]string `json:"exclude,omitempty"` + // Results should have an amount less than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used. CurrencyLessThan *uint64 `json:"currency-less-than,omitempty"` @@ -1056,6 +1093,73 @@ type LookupAccountByIDParams struct { // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. IncludeAll *bool `json:"include-all,omitempty"` + + // Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account. + Exclude *[]string `json:"exclude,omitempty"` +} + +// LookupAccountAppLocalStatesParams defines parameters for LookupAccountAppLocalStates. +type LookupAccountAppLocalStatesParams struct { + + // Application ID + ApplicationId *uint64 `json:"application-id,omitempty"` + + // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. + IncludeAll *bool `json:"include-all,omitempty"` + + // Maximum number of results to return. There could be additional pages even if the limit is not reached. + Limit *uint64 `json:"limit,omitempty"` + + // The next page of results. Use the next token provided by the previous results. + Next *string `json:"next,omitempty"` +} + +// LookupAccountAssetsParams defines parameters for LookupAccountAssets. +type LookupAccountAssetsParams struct { + + // Asset ID + AssetId *uint64 `json:"asset-id,omitempty"` + + // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. + IncludeAll *bool `json:"include-all,omitempty"` + + // Maximum number of results to return. There could be additional pages even if the limit is not reached. + Limit *uint64 `json:"limit,omitempty"` + + // The next page of results. Use the next token provided by the previous results. + Next *string `json:"next,omitempty"` +} + +// LookupAccountCreatedApplicationsParams defines parameters for LookupAccountCreatedApplications. +type LookupAccountCreatedApplicationsParams struct { + + // Application ID + ApplicationId *uint64 `json:"application-id,omitempty"` + + // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. + IncludeAll *bool `json:"include-all,omitempty"` + + // Maximum number of results to return. There could be additional pages even if the limit is not reached. + Limit *uint64 `json:"limit,omitempty"` + + // The next page of results. Use the next token provided by the previous results. + Next *string `json:"next,omitempty"` +} + +// LookupAccountCreatedAssetsParams defines parameters for LookupAccountCreatedAssets. +type LookupAccountCreatedAssetsParams struct { + + // Asset ID + AssetId *uint64 `json:"asset-id,omitempty"` + + // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. + IncludeAll *bool `json:"include-all,omitempty"` + + // Maximum number of results to return. There could be additional pages even if the limit is not reached. + Limit *uint64 `json:"limit,omitempty"` + + // The next page of results. Use the next token provided by the previous results. + Next *string `json:"next,omitempty"` } // LookupAccountTransactionsParams defines parameters for LookupAccountTransactions. @@ -1114,6 +1218,9 @@ type SearchForApplicationsParams struct { // Application ID ApplicationId *uint64 `json:"application-id,omitempty"` + // Filter just applications with the given creator address. + Creator *string `json:"creator,omitempty"` + // Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates. IncludeAll *bool `json:"include-all,omitempty"` @@ -1197,9 +1304,6 @@ type LookupAssetBalancesParams struct { // The next page of results. Use the next token provided by the previous results. Next *string `json:"next,omitempty"` - // Include results for the specified round. - Round *uint64 `json:"round,omitempty"` - // Results should have an amount greater than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used. CurrencyGreaterThan *uint64 `json:"currency-greater-than,omitempty"` diff --git a/api/handlers.go b/api/handlers.go index 481a78ef8..96253674c 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -40,31 +40,9 @@ type ServerImplementation struct { log *log.Logger disabledParams *DisabledMap -} - -///////////////////// -// Limit Constants // -///////////////////// - -// Transactions -const maxTransactionsLimit = 10000 -const defaultTransactionsLimit = 1000 - -// Accounts -const maxAccountsLimit = 1000 -const defaultAccountsLimit = 100 - -// Assets -const maxAssetsLimit = 1000 -const defaultAssetsLimit = 100 -// Asset Balances -const maxBalancesLimit = 10000 -const defaultBalancesLimit = 1000 - -// Applications -const maxApplicationsLimit = 1000 -const defaultApplicationsLimit = 100 + opts ExtraOptions +} ////////////////////// // Helper functions // @@ -147,6 +125,33 @@ func (si *ServerImplementation) MakeHealthCheck(ctx echo.Context) error { }) } +var errInvalidExcludeParameter = errors.New("invalid exclude argument") + +// set query options based on the value of the "exclude" parameter +func setExcludeQueryOptions(exclude []string, opts *idb.AccountQueryOptions) error { + for _, e := range exclude { + switch e { + case "all": + opts.IncludeAssetHoldings = false + opts.IncludeAssetParams = false + opts.IncludeAppLocalState = false + opts.IncludeAppParams = false + case "assets": + opts.IncludeAssetHoldings = false + case "created-assets": + opts.IncludeAssetParams = false + case "apps-local-state": + opts.IncludeAppLocalState = false + case "created-apps": + opts.IncludeAppParams = false + case "none": + default: + return fmt.Errorf(`unknown value "%s": %w`, e, errInvalidExcludeParameter) + } + } + return nil +} + func (si *ServerImplementation) verifyHandler(operationID string, ctx echo.Context) error { return Verify(si.disabledParams, operationID, ctx, si.log) } @@ -158,21 +163,35 @@ func (si *ServerImplementation) LookupAccountByID(ctx echo.Context, accountID st return badRequest(ctx, err.Error()) } - addr, errors := decodeAddress(&accountID, "account-id", make([]string, 0)) - if len(errors) != 0 { - return badRequest(ctx, errors[0]) + addr, decodeErrors := decodeAddress(&accountID, "account-id", make([]string, 0)) + if len(decodeErrors) != 0 { + return badRequest(ctx, decodeErrors[0]) } options := idb.AccountQueryOptions{ EqualToAddress: addr[:], IncludeAssetHoldings: true, IncludeAssetParams: true, + IncludeAppLocalState: true, + IncludeAppParams: true, Limit: 1, IncludeDeleted: boolOrDefault(params.IncludeAll), + MaxResources: uint64(si.opts.MaxAPIResourcesPerAccount), + } + + if params.Exclude != nil { + err := setExcludeQueryOptions(*params.Exclude, &options) + if err != nil { + return badRequest(ctx, err.Error()) + } } accounts, round, err := si.fetchAccounts(ctx.Request().Context(), options, params.Round) if err != nil { + var maxErr idb.MaxAPIResourcesPerAccountError + if errors.As(err, &maxErr) { + return ctx.JSON(http.StatusBadRequest, si.maxAccountsErrorToAccountsErrorResponse(maxErr)) + } return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingAccount, err)) } @@ -190,6 +209,123 @@ func (si *ServerImplementation) LookupAccountByID(ctx echo.Context, accountID st }) } +// LookupAccountAppLocalStates queries indexer for AppLocalState for a given account, and optionally a given app ID. +// (GET /v2/accounts/{account-id}/apps-local-state) +func (si *ServerImplementation) LookupAccountAppLocalStates(ctx echo.Context, accountID string, params generated.LookupAccountAppLocalStatesParams) error { + if err := si.verifyHandler("LookupAccountAppLocalStates", ctx); err != nil { + return badRequest(ctx, err.Error()) + } + + search := generated.SearchForApplicationsParams{ + Creator: &accountID, + ApplicationId: params.ApplicationId, + IncludeAll: params.IncludeAll, + Limit: params.Limit, + Next: params.Next, + } + options, err := si.appParamsToApplicationQuery(search) + if err != nil { + return badRequest(ctx, err.Error()) + } + + apps, round, err := si.fetchAppLocalStates(ctx.Request().Context(), options) + if err != nil { + return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingApplication, err)) + } + + var next *string + if len(apps) > 0 { + next = strPtr(strconv.FormatUint(apps[len(apps)-1].Id, 10)) + } + + out := generated.ApplicationLocalStatesResponse{ + AppsLocalStates: apps, + CurrentRound: round, + NextToken: next, + } + return ctx.JSON(http.StatusOK, out) +} + +// LookupAccountAssets queries indexer for AssetHolding for a given account, and optionally a given asset ID. +// (GET /v2/accounts/{account-id}/assets) +func (si *ServerImplementation) LookupAccountAssets(ctx echo.Context, accountID string, params generated.LookupAccountAssetsParams) error { + if err := si.verifyHandler("LookupAccountAssets", ctx); err != nil { + return badRequest(ctx, err.Error()) + } + + addr, errors := decodeAddress(&accountID, "account-id", make([]string, 0)) + if len(errors) != 0 { + return badRequest(ctx, errors[0]) + } + + var assetGreaterThan uint64 = 0 + if params.Next != nil { + agt, err := strconv.ParseUint(*params.Next, 10, 64) + if err != nil { + return badRequest(ctx, fmt.Sprintf("%s: %v", errUnableToParseNext, err)) + } + assetGreaterThan = agt + } + + query := idb.AssetBalanceQuery{ + Address: addr, + AssetID: uintOrDefault(params.AssetId), + AssetIDGT: assetGreaterThan, + IncludeDeleted: boolOrDefault(params.IncludeAll), + Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultBalancesLimit), si.opts.MaxBalancesLimit), + } + + assets, round, err := si.fetchAssetHoldings(ctx.Request().Context(), query) + if err != nil { + return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingAssetBalances, err)) + } + + var next *string + if len(assets) > 0 { + next = strPtr(strconv.FormatUint(assets[len(assets)-1].AssetId, 10)) + } + + return ctx.JSON(http.StatusOK, generated.AssetHoldingsResponse{ + CurrentRound: round, + NextToken: next, + Assets: assets, + }) +} + +// LookupAccountCreatedApplications queries indexer for AppParams for a given account, and optionally a given app ID. +// (GET /v2/accounts/{account-id}/created-applications) +func (si *ServerImplementation) LookupAccountCreatedApplications(ctx echo.Context, accountID string, params generated.LookupAccountCreatedApplicationsParams) error { + if err := si.verifyHandler("LookupAccountCreatedApplications", ctx); err != nil { + return badRequest(ctx, err.Error()) + } + + search := generated.SearchForApplicationsParams{ + Creator: &accountID, + ApplicationId: params.ApplicationId, + IncludeAll: params.IncludeAll, + Limit: params.Limit, + Next: params.Next, + } + return si.SearchForApplications(ctx, search) +} + +// LookupAccountCreatedAssets queries indexer for AssetParams for a given account, and optionally a given asset ID. +// (GET /v2/accounts/{account-id}/created-assets) +func (si *ServerImplementation) LookupAccountCreatedAssets(ctx echo.Context, accountID string, params generated.LookupAccountCreatedAssetsParams) error { + if err := si.verifyHandler("LookupAccountCreatedAssets", ctx); err != nil { + return badRequest(ctx, err.Error()) + } + + search := generated.SearchForAssetsParams{ + Creator: &accountID, + AssetId: params.AssetId, + IncludeAll: params.IncludeAll, + Limit: params.Limit, + Next: params.Next, + } + return si.SearchForAssets(ctx, search) +} + // SearchForAccounts returns accounts matching the provided parameters // (GET /v2/accounts) func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params generated.SearchForAccountsParams) error { @@ -201,19 +337,29 @@ func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params gener return badRequest(ctx, errMultiAcctRewind) } - spendingAddr, errors := decodeAddress(params.AuthAddr, "account-id", make([]string, 0)) - if len(errors) != 0 { - return badRequest(ctx, errors[0]) + spendingAddr, decodeErrors := decodeAddress(params.AuthAddr, "account-id", make([]string, 0)) + if len(decodeErrors) != 0 { + return badRequest(ctx, decodeErrors[0]) } options := idb.AccountQueryOptions{ IncludeAssetHoldings: true, IncludeAssetParams: true, - Limit: min(uintOrDefaultValue(params.Limit, defaultAccountsLimit), maxAccountsLimit), + IncludeAppLocalState: true, + IncludeAppParams: true, + Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultAccountsLimit), si.opts.MaxAccountsLimit), HasAssetID: uintOrDefault(params.AssetId), HasAppID: uintOrDefault(params.ApplicationId), EqualToAuthAddr: spendingAddr[:], IncludeDeleted: boolOrDefault(params.IncludeAll), + MaxResources: uint64(si.opts.MaxAPIResourcesPerAccount), + } + + if params.Exclude != nil { + err := setExcludeQueryOptions(*params.Exclude, &options) + if err != nil { + return badRequest(ctx, err.Error()) + } } // Set GT/LT on Algos or Asset depending on whether or not an assetID was specified @@ -234,8 +380,11 @@ func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params gener } accounts, round, err := si.fetchAccounts(ctx.Request().Context(), options, params.Round) - if err != nil { + var maxErr idb.MaxAPIResourcesPerAccountError + if errors.As(err, &maxErr) { + return ctx.JSON(http.StatusBadRequest, si.maxAccountsErrorToAccountsErrorResponse(maxErr)) + } return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingAccount, err)) } @@ -297,7 +446,13 @@ func (si *ServerImplementation) SearchForApplications(ctx echo.Context, params g if err := si.verifyHandler("SearchForApplications", ctx); err != nil { return badRequest(ctx, err.Error()) } - apps, round, err := si.fetchApplications(ctx.Request().Context(), params) + + options, err := si.appParamsToApplicationQuery(params) + if err != nil { + return badRequest(ctx, err.Error()) + } + + apps, round, err := si.fetchApplications(ctx.Request().Context(), options) if err != nil { return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingApplication, err)) } @@ -321,13 +476,13 @@ func (si *ServerImplementation) LookupApplicationByID(ctx echo.Context, applicat if err := si.verifyHandler("LookupApplicationByID", ctx); err != nil { return badRequest(ctx, err.Error()) } - p := generated.SearchForApplicationsParams{ - ApplicationId: &applicationID, - IncludeAll: params.IncludeAll, - Limit: uint64Ptr(1), + q := idb.ApplicationQuery{ + ApplicationID: applicationID, + IncludeDeleted: boolOrDefault(params.IncludeAll), + Limit: 1, } - apps, round, err := si.fetchApplications(ctx.Request().Context(), p) + apps, round, err := si.fetchApplications(ctx.Request().Context(), q) if err != nil { return indexerError(ctx, fmt.Errorf("%s: %w", errFailedSearchingApplication, err)) } @@ -364,7 +519,7 @@ func (si *ServerImplementation) LookupApplicationLogsByID(ctx echo.Context, appl Address: params.SenderAddress, } - filter, err := transactionParamsToTransactionFilter(searchParams) + filter, err := si.transactionParamsToTransactionFilter(searchParams) if err != nil { return badRequest(ctx, err.Error()) } @@ -421,7 +576,7 @@ func (si *ServerImplementation) LookupAssetByID(ctx echo.Context, assetID uint64 Limit: uint64Ptr(1), IncludeAll: params.IncludeAll, } - options, err := assetParamsToAssetQuery(search) + options, err := si.assetParamsToAssetQuery(search) if err != nil { return badRequest(ctx, err.Error()) } @@ -457,7 +612,7 @@ func (si *ServerImplementation) LookupAssetBalances(ctx echo.Context, assetID ui AmountGT: params.CurrencyGreaterThan, AmountLT: params.CurrencyLessThan, IncludeDeleted: boolOrDefault(params.IncludeAll), - Limit: min(uintOrDefaultValue(params.Limit, defaultBalancesLimit), maxBalancesLimit), + Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultBalancesLimit), si.opts.MaxBalancesLimit), } if params.Next != nil { @@ -524,7 +679,7 @@ func (si *ServerImplementation) SearchForAssets(ctx echo.Context, params generat return badRequest(ctx, err.Error()) } - options, err := assetParamsToAssetQuery(params) + options, err := si.assetParamsToAssetQuery(params) if err != nil { return badRequest(ctx, err.Error()) } @@ -570,7 +725,7 @@ func (si *ServerImplementation) LookupTransaction(ctx echo.Context, txid string) return badRequest(ctx, err.Error()) } - filter, err := transactionParamsToTransactionFilter(generated.SearchForTransactionsParams{ + filter, err := si.transactionParamsToTransactionFilter(generated.SearchForTransactionsParams{ Txid: strPtr(txid), }) if err != nil { @@ -611,7 +766,7 @@ func (si *ServerImplementation) SearchForTransactions(ctx echo.Context, params g return badRequest(ctx, err.Error()) } - filter, err := transactionParamsToTransactionFilter(params) + filter, err := si.transactionParamsToTransactionFilter(params) if err != nil { return badRequest(ctx, err.Error()) } @@ -677,13 +832,12 @@ func notFound(ctx echo.Context, err string) error { /////////////////////// // fetchApplications fetches all results -func (si *ServerImplementation) fetchApplications(ctx context.Context, params generated.SearchForApplicationsParams) ([]generated.Application, uint64, error) { - params.Limit = uint64Ptr(min(uintOrDefaultValue(params.Limit, defaultApplicationsLimit), maxApplicationsLimit)) +func (si *ServerImplementation) fetchApplications(ctx context.Context, params idb.ApplicationQuery) ([]generated.Application, uint64, error) { var apps []generated.Application var round uint64 err := callWithTimeout(ctx, si.log, si.timeout, func(ctx context.Context) error { var results <-chan idb.ApplicationRow - results, round = si.db.Applications(ctx, ¶ms) + results, round = si.db.Applications(ctx, params) for result := range results { if result.Error != nil { @@ -698,7 +852,31 @@ func (si *ServerImplementation) fetchApplications(ctx context.Context, params ge return nil, 0, err } - return apps, round, err + return apps, round, nil +} + +// fetchAppLocalStates fetches all generated.AppLocalState from a query +func (si *ServerImplementation) fetchAppLocalStates(ctx context.Context, params idb.ApplicationQuery) ([]generated.ApplicationLocalState, uint64, error) { + var als []generated.ApplicationLocalState + var round uint64 + err := callWithTimeout(ctx, si.log, si.timeout, func(ctx context.Context) error { + var results <-chan idb.AppLocalStateRow + results, round = si.db.AppLocalState(ctx, params) + + for result := range results { + if result.Error != nil { + return result.Error + } + als = append(als, result.AppLocalState) + } + + return nil + }) + if err != nil { + return nil, 0, err + } + + return als, round, nil } // fetchAssets fetches all results and converts them into generated.Asset objects @@ -797,6 +975,47 @@ func (si *ServerImplementation) fetchAssetBalances(ctx context.Context, options return balances, round, nil } +// fetchAssetHoldings fetches all balances from a query and converts them into +// generated.AssetHolding objects +func (si *ServerImplementation) fetchAssetHoldings(ctx context.Context, options idb.AssetBalanceQuery) ([]generated.AssetHolding, uint64 /*round*/, error) { + var round uint64 + balances := make([]generated.AssetHolding, 0) + err := callWithTimeout(ctx, si.log, si.timeout, func(ctx context.Context) error { + var assetbalchan <-chan idb.AssetBalanceRow + assetbalchan, round = si.db.AssetBalances(ctx, options) + + for row := range assetbalchan { + if row.Error != nil { + return row.Error + } + + addr := basics.Address{} + if len(row.Address) != len(addr) { + return fmt.Errorf(errInvalidCreatorAddress) + } + copy(addr[:], row.Address[:]) + + bal := generated.AssetHolding{ + Amount: row.Amount, + AssetId: row.AssetID, + IsFrozen: row.Frozen, + OptedInAtRound: row.CreatedRound, + OptedOutAtRound: row.ClosedRound, + Deleted: row.Deleted, + } + + balances = append(balances, bal) + } + + return nil + }) + if err != nil { + return nil, 0, err + } + + return balances, round, nil +} + // fetchBlock looks up a block and converts it into a generated.Block object // the method also loads the transactions into the returned block object. func (si *ServerImplementation) fetchBlock(ctx context.Context, round uint64) (generated.Block, error) { @@ -975,7 +1194,7 @@ func (si *ServerImplementation) fetchTransactions(ctx context.Context, filter id return nil, "", 0, err } - return results, nextToken, round, err + return results, nextToken, round, nil } ////////////////////// diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index 753ddb822..9d675dc06 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -1,16 +1,21 @@ package api import ( + "context" "encoding/base64" "fmt" + "io/ioutil" "net/http" "net/http/httptest" "strconv" + "strings" "testing" "time" "github.com/algorand/go-algorand/crypto" "github.com/labstack/echo/v4" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/algorand/go-algorand-sdk/encoding/json" @@ -26,6 +31,29 @@ import ( "github.com/algorand/indexer/util/test" ) +var defaultOpts = ExtraOptions{ + MaxTransactionsLimit: 10000, + DefaultTransactionsLimit: 1000, + + MaxAccountsLimit: 1000, + DefaultAccountsLimit: 100, + + MaxAssetsLimit: 1000, + DefaultAssetsLimit: 100, + + MaxBalancesLimit: 10000, + DefaultBalancesLimit: 1000, + + MaxApplicationsLimit: 1000, + DefaultApplicationsLimit: 100, + + DisabledMapConfig: MakeDisabledMapConfig(), +} + +func testServerImplementation(db idb.IndexerDb) *ServerImplementation { + return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts} +} + func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*postgres.IndexerDb /*db*/, func() /*shutdownFunc*/) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) @@ -46,12 +74,12 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin return db, newShutdownFunc } -func TestApplicationHandler(t *testing.T) { +func TestApplicationHandlers(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() /////////// - // Given // A block containing an app call txn with ExtraProgramPages + // Given // A block containing an app call txn with ExtraProgramPages, that the creator and another account have opted into /////////// const expectedAppIdx = 1 // must be 1 since this is the first txn @@ -76,8 +104,10 @@ func TestApplicationHandler(t *testing.T) { ApplicationID: expectedAppIdx, }, } + optInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) + optInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) err = db.AddBlock(&block) @@ -87,30 +117,641 @@ func TestApplicationHandler(t *testing.T) { // When // We query the app ////////// - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/applications/:appidx") - c.SetParamNames("appidx") - c.SetParamValues(strconv.Itoa(expectedAppIdx)) + setupReq := func(path, paramName, paramValue string) (echo.Context, *ServerImplementation, *httptest.ResponseRecorder) { + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath(path) + c.SetParamNames(paramName) + c.SetParamValues(paramValue) + api := testServerImplementation(db) + return c, api, rec + } - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + c, api, rec := setupReq("/v2/applications/:appidx", "appidx", strconv.Itoa(expectedAppIdx)) params := generated.LookupApplicationByIDParams{} err = api.LookupApplicationByID(c, expectedAppIdx, params) require.NoError(t, err) require.Equal(t, http.StatusOK, rec.Code, fmt.Sprintf("unexpected return code, body: %s", rec.Body.String())) ////////// - // Then // The response has non-zero ExtraProgramPages + // Then // The response has non-zero ExtraProgramPages and other app data ////////// + checkApp := func(t *testing.T, app *generated.Application) { + require.NotNil(t, app) + require.NotNil(t, app.Params.ExtraProgramPages) + require.Equal(t, uint64(extraPages), *app.Params.ExtraProgramPages) + require.Equal(t, app.Id, uint64(expectedAppIdx)) + require.NotNil(t, app.Params.Creator) + require.Equal(t, *app.Params.Creator, test.AccountA.String()) + require.Equal(t, app.Params.ApprovalProgram, []byte{0x02, 0x20, 0x01, 0x01, 0x22}) + require.Equal(t, app.Params.ClearStateProgram, []byte{0x02, 0x20, 0x01, 0x01, 0x22}) + } + var response generated.ApplicationResponse data := rec.Body.Bytes() err = json.Decode(data, &response) require.NoError(t, err) - require.NotNil(t, response.Application.Params.ExtraProgramPages) - require.Equal(t, uint64(extraPages), *response.Application.Params.ExtraProgramPages) + checkApp(t, response.Application) + + t.Run("created-applications", func(t *testing.T) { + ////////// + // When // We look up the app by creator address + ////////// + + c, api, rec := setupReq("/v2/accounts/:accountid/created-applications", "accountid", test.AccountA.String()) + params := generated.LookupAccountCreatedApplicationsParams{} + err = api.LookupAccountCreatedApplications(c, test.AccountA.String(), params) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rec.Code, fmt.Sprintf("unexpected return code, body: %s", rec.Body.String())) + + ////////// + // Then // The response has non-zero ExtraProgramPages and other app data + ////////// + + var response generated.ApplicationsResponse + data := rec.Body.Bytes() + err = json.Decode(data, &response) + require.NoError(t, err) + require.Len(t, response.Applications, 1) + checkApp(t, &response.Applications[0]) + }) + + checkAppLocalState := func(t *testing.T, ls *generated.ApplicationLocalState) { + require.NotNil(t, ls) + require.NotNil(t, ls.Deleted) + require.False(t, *ls.Deleted) + require.Equal(t, ls.Id, uint64(expectedAppIdx)) + } + + for _, tc := range []struct{ name, addr string }{ + {"creator", test.AccountA.String()}, + {"opted-in-account", test.AccountB.String()}, + } { + t.Run("app-local-state-"+tc.name, func(t *testing.T) { + ////////// + // When // We look up the app's local state for an address that has opted in + ////////// + + c, api, rec := setupReq("/v2/accounts/:accountid/apps-local-state", "accountid", test.AccountA.String()) + params := generated.LookupAccountAppLocalStatesParams{} + err = api.LookupAccountAppLocalStates(c, tc.addr, params) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rec.Code, fmt.Sprintf("unexpected return code, body: %s", rec.Body.String())) + + ////////// + // Then // AppLocalState is available for that address + ////////// + + var response generated.ApplicationLocalStatesResponse + data := rec.Body.Bytes() + err = json.Decode(data, &response) + require.NoError(t, err) + require.Len(t, response.AppsLocalStates, 1) + checkAppLocalState(t, &response.AppsLocalStates[0]) + }) + } +} + +func TestAccountExcludeParameters(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // A block containing a creator of an app, an asset, who also holds and has opted-into those apps. + /////////// + + const expectedAppIdx = 1 // must be 1 since this is the first txn + const expectedAssetIdx = 2 + createAppTxn := test.MakeCreateAppTxn(test.AccountA) + createAssetTxn := test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", "Asset 2", "http://asset2.com", test.AccountA) + appOptInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) + appOptInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) + assetOptInTxnA := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountA) + assetOptInTxnB := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountB) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn, &createAssetTxn, + &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) + require.NoError(t, err) + + err = db.AddBlock(&block) + require.NoError(t, err, "failed to commit") + + ////////// + // When // We look up the address using various exclude parameters. + ////////// + + setupReq := func(path, paramName, paramValue string) (echo.Context, *ServerImplementation, *httptest.ResponseRecorder) { + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath(path) + c.SetParamNames(paramName) + c.SetParamValues(paramValue) + api := testServerImplementation(db) + return c, api, rec + } + + ////////// + // Then // Those parameters are excluded. + ////////// + + testCases := []struct { + address basics.Address + exclude []string + check func(*testing.T, generated.AccountResponse) + errStatus int + }{{ + address: test.AccountA, + exclude: []string{"all"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.Nil(t, r.Account.CreatedAssets) + require.Nil(t, r.Account.CreatedApps) + require.Nil(t, r.Account.Assets) + require.Nil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"none"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.NotNil(t, r.Account.CreatedAssets) + require.NotNil(t, r.Account.CreatedApps) + require.NotNil(t, r.Account.Assets) + require.NotNil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + check: func(t *testing.T, r generated.AccountResponse) { + require.NotNil(t, r.Account.CreatedAssets) + require.NotNil(t, r.Account.CreatedApps) + require.NotNil(t, r.Account.Assets) + require.NotNil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"created-assets", "created-apps", "apps-local-state", "assets"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.Nil(t, r.Account.CreatedAssets) + require.Nil(t, r.Account.CreatedApps) + require.Nil(t, r.Account.Assets) + require.Nil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"created-assets"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.Nil(t, r.Account.CreatedAssets) + require.NotNil(t, r.Account.CreatedApps) + require.NotNil(t, r.Account.Assets) + require.NotNil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"created-apps"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.NotNil(t, r.Account.CreatedAssets) + require.Nil(t, r.Account.CreatedApps) + require.NotNil(t, r.Account.Assets) + require.NotNil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"apps-local-state"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.NotNil(t, r.Account.CreatedAssets) + require.NotNil(t, r.Account.CreatedApps) + require.NotNil(t, r.Account.Assets) + require.Nil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountA, + exclude: []string{"assets"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.NotNil(t, r.Account.CreatedAssets) + require.NotNil(t, r.Account.CreatedApps) + require.Nil(t, r.Account.Assets) + require.NotNil(t, r.Account.AppsLocalState) + }}, { + address: test.AccountB, + exclude: []string{"assets", "apps-local-state"}, + check: func(t *testing.T, r generated.AccountResponse) { + require.Nil(t, r.Account.CreatedAssets) + require.Nil(t, r.Account.CreatedApps) + require.Nil(t, r.Account.Assets) + require.Nil(t, r.Account.AppsLocalState) + }}, + { + address: test.AccountA, + exclude: []string{"abc"}, + errStatus: http.StatusBadRequest, + }, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("exclude %v", tc.exclude), func(t *testing.T) { + c, api, rec := setupReq("/v2/accounts/:account-id", "account-id", tc.address.String()) + err := api.LookupAccountByID(c, tc.address.String(), generated.LookupAccountByIDParams{Exclude: &tc.exclude}) + require.NoError(t, err) + if tc.errStatus != 0 { + require.Equal(t, tc.errStatus, rec.Code) + return + } + require.Equal(t, http.StatusOK, rec.Code, fmt.Sprintf("unexpected return code, body: %s", rec.Body.String())) + data := rec.Body.Bytes() + var response generated.AccountResponse + err = json.Decode(data, &response) + require.NoError(t, err) + tc.check(t, response) + }) + } + +} + +type accountsErrorResponse struct { + Data struct { + Address *string `json:"address,omitempty"` + MaxResults *uint64 `json:"max-results,omitempty"` + Message string `json:"message"` + TotalAppsOptedIn *uint64 `json:"total-apps-opted-in,omitempty"` + TotalAssetsOptedIn *uint64 `json:"total-assets-opted-in,omitempty"` + TotalCreatedApps *uint64 `json:"total-created-apps,omitempty"` + TotalCreatedAssets *uint64 `json:"total-created-assets,omitempty"` + } `json:"data,omitempty"` + Message string `json:"message"` +} + +func TestAccountMaxResultsLimit(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // A block containing an address that has created 10 apps, deleted 5 apps, and created 10 assets, + // // deleted 5 assets, and another address that has opted into the 5 apps and 5 assets remaining + /////////// + + deletedAppIDs := []uint64{1, 2, 3, 4, 5} + deletedAssetIDs := []uint64{6, 7, 8, 9, 10} + expectedAppIDs := []uint64{11, 12, 13, 14, 15} + expectedAssetIDs := []uint64{16, 17, 18, 19, 20} + + var txns []transactions.SignedTxnWithAD + // make apps and assets + for range deletedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) + } + for _, id := range deletedAssetIDs { + txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) + } + for range expectedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) + } + for _, id := range expectedAssetIDs { + txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) + } + // delete some apps and assets + for _, id := range deletedAppIDs { + txns = append(txns, test.MakeAppDestroyTxn(id, test.AccountA)) + } + for _, id := range deletedAssetIDs { + txns = append(txns, test.MakeAssetDestroyTxn(id, test.AccountA)) + } + + // opt in to the remaining ones + for _, id := range expectedAppIDs { + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountA)) + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountB)) + } + for _, id := range expectedAssetIDs { + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountA)) + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountB)) + } + + ptxns := make([]*transactions.SignedTxnWithAD, len(txns)) + for i := range txns { + ptxns[i] = &txns[i] + } + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...) + require.NoError(t, err) + + err = db.AddBlock(&block) + require.NoError(t, err, "failed to commit") + + ////////// + // When // We look up the address using a ServerImplementation with a maxAccountsAPIResults limit set, + // // and addresses with max # apps over & under the limit + ////////// + + maxResults := 14 + serverCtx, serverCancel := context.WithCancel(context.Background()) + defer serverCancel() + opts := defaultOpts + opts.MaxAPIResourcesPerAccount = uint64(maxResults) + listenAddr := "localhost:8989" + go Serve(serverCtx, listenAddr, db, nil, logrus.New(), opts) + + // wait at most a few seconds for server to come up + serverUp := false + for maxWait := 3 * time.Second; !serverUp && maxWait > 0; maxWait -= 50 * time.Millisecond { + time.Sleep(50 * time.Millisecond) + resp, err := http.Get("http://" + listenAddr + "/health") + if err != nil { + t.Log("waiting for server:", err) + continue + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Log("waiting for server OK:", resp.StatusCode) + continue + } + serverUp = true // server is up now + } + require.True(t, serverUp, "api.Serve did not start server in time") + + // make a real HTTP request (to additionally test generated param parsing logic) + makeReq := func(t *testing.T, path string, exclude []string, includeDeleted bool, next *string, limit *uint64) (*http.Response, []byte) { + var query []string + if len(exclude) > 0 { + query = append(query, "exclude="+strings.Join(exclude, ",")) + } + if includeDeleted { + query = append(query, "include-all=true") + } + if next != nil { + query = append(query, "next="+*next) + } + if limit != nil { + query = append(query, fmt.Sprintf("limit=%d", *limit)) + } + if len(query) > 0 { + path += "?" + strings.Join(query, "&") + } + t.Log("making HTTP request path", path) + resp, err := http.Get("http://" + listenAddr + path) + require.NoError(t, err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + return resp, body + } + + ////////// + // Then // The limit is enforced, leading to a 400 error + ////////// + + checkExclude := func(t *testing.T, acct generated.Account, exclude []string) { + for _, exc := range exclude { + switch exc { + case "all": + assert.Nil(t, acct.CreatedApps) + assert.Nil(t, acct.AppsLocalState) + assert.Nil(t, acct.CreatedAssets) + assert.Nil(t, acct.Assets) + case "created-assets": + assert.Nil(t, acct.CreatedAssets) + case "apps-local-state": + assert.Nil(t, acct.AppsLocalState) + case "created-apps": + assert.Nil(t, acct.CreatedApps) + case "assets": + assert.Nil(t, acct.Assets) + } + } + } + + testCases := []struct { + address basics.Address + exclude []string + includeDeleted bool + errStatus int + }{ + {address: test.AccountA, exclude: []string{"all"}}, + {address: test.AccountA, exclude: []string{"created-assets", "created-apps", "apps-local-state", "assets"}}, + {address: test.AccountA, exclude: []string{"assets", "created-apps"}}, + {address: test.AccountA, exclude: []string{"assets", "apps-local-state"}}, + {address: test.AccountA, exclude: []string{"assets", "apps-local-state"}, includeDeleted: true, errStatus: http.StatusBadRequest}, + {address: test.AccountB, exclude: []string{"created-assets", "apps-local-state"}}, + {address: test.AccountB, exclude: []string{"assets", "apps-local-state"}}, + {address: test.AccountA, exclude: []string{"created-assets"}, errStatus: http.StatusBadRequest}, + {address: test.AccountA, exclude: []string{"created-apps"}, errStatus: http.StatusBadRequest}, + {address: test.AccountA, exclude: []string{"apps-local-state"}, errStatus: http.StatusBadRequest}, + {address: test.AccountA, exclude: []string{"assets"}, errStatus: http.StatusBadRequest}, + } + + for _, tc := range testCases { + maxResults := 14 + t.Run(fmt.Sprintf("LookupAccountByID exclude %v", tc.exclude), func(t *testing.T) { + path := "/v2/accounts/" + tc.address.String() + resp, data := makeReq(t, path, tc.exclude, tc.includeDeleted, nil, nil) + if tc.errStatus != 0 { // was a 400 error expected? check error response + require.Equal(t, tc.errStatus, resp.StatusCode) + var response accountsErrorResponse + err = json.Decode(data, &response) + require.NoError(t, err) + assert.Equal(t, tc.address.String(), *response.Data.Address) + assert.Equal(t, uint64(maxResults), *response.Data.MaxResults) + if tc.includeDeleted { + assert.Equal(t, uint64(10), *response.Data.TotalCreatedApps) + assert.Equal(t, uint64(10), *response.Data.TotalCreatedAssets) + } else { + assert.Equal(t, uint64(5), *response.Data.TotalAppsOptedIn) + assert.Equal(t, uint64(5), *response.Data.TotalAssetsOptedIn) + assert.Equal(t, uint64(5), *response.Data.TotalCreatedApps) + assert.Equal(t, uint64(5), *response.Data.TotalCreatedAssets) + } + return + } + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.AccountResponse + err = json.Decode(data, &response) + require.NoError(t, err) + checkExclude(t, response.Account, tc.exclude) + }) + } + + ////////// + // When // We search all addresses using a ServerImplementation with a maxAccountsAPIResults limit set, + // // and one of those addresses is over the limit, but another address is not + ////////// + + for _, tc := range []struct { + exclude []string + errStatus int + errAddress basics.Address + }{ + {exclude: []string{"all"}}, + {exclude: []string{"created-assets", "created-apps", "apps-local-state", "assets"}}, + {exclude: []string{"assets", "apps-local-state"}}, + {errAddress: test.AccountA, exclude: nil, errStatus: 400}, + {errAddress: test.AccountA, exclude: []string{"created-assets"}, errStatus: http.StatusBadRequest}, + {errAddress: test.AccountA, exclude: []string{"created-apps"}, errStatus: http.StatusBadRequest}, + {errAddress: test.AccountA, exclude: []string{"apps-local-state"}, errStatus: http.StatusBadRequest}, + {errAddress: test.AccountA, exclude: []string{"assets"}, errStatus: http.StatusBadRequest}, + } { + t.Run(fmt.Sprintf("SearchForAccounts exclude %v", tc.exclude), func(t *testing.T) { + maxResults := 14 + resp, data := makeReq(t, "/v2/accounts", tc.exclude, false, nil, nil) + if tc.errStatus != 0 { // was a 400 error expected? check error response + require.Equal(t, tc.errStatus, resp.StatusCode) + var response accountsErrorResponse + err = json.Decode(data, &response) + require.NoError(t, err) + require.Equal(t, *response.Data.Address, tc.errAddress.String()) + require.Equal(t, *response.Data.MaxResults, uint64(maxResults)) + require.Equal(t, *response.Data.TotalAppsOptedIn, uint64(5)) + require.Equal(t, *response.Data.TotalCreatedApps, uint64(5)) + require.Equal(t, *response.Data.TotalAssetsOptedIn, uint64(5)) + require.Equal(t, *response.Data.TotalCreatedAssets, uint64(5)) + return + } + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.AccountsResponse + err = json.Decode(data, &response) + require.NoError(t, err) + + // check that the accounts are in there + var sawAccountA, sawAccountB bool + for _, acct := range response.Accounts { + switch acct.Address { + case test.AccountA.String(): + sawAccountA = true + require.Equal(t, acct.TotalAppsOptedIn, uint64(5)) + require.Equal(t, acct.TotalCreatedApps, uint64(5)) + require.Equal(t, acct.TotalAssetsOptedIn, uint64(5)) + require.Equal(t, acct.TotalCreatedAssets, uint64(5)) + case test.AccountB.String(): + sawAccountB = true + require.Equal(t, acct.TotalAppsOptedIn, uint64(5)) + require.Equal(t, acct.TotalCreatedApps, uint64(0)) + require.Equal(t, acct.TotalAssetsOptedIn, uint64(5)) + require.Equal(t, acct.TotalCreatedAssets, uint64(0)) + } + checkExclude(t, acct, tc.exclude) + } + require.True(t, sawAccountA && sawAccountB) + }) + } + + ////////// + // When // We look up the assets an account holds, and paginate through them using "Next" + ////////// + + t.Run("LookupAccountAssets", func(t *testing.T) { + var next *string // nil/unset to start + limit := uint64(2) // 2 at a time + var assets []generated.AssetHolding + for { + resp, data := makeReq(t, "/v2/accounts/"+test.AccountB.String()+"/assets", nil, false, next, &limit) + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.AssetHoldingsResponse + err = json.Decode(data, &response) + require.NoError(t, err) + if len(response.Assets) == 0 { + require.Nil(t, response.NextToken) + break + } + require.NotEmpty(t, response.Assets) + assets = append(assets, response.Assets...) + next = response.NextToken // paginate + } + ////////// + // Then // We can see all the assets, even though there were more than the limit + ////////// + require.Len(t, assets, 5) + for i, asset := range assets { + require.Equal(t, expectedAssetIDs[i], asset.AssetId) + } + }) + + ////////// + // When // We look up the assets an account has created, and paginate through them using "Next" + ////////// + + t.Run("LookupAccountCreatedAssets", func(t *testing.T) { + var next *string // nil/unset to start + limit := uint64(2) // 2 at a time + var assets []generated.Asset + for { + resp, data := makeReq(t, "/v2/accounts/"+test.AccountA.String()+"/created-assets", nil, false, next, &limit) + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.AssetsResponse + err = json.Decode(data, &response) + require.NoError(t, err) + if len(response.Assets) == 0 { + require.Nil(t, response.NextToken) + break + } + require.NotEmpty(t, response.Assets) + assets = append(assets, response.Assets...) + next = response.NextToken // paginate + } + ////////// + // Then // We can see all the assets, even though there were more than the limit + ////////// + require.Len(t, assets, 5) + for i, asset := range assets { + require.Equal(t, expectedAssetIDs[i], asset.Index) + } + }) + + ////////// + // When // We look up the apps an account has opted in to, and paginate through them using "Next" + ////////// + + t.Run("LookupAccountAppLocalStates", func(t *testing.T) { + var next *string // nil/unset to start + limit := uint64(2) // 2 at a time + var apps []generated.ApplicationLocalState + for { + resp, data := makeReq(t, "/v2/accounts/"+test.AccountA.String()+"/apps-local-state", nil, false, next, &limit) + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.ApplicationLocalStatesResponse + err = json.Decode(data, &response) + require.NoError(t, err) + if len(response.AppsLocalStates) == 0 { + require.Nil(t, response.NextToken) + break + } + require.NotEmpty(t, response.AppsLocalStates) + apps = append(apps, response.AppsLocalStates...) + next = response.NextToken // paginate + } + ////////// + // Then // We can see all the apps, even though there were more than the limit + ////////// + require.Len(t, apps, 5) + for i, app := range apps { + require.Equal(t, expectedAppIDs[i], app.Id) + } + }) + + ////////// + // When // We look up the apps an account has opted in to, and paginate through them using "Next" + ////////// + + t.Run("LookupAccountCreatedApplications", func(t *testing.T) { + var next *string // nil/unset to start + limit := uint64(2) // 2 at a time + var apps []generated.Application + for { + resp, data := makeReq(t, "/v2/accounts/"+test.AccountA.String()+"/created-applications", nil, false, next, &limit) + require.Equal(t, http.StatusOK, resp.StatusCode, fmt.Sprintf("unexpected return code, body: %s", string(data))) + var response generated.ApplicationsResponse + err = json.Decode(data, &response) + require.NoError(t, err) + if len(response.Applications) == 0 { + require.Nil(t, response.NextToken) + break + } + require.NotEmpty(t, response.Applications) + apps = append(apps, response.Applications...) + next = response.NextToken // paginate + } + ////////// + // Then // We can see all the apps, even though there were more than the limit + ////////// + require.Len(t, apps, 5) + for i, app := range apps { + require.Equal(t, expectedAppIDs[i], app.Id) + } + }) } func TestBlockNotFound(t *testing.T) { @@ -132,7 +773,7 @@ func TestBlockNotFound(t *testing.T) { c.SetParamNames("round-number") c.SetParamValues(strconv.Itoa(100)) - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err := api.LookupBlock(c, 100) require.NoError(t, err) @@ -205,7 +846,7 @@ func TestInnerTxn(t *testing.T) { c := e.NewContext(req, rec) c.SetPath("/v2/transactions/") - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.SearchForTransactions(c, tc.filter) require.NoError(t, err) @@ -214,7 +855,8 @@ func TestInnerTxn(t *testing.T) { ////////// require.Equal(t, http.StatusOK, rec.Code) var response generated.TransactionsResponse - json.Decode(rec.Body.Bytes(), &response) + err = json.Decode(rec.Body.Bytes(), &response) + require.NoError(t, err) require.Len(t, response.Transactions, 1) require.Equal(t, expectedID, *(response.Transactions[0].Id)) @@ -276,13 +918,14 @@ func TestPagingRootTxnDeduplication(t *testing.T) { // Get first page with limit 1. // Address filter causes results to return newest to oldest. - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.SearchForTransactions(c, tc.params) require.NoError(t, err) require.Equal(t, http.StatusOK, rec1.Code) var response generated.TransactionsResponse - json.Decode(rec1.Body.Bytes(), &response) + err = json.Decode(rec1.Body.Bytes(), &response) + require.NoError(t, err) require.Len(t, response.Transactions, 1) require.Equal(t, expectedID, *(response.Transactions[0].Id)) pageOneNextToken := *response.NextToken @@ -304,7 +947,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) { ////////// var response2 generated.TransactionsResponse require.Equal(t, http.StatusOK, rec2.Code) - json.Decode(rec2.Body.Bytes(), &response2) + err = json.Decode(rec2.Body.Bytes(), &response2) + require.NoError(t, err) require.Len(t, response2.Transactions, 0) // The fact that NextToken changes indicates that the search results were different. @@ -327,7 +971,7 @@ func TestPagingRootTxnDeduplication(t *testing.T) { // Get first page with limit 1. // Address filter causes results to return newest to oldest. - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.LookupBlock(c, uint64(block.Round())) require.NoError(t, err) @@ -336,7 +980,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) { ////////// var response generated.BlockResponse require.Equal(t, http.StatusOK, rec.Code) - json.Decode(rec.Body.Bytes(), &response) + err = json.Decode(rec.Body.Bytes(), &response) + require.NoError(t, err) require.NotNil(t, response.Transactions) require.Len(t, *response.Transactions, 1) @@ -442,7 +1087,7 @@ func TestVersion(t *testing.T) { /////////// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) e := echo.New() req := httptest.NewRequest(http.MethodGet, "/", nil) @@ -518,7 +1163,7 @@ func TestAccountClearsNonUTF8(t *testing.T) { c := e.NewContext(req, rec) c.SetPath("/v2/assets/") - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.SearchForAssets(c, generated.SearchForAssetsParams{}) require.NoError(t, err) @@ -543,7 +1188,7 @@ func TestAccountClearsNonUTF8(t *testing.T) { c := e.NewContext(req, rec) c.SetPath("/v2/accounts/") - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.LookupAccountByID(c, test.AccountA.String(), generated.LookupAccountByIDParams{}) require.NoError(t, err) @@ -626,7 +1271,7 @@ func TestLookupInnerLogs(t *testing.T) { c.SetParamNames("appIdx") c.SetParamValues(fmt.Sprintf("%d", tc.appID)) - api := &ServerImplementation{db: db, timeout: 30 * time.Second} + api := testServerImplementation(db) err = api.LookupApplicationLogsByID(c, tc.appID, params) require.NoError(t, err) diff --git a/api/handlers_test.go b/api/handlers_test.go index 4d62cd6a6..b1dfdcea7 100644 --- a/api/handlers_test.go +++ b/api/handlers_test.go @@ -41,49 +41,49 @@ func TestTransactionParamToTransactionFilter(t *testing.T) { { "Default", generated.SearchForTransactionsParams{}, - idb.TransactionFilter{Limit: defaultTransactionsLimit}, + idb.TransactionFilter{Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { "Limit", - generated.SearchForTransactionsParams{Limit: uint64Ptr(defaultTransactionsLimit + 10)}, - idb.TransactionFilter{Limit: defaultTransactionsLimit + 10}, + generated.SearchForTransactionsParams{Limit: uint64Ptr(defaultOpts.DefaultTransactionsLimit + 10)}, + idb.TransactionFilter{Limit: defaultOpts.DefaultTransactionsLimit + 10}, nil, }, { "Limit Max", - generated.SearchForTransactionsParams{Limit: uint64Ptr(maxTransactionsLimit + 10)}, - idb.TransactionFilter{Limit: maxTransactionsLimit}, + generated.SearchForTransactionsParams{Limit: uint64Ptr(defaultOpts.MaxTransactionsLimit + 10)}, + idb.TransactionFilter{Limit: defaultOpts.MaxTransactionsLimit}, nil, }, { "Int field", generated.SearchForTransactionsParams{AssetId: uint64Ptr(1234)}, - idb.TransactionFilter{AssetID: 1234, Limit: defaultTransactionsLimit}, + idb.TransactionFilter{AssetID: 1234, Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { "Pointer field", generated.SearchForTransactionsParams{Round: uint64Ptr(1234)}, - idb.TransactionFilter{Round: uint64Ptr(1234), Limit: defaultTransactionsLimit}, + idb.TransactionFilter{Round: uint64Ptr(1234), Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { "Base64 field", generated.SearchForTransactionsParams{NotePrefix: strPtr(base64.StdEncoding.EncodeToString([]byte("SomeData")))}, - idb.TransactionFilter{NotePrefix: []byte("SomeData"), Limit: defaultTransactionsLimit}, + idb.TransactionFilter{NotePrefix: []byte("SomeData"), Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { "Enum fields", generated.SearchForTransactionsParams{TxType: strPtr("pay"), SigType: strPtr("lsig")}, - idb.TransactionFilter{TypeEnum: 1, SigType: "lsig", Limit: defaultTransactionsLimit}, + idb.TransactionFilter{TypeEnum: 1, SigType: "lsig", Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { "Date time fields", generated.SearchForTransactionsParams{AfterTime: timePtr(time.Date(2020, 3, 4, 12, 0, 0, 0, time.FixedZone("UTC", 0)))}, - idb.TransactionFilter{AfterTime: time.Date(2020, 3, 4, 12, 0, 0, 0, time.FixedZone("UTC", 0)), Limit: defaultTransactionsLimit}, + idb.TransactionFilter{AfterTime: time.Date(2020, 3, 4, 12, 0, 0, 0, time.FixedZone("UTC", 0)), Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { @@ -95,7 +95,7 @@ func TestTransactionParamToTransactionFilter(t *testing.T) { { "As many fields as possible", generated.SearchForTransactionsParams{ - Limit: uint64Ptr(defaultTransactionsLimit + 1), + Limit: uint64Ptr(defaultOpts.DefaultTransactionsLimit + 1), Next: strPtr("next-token"), NotePrefix: strPtr(base64.StdEncoding.EncodeToString([]byte("custom-note"))), TxType: strPtr("pay"), @@ -115,7 +115,7 @@ func TestTransactionParamToTransactionFilter(t *testing.T) { ApplicationId: uint64Ptr(7), }, idb.TransactionFilter{ - Limit: defaultTransactionsLimit + 1, + Limit: defaultOpts.DefaultTransactionsLimit + 1, NextToken: "next-token", NotePrefix: []byte("custom-note"), TypeEnum: 1, @@ -157,56 +157,57 @@ func TestTransactionParamToTransactionFilter(t *testing.T) { { name: "Bitmask sender + closeTo(true)", params: generated.SearchForTransactionsParams{AddressRole: strPtr("sender"), ExcludeCloseTo: boolPtr(true)}, - filter: idb.TransactionFilter{AddressRole: 9, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 9, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Bitmask sender + closeTo(false)", params: generated.SearchForTransactionsParams{AddressRole: strPtr("sender"), ExcludeCloseTo: boolPtr(false)}, - filter: idb.TransactionFilter{AddressRole: 9, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 9, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Bitmask receiver + closeTo(true)", params: generated.SearchForTransactionsParams{AddressRole: strPtr("receiver"), ExcludeCloseTo: boolPtr(true)}, - filter: idb.TransactionFilter{AddressRole: 18, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 18, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Bitmask receiver + closeTo(false)", params: generated.SearchForTransactionsParams{AddressRole: strPtr("receiver"), ExcludeCloseTo: boolPtr(false)}, - filter: idb.TransactionFilter{AddressRole: 54, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 54, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Bitmask receiver + implicit closeTo (false)", params: generated.SearchForTransactionsParams{AddressRole: strPtr("receiver")}, - filter: idb.TransactionFilter{AddressRole: 54, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 54, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Bitmask freeze-target", params: generated.SearchForTransactionsParams{AddressRole: strPtr("freeze-target")}, - filter: idb.TransactionFilter{AddressRole: 64, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AddressRole: 64, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Currency to Algos when no asset-id", params: generated.SearchForTransactionsParams{CurrencyGreaterThan: uint64Ptr(10), CurrencyLessThan: uint64Ptr(20)}, - filter: idb.TransactionFilter{AlgosGT: uint64Ptr(10), AlgosLT: uint64Ptr(20), Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{AlgosGT: uint64Ptr(10), AlgosLT: uint64Ptr(20), Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, { name: "Searching by application-id", params: generated.SearchForTransactionsParams{ApplicationId: uint64Ptr(1234)}, - filter: idb.TransactionFilter{ApplicationID: 1234, Limit: defaultTransactionsLimit}, + filter: idb.TransactionFilter{ApplicationID: 1234, Limit: defaultOpts.DefaultTransactionsLimit}, errorContains: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - filter, err := transactionParamsToTransactionFilter(test.params) + si := testServerImplementation(nil) + filter, err := si.transactionParamsToTransactionFilter(test.params) if len(test.errorContains) > 0 { require.Error(t, err) for _, msg := range test.errorContains { @@ -228,7 +229,7 @@ func TestValidateTransactionFilter(t *testing.T) { }{ { "Default", - idb.TransactionFilter{Limit: defaultTransactionsLimit}, + idb.TransactionFilter{Limit: defaultOpts.DefaultTransactionsLimit}, nil, }, { @@ -540,11 +541,9 @@ func TestFetchTransactions(t *testing.T) { // Setup the mocked responses mockIndexer := &mocks.IndexerDb{} - si := ServerImplementation{ - EnableAddressSearchRoundRewind: true, - db: mockIndexer, - timeout: 1 * time.Second, - } + si := testServerImplementation(mockIndexer) + si.EnableAddressSearchRoundRewind = true + si.timeout = 1 * time.Second roundTime := time.Now() roundTime64 := uint64(roundTime.Unix()) @@ -627,10 +626,8 @@ func TestFetchAccountsRewindRoundTooLarge(t *testing.T) { db := &mocks.IndexerDb{} db.On("GetAccounts", mock.Anything, mock.Anything).Return(outCh, uint64(7)).Once() - si := ServerImplementation{ - EnableAddressSearchRoundRewind: true, - db: db, - } + si := testServerImplementation(db) + si.EnableAddressSearchRoundRewind = true atRound := uint64(8) _, _, err := si.fetchAccounts(context.Background(), idb.AccountQueryOptions{}, &atRound) assert.Error(t, err) @@ -680,10 +677,8 @@ func createTxn(t *testing.T, target string) []byte { func TestLookupApplicationLogsByID(t *testing.T) { mockIndexer := &mocks.IndexerDb{} - si := ServerImplementation{ - EnableAddressSearchRoundRewind: true, - db: mockIndexer, - } + si := testServerImplementation(mockIndexer) + si.EnableAddressSearchRoundRewind = true txnBytes := loadResourceFileOrPanic("test_resources/app_call_logs.txn") var stxn transactions.SignedTxnWithAD @@ -894,10 +889,8 @@ func TestTimeouts(t *testing.T) { // Make a mock indexer and tell the mock to timeout. mockIndexer := &mocks.IndexerDb{} - si := ServerImplementation{ - db: mockIndexer, - timeout: 5 * time.Millisecond, - } + si := testServerImplementation(mockIndexer) + si.timeout = 5 * time.Millisecond // Setup context... e := echo.New() @@ -907,7 +900,7 @@ func TestTimeouts(t *testing.T) { // configure the mock to timeout, then call the handler. tc.mockCall(mockIndexer, timeout) - err := tc.callHandler(c, si) + err := tc.callHandler(c, *si) require.NoError(t, err) bodyStr := rec1.Body.String() @@ -927,21 +920,19 @@ func TestApplicationLimits(t *testing.T) { { name: "Default", limit: nil, - expected: defaultApplicationsLimit, + expected: defaultOpts.DefaultApplicationsLimit, }, { name: "Max", limit: uint64Ptr(math.MaxUint64), - expected: maxApplicationsLimit, + expected: defaultOpts.MaxApplicationsLimit, }, } // Mock backend to capture default limits mockIndexer := &mocks.IndexerDb{} - si := ServerImplementation{ - db: mockIndexer, - timeout: 5 * time.Millisecond, - } + si := testServerImplementation(mockIndexer) + si.timeout = 5 * time.Millisecond for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -957,10 +948,9 @@ func TestApplicationLimits(t *testing.T) { Return(nil, uint64(0)). Run(func(args mock.Arguments) { require.Len(t, args, 2) - require.IsType(t, &generated.SearchForApplicationsParams{}, args[1]) - params := args[1].(*generated.SearchForApplicationsParams) - require.NotNil(t, params.Limit) - require.Equal(t, *params.Limit, tc.expected) + require.IsType(t, idb.ApplicationQuery{}, args[1]) + params := args[1].(idb.ApplicationQuery) + require.Equal(t, params.Limit, tc.expected) }) err := si.SearchForApplications(c, generated.SearchForApplicationsParams{ diff --git a/api/indexer.oas2.json b/api/indexer.oas2.json index 342da079f..a3c33ff8b 100644 --- a/api/indexer.oas2.json +++ b/api/indexer.oas2.json @@ -69,6 +69,9 @@ { "$ref": "#/parameters/include-all" }, + { + "$ref": "#/parameters/exclude" + }, { "$ref": "#/parameters/currency-less-than" }, @@ -120,6 +123,9 @@ }, { "$ref": "#/parameters/include-all" + }, + { + "$ref": "#/parameters/exclude" } ], "responses": { @@ -138,6 +144,190 @@ } } }, + "/v2/accounts/{account-id}/assets": { + "get": { + "description": "Lookup an account's asset holdings, optionally for a specific ID.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "lookup" + ], + "operationId": "lookupAccountAssets", + "parameters": [ + { + "$ref": "#/parameters/account-id" + }, + { + "$ref": "#/parameters/asset-id" + }, + { + "$ref": "#/parameters/include-all" + }, + { + "$ref": "#/parameters/limit" + }, + { + "$ref": "#/parameters/next" + } + ], + "responses": { + "200": { + "$ref": "#/responses/AssetHoldingsResponse" + }, + "400": { + "$ref": "#/responses/ErrorResponse" + }, + "404": { + "$ref": "#/responses/ErrorResponse" + }, + "500": { + "$ref": "#/responses/ErrorResponse" + } + } + } + }, + "/v2/accounts/{account-id}/created-assets": { + "get": { + "description": "Lookup an account's created asset parameters, optionally for a specific ID.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "lookup" + ], + "operationId": "lookupAccountCreatedAssets", + "parameters": [ + { + "$ref": "#/parameters/account-id" + }, + { + "$ref": "#/parameters/asset-id" + }, + { + "$ref": "#/parameters/include-all" + }, + { + "$ref": "#/parameters/limit" + }, + { + "$ref": "#/parameters/next" + } + ], + "responses": { + "200": { + "$ref": "#/responses/AssetsResponse" + }, + "400": { + "$ref": "#/responses/ErrorResponse" + }, + "404": { + "$ref": "#/responses/ErrorResponse" + }, + "500": { + "$ref": "#/responses/ErrorResponse" + } + } + } + }, + "/v2/accounts/{account-id}/apps-local-state": { + "get": { + "description": "Lookup an account's asset holdings, optionally for a specific ID.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "lookup" + ], + "operationId": "lookupAccountAppLocalStates", + "parameters": [ + { + "$ref": "#/parameters/account-id" + }, + { + "$ref": "#/parameters/application-id" + }, + { + "$ref": "#/parameters/include-all" + }, + { + "$ref": "#/parameters/limit" + }, + { + "$ref": "#/parameters/next" + } + ], + "responses": { + "200": { + "$ref": "#/responses/ApplicationLocalStatesResponse" + }, + "400": { + "$ref": "#/responses/ErrorResponse" + }, + "404": { + "$ref": "#/responses/ErrorResponse" + }, + "500": { + "$ref": "#/responses/ErrorResponse" + } + } + } + }, + "/v2/accounts/{account-id}/created-applications": { + "get": { + "description": "Lookup an account's created application parameters, optionally for a specific ID.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "lookup" + ], + "operationId": "lookupAccountCreatedApplications", + "parameters": [ + { + "$ref": "#/parameters/account-id" + }, + { + "$ref": "#/parameters/application-id" + }, + { + "$ref": "#/parameters/include-all" + }, + { + "$ref": "#/parameters/limit" + }, + { + "$ref": "#/parameters/next" + } + ], + "responses": { + "200": { + "$ref": "#/responses/ApplicationsResponse" + }, + "400": { + "$ref": "#/responses/ErrorResponse" + }, + "404": { + "$ref": "#/responses/ErrorResponse" + }, + "500": { + "$ref": "#/responses/ErrorResponse" + } + } + } + }, "/v2/accounts/{account-id}/transactions": { "get": { "description": "Lookup account transactions.", @@ -231,6 +421,12 @@ { "$ref": "#/parameters/application-id" }, + { + "type": "string", + "description": "Filter just applications with the given creator address.", + "name": "creator", + "in": "query" + }, { "$ref": "#/parameters/include-all" }, @@ -455,9 +651,6 @@ { "$ref": "#/parameters/next" }, - { - "$ref": "#/parameters/round" - }, { "$ref": "#/parameters/currency-greater-than" }, @@ -734,7 +927,11 @@ "pending-rewards", "amount-without-pending-rewards", "rewards", - "status" + "status", + "total-apps-opted-in", + "total-assets-opted-in", + "total-created-apps", + "total-created-assets" ], "properties": { "address": { @@ -817,6 +1014,22 @@ "lsig" ] }, + "total-apps-opted-in": { + "description": "The count of all applications that have been opted in, equivalent to the count of application local data (AppLocalState objects) stored in this account.", + "type": "integer" + }, + "total-assets-opted-in": { + "description": "The count of all assets that have been opted in, equivalent to the count of AssetHolding objects held by this account.", + "type": "integer" + }, + "total-created-apps": { + "description": "The count of all apps (AppParams objects) created by this account.", + "type": "integer" + }, + "total-created-assets": { + "description": "The count of all assets (AssetParams objects) created by this account.", + "type": "integer" + }, "auth-addr": { "description": "\\[spend\\] the address against which signing should be checked. If empty, the address of the current account is used. This field can be updated in any transaction by setting the RekeyTo field.", "type": "string", @@ -1112,7 +1325,6 @@ "type": "object", "required": [ "asset-id", - "creator", "amount", "is-frozen" ], @@ -1126,10 +1338,6 @@ "description": "Asset ID of the holding.", "type": "integer" }, - "creator": { - "description": "Address that created this asset. This is the address where the parameters for this asset can be found, and also the address where unwanted asset units can be sent in the worst case.", - "type": "string" - }, "is-frozen": { "description": "\\[f\\] whether or not the holding is frozen.", "type": "boolean" @@ -2092,6 +2300,23 @@ "name": "include-all", "in": "query" }, + "exclude": { + "description": "Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account.", + "name": "exclude", + "in": "query", + "type": "array", + "items": { + "type": "string", + "enum": [ + "all", + "assets", + "created-assets", + "apps-local-state", + "created-apps", + "none" + ] + } + }, "limit": { "type": "integer", "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", @@ -2207,6 +2432,32 @@ } } }, + "AssetHoldingsResponse": { + "description": "(empty)", + "schema": { + "type": "object", + "required": [ + "current-round", + "assets" + ], + "properties": { + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + }, + "assets": { + "type": "array", + "items": { + "$ref": "#/definitions/AssetHolding" + } + } + } + } + }, "AccountsResponse": { "description": "(empty)", "schema": { @@ -2334,6 +2585,32 @@ } } }, + "ApplicationLocalStatesResponse": { + "description": "(empty)", + "schema": { + "type": "object", + "required": [ + "current-round", + "apps-local-states" + ], + "properties": { + "apps-local-states": { + "type": "array", + "items": { + "$ref": "#/definitions/ApplicationLocalState" + } + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + } + } + }, "AssetResponse": { "description": "(empty)", "schema": { diff --git a/api/indexer.oas3.yml b/api/indexer.oas3.yml index afb6456dc..4f86e7c9d 100644 --- a/api/indexer.oas3.yml +++ b/api/indexer.oas3.yml @@ -97,6 +97,27 @@ "type": "integer" } }, + "exclude": { + "description": "Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account.", + "explode": false, + "in": "query", + "name": "exclude", + "schema": { + "items": { + "enum": [ + "all", + "assets", + "created-assets", + "apps-local-state", + "created-apps", + "none" + ], + "type": "string" + }, + "type": "array" + }, + "style": "form" + }, "exclude-close-to": { "description": "Combine with address and address-role parameters to define what type of address to search for. The close to fields are normally treated as a receiver, if you would like to exclude them set this parameter to true.", "in": "query", @@ -281,6 +302,36 @@ }, "description": "(empty)" }, + "ApplicationLocalStatesResponse": { + "content": { + "application/json": { + "schema": { + "properties": { + "apps-local-states": { + "items": { + "$ref": "#/components/schemas/ApplicationLocalState" + }, + "type": "array" + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + }, + "required": [ + "apps-local-states", + "current-round" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, "ApplicationLogsResponse": { "content": { "application/json": { @@ -397,6 +448,36 @@ }, "description": "(empty)" }, + "AssetHoldingsResponse": { + "content": { + "application/json": { + "schema": { + "properties": { + "assets": { + "items": { + "$ref": "#/components/schemas/AssetHolding" + }, + "type": "array" + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + }, + "required": [ + "assets", + "current-round" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, "AssetResponse": { "content": { "application/json": { @@ -647,6 +728,22 @@ "status": { "description": "\\[onl\\] delegation status of the account's MicroAlgos\n* Offline - indicates that the associated account is delegated.\n* Online - indicates that the associated account used as part of the delegation pool.\n* NotParticipating - indicates that the associated account is neither a delegator nor a delegate.", "type": "string" + }, + "total-apps-opted-in": { + "description": "The count of all applications that have been opted in, equivalent to the count of application local data (AppLocalState objects) stored in this account.", + "type": "integer" + }, + "total-assets-opted-in": { + "description": "The count of all assets that have been opted in, equivalent to the count of AssetHolding objects held by this account.", + "type": "integer" + }, + "total-created-apps": { + "description": "The count of all apps (AppParams objects) created by this account.", + "type": "integer" + }, + "total-created-assets": { + "description": "The count of all assets (AssetParams objects) created by this account.", + "type": "integer" } }, "required": [ @@ -656,7 +753,11 @@ "pending-rewards", "rewards", "round", - "status" + "status", + "total-apps-opted-in", + "total-assets-opted-in", + "total-created-apps", + "total-created-assets" ], "type": "object" }, @@ -910,10 +1011,6 @@ "description": "Asset ID of the holding.", "type": "integer" }, - "creator": { - "description": "Address that created this asset. This is the address where the parameters for this asset can be found, and also the address where unwanted asset units can be sent in the worst case.", - "type": "string" - }, "deleted": { "description": "Whether or not the asset holding is currently deleted from its account.", "type": "boolean" @@ -936,7 +1033,6 @@ "required": [ "amount", "asset-id", - "creator", "is-frozen" ], "type": "object" @@ -1923,6 +2019,27 @@ "type": "boolean" } }, + { + "description": "Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account.", + "explode": false, + "in": "query", + "name": "exclude", + "schema": { + "items": { + "enum": [ + "all", + "assets", + "created-assets", + "apps-local-state", + "created-apps", + "none" + ], + "type": "string" + }, + "type": "array" + }, + "style": "form" + }, { "description": "Results should have an amount less than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", "in": "query", @@ -2068,6 +2185,27 @@ "schema": { "type": "boolean" } + }, + { + "description": "Exclude additional items such as asset holdings, application local data stored for this account, asset parameters created by this account, and application parameters created by this account.", + "explode": false, + "in": "query", + "name": "exclude", + "schema": { + "items": { + "enum": [ + "all", + "assets", + "created-assets", + "apps-local-state", + "created-apps", + "none" + ], + "type": "string" + }, + "type": "array" + }, + "style": "form" } ], "responses": { @@ -2166,159 +2304,51 @@ ] } }, - "/v2/accounts/{account-id}/transactions": { + "/v2/accounts/{account-id}/apps-local-state": { "get": { - "description": "Lookup account transactions.", - "operationId": "lookupAccountTransactions", + "description": "Lookup an account's asset holdings, optionally for a specific ID.", + "operationId": "lookupAccountAppLocalStates", "parameters": [ { - "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", - "in": "query", - "name": "limit", - "schema": { - "type": "integer" - } - }, - { - "description": "The next page of results. Use the next token provided by the previous results.", - "in": "query", - "name": "next", - "schema": { - "type": "string" - } - }, - { - "description": "Specifies a prefix which must be contained in the note field.", - "in": "query", - "name": "note-prefix", - "schema": { - "type": "string", - "x-algorand-format": "base64" - }, - "x-algorand-format": "base64" - }, - { - "in": "query", - "name": "tx-type", - "schema": { - "enum": [ - "pay", - "keyreg", - "acfg", - "axfer", - "afrz", - "appl" - ], - "type": "string" - } - }, - { - "description": "SigType filters just results using the specified type of signature:\n* sig - Standard\n* msig - MultiSig\n* lsig - LogicSig", - "in": "query", - "name": "sig-type", - "schema": { - "enum": [ - "sig", - "msig", - "lsig" - ], - "type": "string" - } - }, - { - "description": "Lookup the specific transaction by ID.", - "in": "query", - "name": "txid", + "description": "account string", + "in": "path", + "name": "account-id", + "required": true, "schema": { "type": "string" } }, { - "description": "Include results for the specified round.", - "in": "query", - "name": "round", - "schema": { - "type": "integer" - } - }, - { - "description": "Include results at or after the specified min-round.", - "in": "query", - "name": "min-round", - "schema": { - "type": "integer" - } - }, - { - "description": "Include results at or before the specified max-round.", + "description": "Application ID", "in": "query", - "name": "max-round", + "name": "application-id", "schema": { "type": "integer" } }, { - "description": "Asset ID", + "description": "Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates.", "in": "query", - "name": "asset-id", + "name": "include-all", "schema": { - "type": "integer" + "type": "boolean" } }, { - "description": "Include results before the given time. Must be an RFC 3339 formatted string.", - "in": "query", - "name": "before-time", - "schema": { - "format": "date-time", - "type": "string", - "x-algorand-format": "RFC3339 String" - }, - "x-algorand-format": "RFC3339 String" - }, - { - "description": "Include results after the given time. Must be an RFC 3339 formatted string.", - "in": "query", - "name": "after-time", - "schema": { - "format": "date-time", - "type": "string", - "x-algorand-format": "RFC3339 String" - }, - "x-algorand-format": "RFC3339 String" - }, - { - "description": "Results should have an amount greater than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", + "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", "in": "query", - "name": "currency-greater-than", + "name": "limit", "schema": { "type": "integer" } }, { - "description": "Results should have an amount less than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", + "description": "The next page of results. Use the next token provided by the previous results.", "in": "query", - "name": "currency-less-than", - "schema": { - "type": "integer" - } - }, - { - "description": "account string", - "in": "path", - "name": "account-id", - "required": true, + "name": "next", "schema": { "type": "string" } - }, - { - "description": "Include results which include the rekey-to field.", - "in": "query", - "name": "rekey-to", - "schema": { - "type": "boolean" - } } ], "responses": { @@ -2327,6 +2357,12 @@ "application/json": { "schema": { "properties": { + "apps-local-states": { + "items": { + "$ref": "#/components/schemas/ApplicationLocalState" + }, + "type": "array" + }, "current-round": { "description": "Round at which the results were computed.", "type": "integer" @@ -2334,17 +2370,11 @@ "next-token": { "description": "Used for pagination, when making another request provide this token with the next parameter.", "type": "string" - }, - "transactions": { - "items": { - "$ref": "#/components/schemas/Transaction" - }, - "type": "array" } }, "required": [ - "current-round", - "transactions" + "apps-local-states", + "current-round" ], "type": "object" } @@ -2374,7 +2404,7 @@ }, "description": "Response for errors" }, - "500": { + "404": { "content": { "application/json": { "schema": { @@ -2395,24 +2425,740 @@ } }, "description": "Response for errors" - } - }, - "tags": [ - "lookup" - ] - } - }, - "/v2/applications": { - "get": { - "description": "Search for applications", - "operationId": "searchForApplications", - "parameters": [ - { - "description": "Application ID", - "in": "query", - "name": "application-id", - "schema": { - "type": "integer" + }, + "500": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + } + }, + "tags": [ + "lookup" + ] + } + }, + "/v2/accounts/{account-id}/assets": { + "get": { + "description": "Lookup an account's asset holdings, optionally for a specific ID.", + "operationId": "lookupAccountAssets", + "parameters": [ + { + "description": "account string", + "in": "path", + "name": "account-id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Asset ID", + "in": "query", + "name": "asset-id", + "schema": { + "type": "integer" + } + }, + { + "description": "Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates.", + "in": "query", + "name": "include-all", + "schema": { + "type": "boolean" + } + }, + { + "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "description": "The next page of results. Use the next token provided by the previous results.", + "in": "query", + "name": "next", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "assets": { + "items": { + "$ref": "#/components/schemas/AssetHolding" + }, + "type": "array" + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + }, + "required": [ + "assets", + "current-round" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, + "400": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "404": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "500": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + } + }, + "tags": [ + "lookup" + ] + } + }, + "/v2/accounts/{account-id}/created-applications": { + "get": { + "description": "Lookup an account's created application parameters, optionally for a specific ID.", + "operationId": "lookupAccountCreatedApplications", + "parameters": [ + { + "description": "account string", + "in": "path", + "name": "account-id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Application ID", + "in": "query", + "name": "application-id", + "schema": { + "type": "integer" + } + }, + { + "description": "Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates.", + "in": "query", + "name": "include-all", + "schema": { + "type": "boolean" + } + }, + { + "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "description": "The next page of results. Use the next token provided by the previous results.", + "in": "query", + "name": "next", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "applications": { + "items": { + "$ref": "#/components/schemas/Application" + }, + "type": "array" + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + }, + "required": [ + "applications", + "current-round" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, + "400": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "404": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "500": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + } + }, + "tags": [ + "lookup" + ] + } + }, + "/v2/accounts/{account-id}/created-assets": { + "get": { + "description": "Lookup an account's created asset parameters, optionally for a specific ID.", + "operationId": "lookupAccountCreatedAssets", + "parameters": [ + { + "description": "account string", + "in": "path", + "name": "account-id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Asset ID", + "in": "query", + "name": "asset-id", + "schema": { + "type": "integer" + } + }, + { + "description": "Include all items including closed accounts, deleted applications, destroyed assets, opted-out asset holdings, and closed-out application localstates.", + "in": "query", + "name": "include-all", + "schema": { + "type": "boolean" + } + }, + { + "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "description": "The next page of results. Use the next token provided by the previous results.", + "in": "query", + "name": "next", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "assets": { + "items": { + "$ref": "#/components/schemas/Asset" + }, + "type": "array" + }, + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + } + }, + "required": [ + "assets", + "current-round" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, + "400": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "404": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "500": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + } + }, + "tags": [ + "lookup" + ] + } + }, + "/v2/accounts/{account-id}/transactions": { + "get": { + "description": "Lookup account transactions.", + "operationId": "lookupAccountTransactions", + "parameters": [ + { + "description": "Maximum number of results to return. There could be additional pages even if the limit is not reached.", + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "description": "The next page of results. Use the next token provided by the previous results.", + "in": "query", + "name": "next", + "schema": { + "type": "string" + } + }, + { + "description": "Specifies a prefix which must be contained in the note field.", + "in": "query", + "name": "note-prefix", + "schema": { + "type": "string", + "x-algorand-format": "base64" + }, + "x-algorand-format": "base64" + }, + { + "in": "query", + "name": "tx-type", + "schema": { + "enum": [ + "pay", + "keyreg", + "acfg", + "axfer", + "afrz", + "appl" + ], + "type": "string" + } + }, + { + "description": "SigType filters just results using the specified type of signature:\n* sig - Standard\n* msig - MultiSig\n* lsig - LogicSig", + "in": "query", + "name": "sig-type", + "schema": { + "enum": [ + "sig", + "msig", + "lsig" + ], + "type": "string" + } + }, + { + "description": "Lookup the specific transaction by ID.", + "in": "query", + "name": "txid", + "schema": { + "type": "string" + } + }, + { + "description": "Include results for the specified round.", + "in": "query", + "name": "round", + "schema": { + "type": "integer" + } + }, + { + "description": "Include results at or after the specified min-round.", + "in": "query", + "name": "min-round", + "schema": { + "type": "integer" + } + }, + { + "description": "Include results at or before the specified max-round.", + "in": "query", + "name": "max-round", + "schema": { + "type": "integer" + } + }, + { + "description": "Asset ID", + "in": "query", + "name": "asset-id", + "schema": { + "type": "integer" + } + }, + { + "description": "Include results before the given time. Must be an RFC 3339 formatted string.", + "in": "query", + "name": "before-time", + "schema": { + "format": "date-time", + "type": "string", + "x-algorand-format": "RFC3339 String" + }, + "x-algorand-format": "RFC3339 String" + }, + { + "description": "Include results after the given time. Must be an RFC 3339 formatted string.", + "in": "query", + "name": "after-time", + "schema": { + "format": "date-time", + "type": "string", + "x-algorand-format": "RFC3339 String" + }, + "x-algorand-format": "RFC3339 String" + }, + { + "description": "Results should have an amount greater than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", + "in": "query", + "name": "currency-greater-than", + "schema": { + "type": "integer" + } + }, + { + "description": "Results should have an amount less than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", + "in": "query", + "name": "currency-less-than", + "schema": { + "type": "integer" + } + }, + { + "description": "account string", + "in": "path", + "name": "account-id", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Include results which include the rekey-to field.", + "in": "query", + "name": "rekey-to", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current-round": { + "description": "Round at which the results were computed.", + "type": "integer" + }, + "next-token": { + "description": "Used for pagination, when making another request provide this token with the next parameter.", + "type": "string" + }, + "transactions": { + "items": { + "$ref": "#/components/schemas/Transaction" + }, + "type": "array" + } + }, + "required": [ + "current-round", + "transactions" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, + "400": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + }, + "500": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": {}, + "type": "object" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + } + } + }, + "description": "Response for errors" + } + }, + "tags": [ + "lookup" + ] + } + }, + "/v2/applications": { + "get": { + "description": "Search for applications", + "operationId": "searchForApplications", + "parameters": [ + { + "description": "Application ID", + "in": "query", + "name": "application-id", + "schema": { + "type": "integer" + } + }, + { + "description": "Filter just applications with the given creator address.", + "in": "query", + "name": "creator", + "schema": { + "type": "string" } }, { @@ -2989,14 +3735,6 @@ "type": "string" } }, - { - "description": "Include results for the specified round.", - "in": "query", - "name": "round", - "schema": { - "type": "integer" - } - }, { "description": "Results should have an amount greater than this value. MicroAlgos are the default currency unless an asset-id is provided, in which case the asset will be used.", "in": "query", diff --git a/api/server.go b/api/server.go index 74f5616ee..6854eb7a8 100644 --- a/api/server.go +++ b/api/server.go @@ -40,6 +40,35 @@ type ExtraOptions struct { // DisabledMapConfig is the disabled map configuration that is being used by the server DisabledMapConfig *DisabledMapConfig + + // MaxAPIResourcesPerAccount is the maximum number of combined AppParams, AppLocalState, AssetParams, + // and AssetHolding resources per address that can be returned by the /v2/accounts endpoints. + // If an address exceeds this number, a 400 error is returned. Zero means unlimited. + MaxAPIResourcesPerAccount uint64 + + ///////////////////// + // Limit Constants // + ///////////////////// + + // Transactions + MaxTransactionsLimit uint64 + DefaultTransactionsLimit uint64 + + // Accounts + MaxAccountsLimit uint64 + DefaultAccountsLimit uint64 + + // Assets + MaxAssetsLimit uint64 + DefaultAssetsLimit uint64 + + // Asset Balances + MaxBalancesLimit uint64 + DefaultBalancesLimit uint64 + + // Applications + MaxApplicationsLimit uint64 + DefaultApplicationsLimit uint64 } func (e ExtraOptions) handlerTimeout() time.Duration { @@ -98,6 +127,7 @@ func Serve(ctx context.Context, serveAddr string, db idb.IndexerDb, fetcherError timeout: options.handlerTimeout(), log: log, disabledParams: disabledMap, + opts: options, } generated.RegisterHandlers(e, &api, middleware...) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index c0d63efbd..dba0b3ff5 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -24,19 +24,30 @@ import ( ) var ( - algodDataDir string - algodAddr string - algodToken string - daemonServerAddr string - noAlgod bool - developerMode bool - allowMigration bool - metricsMode string - tokenString string - writeTimeout time.Duration - readTimeout time.Duration - maxConn uint32 - enableAllParameters bool + algodDataDir string + algodAddr string + algodToken string + daemonServerAddr string + noAlgod bool + developerMode bool + allowMigration bool + metricsMode string + tokenString string + writeTimeout time.Duration + readTimeout time.Duration + maxConn uint32 + maxAPIResourcesPerAccount uint32 + maxTransactionsLimit uint32 + defaultTransactionsLimit uint32 + maxAccountsLimit uint32 + defaultAccountsLimit uint32 + maxAssetsLimit uint32 + defaultAssetsLimit uint32 + maxBalancesLimit uint32 + defaultBalancesLimit uint32 + maxApplicationsLimit uint32 + defaultApplicationsLimit uint32 + enableAllParameters bool ) const paramConfigEnableFlag = false @@ -168,6 +179,19 @@ func init() { daemonCmd.Flags().MarkHidden("enable-all-parameters") } + daemonCmd.Flags().Uint32VarP(&maxAPIResourcesPerAccount, "max-api-resources-per-account", "", 0, "set the maximum total number of resources (created assets, created apps, asset holdings, and application local state) per account that will be allowed in REST API lookupAccountByID and searchForAccounts responses before returning a 400 Bad Request. Set zero for no limit (default: unlimited)") + + daemonCmd.Flags().Uint32VarP(&maxTransactionsLimit, "max-transactions-limit", "", 10000, "set the maximum allowed Limit parameter for querying transactions") + daemonCmd.Flags().Uint32VarP(&defaultTransactionsLimit, "default-transactions-limit", "", 1000, "set the default Limit parameter for querying transactions, if none is provided") + daemonCmd.Flags().Uint32VarP(&maxAccountsLimit, "max-accounts-limit", "", 1000, "set the maximum allowed Limit parameter for querying accounts") + daemonCmd.Flags().Uint32VarP(&defaultAccountsLimit, "default-accounts-limit", "", 100, "set the default Limit parameter for querying accounts, if none is provided") + daemonCmd.Flags().Uint32VarP(&maxAssetsLimit, "max-assets-limit", "", 1000, "set the maximum allowed Limit parameter for querying assets") + daemonCmd.Flags().Uint32VarP(&defaultAssetsLimit, "default-assets-limit", "", 100, "set the default Limit parameter for querying assets, if none is provided") + daemonCmd.Flags().Uint32VarP(&maxBalancesLimit, "max-balances-limit", "", 10000, "set the maximum allowed Limit parameter for querying balances") + daemonCmd.Flags().Uint32VarP(&defaultBalancesLimit, "default-balances-limit", "", 1000, "set the default Limit parameter for querying balances, if none is provided") + daemonCmd.Flags().Uint32VarP(&maxApplicationsLimit, "max-applications-limit", "", 1000, "set the maximum allowed Limit parameter for querying applications") + daemonCmd.Flags().Uint32VarP(&defaultApplicationsLimit, "default-applications-limit", "", 100, "set the default Limit parameter for querying applications, if none is provided") + viper.RegisterAlias("algod", "algod-data-dir") viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") @@ -195,6 +219,18 @@ func makeOptions() (options api.ExtraOptions) { options.WriteTimeout = writeTimeout options.ReadTimeout = readTimeout + options.MaxAPIResourcesPerAccount = uint64(maxAPIResourcesPerAccount) + options.MaxTransactionsLimit = uint64(maxTransactionsLimit) + options.DefaultTransactionsLimit = uint64(defaultTransactionsLimit) + options.MaxAccountsLimit = uint64(maxAccountsLimit) + options.DefaultAccountsLimit = uint64(defaultAccountsLimit) + options.MaxAssetsLimit = uint64(maxAssetsLimit) + options.DefaultAssetsLimit = uint64(defaultAssetsLimit) + options.MaxBalancesLimit = uint64(maxBalancesLimit) + options.DefaultBalancesLimit = uint64(defaultBalancesLimit) + options.MaxApplicationsLimit = uint64(maxApplicationsLimit) + options.DefaultApplicationsLimit = uint64(defaultApplicationsLimit) + if paramConfigEnableFlag { if enableAllParameters { options.DisabledMapConfig = api.MakeDisabledMapConfig() diff --git a/cmd/block-generator/generator/generate.go b/cmd/block-generator/generator/generate.go index 6ed7eb4ea..9b221eaa9 100644 --- a/cmd/block-generator/generator/generate.go +++ b/cmd/block-generator/generator/generate.go @@ -681,7 +681,6 @@ func (g *generator) WriteAccount(output io.Writer, accountString string) error { assets = append(assets, generated.AssetHolding{ Amount: holding.balance, AssetId: a.assetID, - Creator: indexToAccount(a.creator).String(), IsFrozen: false, }) } diff --git a/cmd/idbtest/idbtest.go b/cmd/idbtest/idbtest.go index 1f945342c..465932287 100644 --- a/cmd/idbtest/idbtest.go +++ b/cmd/idbtest/idbtest.go @@ -134,7 +134,13 @@ func main() { <-availableCh if accounttest { - printAccountQuery(db, idb.AccountQueryOptions{IncludeAssetHoldings: true, IncludeAssetParams: true, AlgosGreaterThan: uint64Ptr(10000000000), Limit: 20}) + printAccountQuery(db, idb.AccountQueryOptions{ + IncludeAssetHoldings: true, + IncludeAssetParams: true, + IncludeAppLocalState: true, + IncludeAppParams: true, + AlgosGreaterThan: uint64Ptr(10000000000), + Limit: 20}) printAccountQuery(db, idb.AccountQueryOptions{HasAssetID: 312769, Limit: 19}) } if assettest { diff --git a/cmd/import-validator/README.md b/cmd/import-validator/README.md index 7a9e83000..aaffa31e7 100644 --- a/cmd/import-validator/README.md +++ b/cmd/import-validator/README.md @@ -1,14 +1,14 @@ # Import validation tool The import validator tool imports blocks into indexer database and algod's -sqlite database in lockstep and checks that the modified accounts are the same -in the two databases. +sqlite database in lockstep and checks that the modified accounts and resources +are the same in the two databases. It lets us detect the first round where an accounting discrepancy occurs and it prints out the difference before crashing. There is a small limitation, however. -The set of modified accounts is computed using the sqlite database. -Thus, if indexer's accounting were to modify a superset of those accounts, +The list of modified address and resources is computed using the sqlite database. +Thus, if indexer's accounting were to modify a superset of that data, this tool would not detect it. This, however, should be unlikely. @@ -45,5 +45,5 @@ or one round behind; otherwise, the import validator will fail to start. Reading and writing to/from the sqlite database is negligible compared to importing blocks into the postgres database. -However, the tool has to read the modified accounts after importing each block. -Thus, we can expect the import validator to be 1.5 times slower than indexer. +However, the tool has to read the modified state after importing each block. +Thus, we can expect the import validator to be about 1.5 times slower than indexer. diff --git a/cmd/import-validator/core/service.go b/cmd/import-validator/core/service.go index 27874faf6..66a91afc2 100644 --- a/cmd/import-validator/core/service.go +++ b/cmd/import-validator/core/service.go @@ -128,37 +128,91 @@ func openLedger(ledgerPath string, genesis *bookkeeping.Genesis, genesisBlock *b return ledger, nil } -func getModifiedAccounts(l *ledger.Ledger, block *bookkeeping.Block) ([]basics.Address, error) { +func getModifiedState(l *ledger.Ledger, block *bookkeeping.Block) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}, error) { eval, err := l.StartEvaluator(block.BlockHeader, len(block.Payset), 0) if err != nil { - return nil, fmt.Errorf("changedAccounts() start evaluator err: %w", err) + return nil, nil, fmt.Errorf("getModifiedState() start evaluator err: %w", err) } paysetgroups, err := block.DecodePaysetGroups() if err != nil { - return nil, fmt.Errorf("changedAccounts() decode payset groups err: %w", err) + return nil, nil, fmt.Errorf("getModifiedState() decode payset groups err: %w", err) } for _, group := range paysetgroups { err = eval.TransactionGroup(group) if err != nil { - return nil, fmt.Errorf("changedAccounts() apply transaction group err: %w", err) + return nil, nil, + fmt.Errorf("getModifiedState() apply transaction group err: %w", err) } } vb, err := eval.GenerateBlock() if err != nil { - return nil, fmt.Errorf("changedAccounts() generate block err: %w", err) + return nil, nil, fmt.Errorf("getModifiedState() generate block err: %w", err) } accountDeltas := vb.Delta().Accts - return accountDeltas.ModifiedAccounts(), nil + + modifiedAccounts := make(map[basics.Address]struct{}) + for _, address := range accountDeltas.ModifiedAccounts() { + modifiedAccounts[address] = struct{}{} + } + + modifiedResources := make(map[basics.Address]map[ledger.Creatable]struct{}) + for _, r := range accountDeltas.GetAllAssetResources() { + c, ok := modifiedResources[r.Addr] + if !ok { + c = make(map[ledger.Creatable]struct{}) + modifiedResources[r.Addr] = c + } + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(r.Aidx), + Type: basics.AssetCreatable, + } + c[creatable] = struct{}{} + } + for _, r := range accountDeltas.GetAllAppResources() { + c, ok := modifiedResources[r.Addr] + if !ok { + c = make(map[ledger.Creatable]struct{}) + modifiedResources[r.Addr] = c + } + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(r.Aidx), + Type: basics.AppCreatable, + } + c[creatable] = struct{}{} + } + + return modifiedAccounts, modifiedResources, nil +} + +func normalizeAccountResource(r ledgercore.AccountResource) ledgercore.AccountResource { + if (r.AppParams != nil) && (len(r.AppParams.GlobalState) == 0) { + // Make a copy of `AppParams` to avoid modifying ledger's storage. + appParams := new(basics.AppParams) + *appParams = *r.AppParams + appParams.GlobalState = nil + r.AppParams = appParams + } + if (r.AppLocalState != nil) && (len(r.AppLocalState.KeyValue) == 0) { + // Make a copy of `AppLocalState` to avoid modifying ledger's storage. + appLocalState := new(basics.AppLocalState) + *appLocalState = *r.AppLocalState + appLocalState.KeyValue = nil + r.AppLocalState = appLocalState + } + + return r } -func checkModifiedAccounts(db *postgres.IndexerDb, l *ledger.Ledger, block *bookkeeping.Block, addresses []basics.Address) error { - var accountsIndexer map[basics.Address]basics.AccountData +func checkModifiedState(db *postgres.IndexerDb, l *ledger.Ledger, block *bookkeeping.Block, addresses map[basics.Address]struct{}, resources map[basics.Address]map[ledger.Creatable]struct{}) error { + var accountsIndexer map[basics.Address]ledgercore.AccountData + var resourcesIndexer map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource var err0 error - var accountsAlgod map[basics.Address]basics.AccountData + var accountsAlgod map[basics.Address]ledgercore.AccountData + var resourcesAlgod map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource var err1 error var wg sync.WaitGroup @@ -166,64 +220,50 @@ func checkModifiedAccounts(db *postgres.IndexerDb, l *ledger.Ledger, block *book go func() { defer wg.Done() - accountsIndexer, err0 = db.GetAccountData(addresses) + var accounts map[basics.Address]*ledgercore.AccountData + accounts, resourcesIndexer, err0 = db.GetAccountState(addresses, resources) if err0 != nil { - err0 = fmt.Errorf("checkModifiedAccounts() err0: %w", err0) + err0 = fmt.Errorf("checkModifiedState() err0: %w", err0) return } + + accountsIndexer = make(map[basics.Address]ledgercore.AccountData, len(accounts)) + for address, accountData := range accounts { + if accountData == nil { + accountsIndexer[address] = ledgercore.AccountData{} + } else { + accountsIndexer[address] = *accountData + } + } }() wg.Add(1) go func() { defer wg.Done() - accountsAlgod = make(map[basics.Address]basics.AccountData, len(addresses)) - for _, address := range addresses { - var accountData basics.AccountData - accountData, _, err1 = l.LookupWithoutRewards(block.Round(), address) + accountsAlgod = make(map[basics.Address]ledgercore.AccountData, len(addresses)) + for address := range addresses { + accountsAlgod[address], _, err1 = l.LookupWithoutRewards(block.Round(), address) if err1 != nil { - err1 = fmt.Errorf("checkModifiedAccounts() lookup err1: %w", err1) + err1 = fmt.Errorf("checkModifiedState() lookup account err1: %w", err1) return } - - // Indexer returns nil for these maps if they are empty. Unfortunately, - // in go-algorand it's not well defined, and sometimes ledger returns empty - // maps and sometimes nil maps. So we set those maps to nil if they are empty so - // that comparison works. - if len(accountData.AssetParams) == 0 { - accountData.AssetParams = nil - } - if len(accountData.Assets) == 0 { - accountData.Assets = nil - } - - if accountData.AppParams != nil { - // Make a copy of `AppParams` to avoid modifying ledger's storage. - appParams := - make(map[basics.AppIndex]basics.AppParams, len(accountData.AppParams)) - for index, params := range accountData.AppParams { - if len(params.GlobalState) == 0 { - params.GlobalState = nil - } - appParams[index] = params - } - accountData.AppParams = appParams - } - - if accountData.AppLocalStates != nil { - // Make a copy of `AppLocalStates` to avoid modifying ledger's storage. - appLocalStates := - make(map[basics.AppIndex]basics.AppLocalState, len(accountData.AppLocalStates)) - for index, state := range accountData.AppLocalStates { - if len(state.KeyValue) == 0 { - state.KeyValue = nil - } - appLocalStates[index] = state + } + resourcesAlgod = + make(map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource) + for address, creatables := range resources { + resourcesForAddress := make(map[ledger.Creatable]ledgercore.AccountResource) + resourcesAlgod[address] = resourcesForAddress + for creatable := range creatables { + var resource ledgercore.AccountResource + resource, err1 = + l.LookupResource(block.Round(), address, creatable.Index, creatable.Type) + if err1 != nil { + err1 = fmt.Errorf("checkModifiedState() lookup resource err1: %w", err1) + return } - accountData.AppLocalStates = appLocalStates + resourcesForAddress[creatable] = normalizeAccountResource(resource) } - - accountsAlgod[address] = accountData } }() @@ -238,10 +278,17 @@ func checkModifiedAccounts(db *postgres.IndexerDb, l *ledger.Ledger, block *book if !reflect.DeepEqual(accountsIndexer, accountsAlgod) { diff := util.Diff(accountsAlgod, accountsIndexer) return fmt.Errorf( - "checkModifiedAccounts() accounts differ,"+ + "checkModifiedState() accounts differ,"+ "\naccountsIndexer: %+v,\naccountsAlgod: %+v,\ndiff: %s", accountsIndexer, accountsAlgod, diff) } + if !reflect.DeepEqual(resourcesIndexer, resourcesAlgod) { + diff := util.Diff(resourcesAlgod, resourcesIndexer) + return fmt.Errorf( + "checkModifiedState() resources differ,"+ + "\nresourcesIndexer: %+v,\nresourcesAlgod: %+v,\ndiff: %s", + resourcesIndexer, resourcesAlgod, diff) + } return nil } @@ -267,14 +314,15 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg } blockHandlerFunc := func(ctx context.Context, block *rpcs.EncodedBlockCert) error { - var modifiedAccounts []basics.Address + var modifiedAccounts map[basics.Address]struct{} + var modifiedResources map[basics.Address]map[ledger.Creatable]struct{} var err0 error var err1 error var wg sync.WaitGroup wg.Add(1) go func() { - modifiedAccounts, err0 = getModifiedAccounts(l, &block.Block) + modifiedAccounts, modifiedResources, err0 = getModifiedState(l, &block.Block) wg.Done() }() @@ -307,7 +355,8 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg } nextRoundLedger++ - return checkModifiedAccounts(db, l, &block.Block, modifiedAccounts) + return checkModifiedState( + db, l, &block.Block, modifiedAccounts, modifiedResources) } bot.SetBlockHandler(blockHandlerFunc) bot.SetNextRound(nextRoundLedger) diff --git a/idb/dummy/dummy.go b/idb/dummy/dummy.go index 29ca96e61..d4cc71980 100644 --- a/idb/dummy/dummy.go +++ b/idb/dummy/dummy.go @@ -7,7 +7,6 @@ import ( "github.com/algorand/go-algorand/data/transactions" log "github.com/sirupsen/logrus" - models "github.com/algorand/indexer/api/generated/v2" "github.com/algorand/indexer/idb" ) @@ -74,7 +73,12 @@ func (db *dummyIndexerDb) AssetBalances(ctx context.Context, abq idb.AssetBalanc } // Applications is part of idb.IndexerDB -func (db *dummyIndexerDb) Applications(ctx context.Context, filter *models.SearchForApplicationsParams) (<-chan idb.ApplicationRow, uint64) { +func (db *dummyIndexerDb) Applications(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.ApplicationRow, uint64) { + return nil, 0 +} + +// AppLocalState is part of idb.IndexerDB +func (db *dummyIndexerDb) AppLocalState(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.AppLocalStateRow, uint64) { return nil, 0 } diff --git a/idb/idb.go b/idb/idb.go index fd86382a7..82901ed1c 100644 --- a/idb/idb.go +++ b/idb/idb.go @@ -179,7 +179,8 @@ type IndexerDb interface { GetAccounts(ctx context.Context, opts AccountQueryOptions) (<-chan AccountRow, uint64) Assets(ctx context.Context, filter AssetsQuery) (<-chan AssetRow, uint64) AssetBalances(ctx context.Context, abq AssetBalanceQuery) (<-chan AssetBalanceRow, uint64) - Applications(ctx context.Context, filter *models.SearchForApplicationsParams) (<-chan ApplicationRow, uint64) + Applications(ctx context.Context, filter ApplicationQuery) (<-chan ApplicationRow, uint64) + AppLocalState(ctx context.Context, filter ApplicationQuery) (<-chan AppLocalStateRow, uint64) Health(ctx context.Context) (status Health, err error) } @@ -259,6 +260,11 @@ type AccountQueryOptions struct { IncludeAssetHoldings bool IncludeAssetParams bool + IncludeAppLocalState bool + IncludeAppParams bool + + // MaxResources is the maximum combined number of AppParam, AppLocalState, AssetParam, and AssetHolding objects allowed. + MaxResources uint64 // IncludeDeleted indicated whether to include deleted Assets, Applications, etc within the account. IncludeDeleted bool @@ -269,7 +275,18 @@ type AccountQueryOptions struct { // AccountRow is metadata relating to one account in a account query. type AccountRow struct { Account models.Account - Error error + Error error // could be MaxAPIResourcesPerAccountError +} + +// MaxAPIResourcesPerAccountError records the offending address and resource count that exceeded the limit. +type MaxAPIResourcesPerAccountError struct { + Address basics.Address + + TotalAppLocalStates, TotalAppParams, TotalAssets, TotalAssetParams uint64 +} + +func (e MaxAPIResourcesPerAccountError) Error() string { + return "Max accounts API results limit exceeded" } // AssetsQuery is a parameter object with all of the asset filter options. @@ -306,9 +323,12 @@ type AssetRow struct { // AssetBalanceQuery is a parameter object with all of the asset balance filter options. type AssetBalanceQuery struct { - AssetID uint64 - AmountGT *uint64 // only rows > this - AmountLT *uint64 // only rows < this + AssetID uint64 + AssetIDGT uint64 + AmountGT *uint64 // only rows > this + AmountLT *uint64 // only rows < this + + Address []byte // IncludeDeleted indicated whether to include deleted AssetHoldingss in the results. IncludeDeleted bool @@ -332,12 +352,27 @@ type AssetBalanceRow struct { Deleted *bool } -// ApplicationRow is metadata relating to one application in an application query. +// ApplicationRow is metadata and global state (AppParams) relating to one application in an application query. type ApplicationRow struct { Application models.Application Error error } +// ApplicationQuery is a parameter object used for query local and global application state. +type ApplicationQuery struct { + Address []byte + ApplicationID uint64 + ApplicationIDGreaterThan uint64 + IncludeDeleted bool + Limit uint64 +} + +// AppLocalStateRow is metadata and local state (AppLocalState) relating to one application in an application query. +type AppLocalStateRow struct { + AppLocalState models.ApplicationLocalState + Error error +} + // IndexerDbOptions are the options common to all indexer backends. type IndexerDbOptions struct { ReadOnly bool diff --git a/idb/mocks/IndexerDb.go b/idb/mocks/IndexerDb.go index 1a110084a..0de088c21 100644 --- a/idb/mocks/IndexerDb.go +++ b/idb/mocks/IndexerDb.go @@ -7,8 +7,6 @@ import ( bookkeeping "github.com/algorand/go-algorand/data/bookkeeping" - generated "github.com/algorand/indexer/api/generated/v2" - idb "github.com/algorand/indexer/idb" mock "github.com/stretchr/testify/mock" @@ -35,12 +33,35 @@ func (_m *IndexerDb) AddBlock(block *bookkeeping.Block) error { return r0 } +// AppLocalState provides a mock function with given fields: ctx, filter +func (_m *IndexerDb) AppLocalState(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.AppLocalStateRow, uint64) { + ret := _m.Called(ctx, filter) + + var r0 <-chan idb.AppLocalStateRow + if rf, ok := ret.Get(0).(func(context.Context, idb.ApplicationQuery) <-chan idb.AppLocalStateRow); ok { + r0 = rf(ctx, filter) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan idb.AppLocalStateRow) + } + } + + var r1 uint64 + if rf, ok := ret.Get(1).(func(context.Context, idb.ApplicationQuery) uint64); ok { + r1 = rf(ctx, filter) + } else { + r1 = ret.Get(1).(uint64) + } + + return r0, r1 +} + // Applications provides a mock function with given fields: ctx, filter -func (_m *IndexerDb) Applications(ctx context.Context, filter *generated.SearchForApplicationsParams) (<-chan idb.ApplicationRow, uint64) { +func (_m *IndexerDb) Applications(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.ApplicationRow, uint64) { ret := _m.Called(ctx, filter) var r0 <-chan idb.ApplicationRow - if rf, ok := ret.Get(0).(func(context.Context, *generated.SearchForApplicationsParams) <-chan idb.ApplicationRow); ok { + if rf, ok := ret.Get(0).(func(context.Context, idb.ApplicationQuery) <-chan idb.ApplicationRow); ok { r0 = rf(ctx, filter) } else { if ret.Get(0) != nil { @@ -49,7 +70,7 @@ func (_m *IndexerDb) Applications(ctx context.Context, filter *generated.SearchF } var r1 uint64 - if rf, ok := ret.Get(1).(func(context.Context, *generated.SearchForApplicationsParams) uint64); ok { + if rf, ok := ret.Get(1).(func(context.Context, idb.ApplicationQuery) uint64); ok { r1 = rf(ctx, filter) } else { r1 = ret.Get(1).(uint64) diff --git a/idb/postgres/internal/encoding/encoding.go b/idb/postgres/internal/encoding/encoding.go index b792cae86..6f6bf35dd 100644 --- a/idb/postgres/internal/encoding/encoding.go +++ b/idb/postgres/internal/encoding/encoding.go @@ -366,20 +366,6 @@ func DecodeSignedTxnWithAD(data []byte) (transactions.SignedTxnWithAD, error) { return unconvertSignedTxnWithAD(stxn), nil } -// TrimAccountData deletes various information from account data that we do not write to -// `account.account_data`. -func TrimAccountData(ad basics.AccountData) basics.AccountData { - ad.MicroAlgos = basics.MicroAlgos{} - ad.RewardsBase = 0 - ad.RewardedMicroAlgos = basics.MicroAlgos{} - ad.AssetParams = nil - ad.Assets = nil - ad.AppLocalStates = nil - ad.AppParams = nil - - return ad -} - func convertTrimmedAccountData(ad basics.AccountData) trimmedAccountData { return trimmedAccountData{ AccountData: ad, @@ -683,3 +669,72 @@ func DecodeNetworkState(data []byte) (types.NetworkState, error) { return state, nil } + +// TrimLcAccountData deletes various information from account data that we do not write +// to `account.account_data`. +func TrimLcAccountData(ad ledgercore.AccountData) ledgercore.AccountData { + ad.MicroAlgos = basics.MicroAlgos{} + ad.RewardsBase = 0 + ad.RewardedMicroAlgos = basics.MicroAlgos{} + return ad +} + +func convertTrimmedLcAccountData(ad ledgercore.AccountData) baseAccountData { + return baseAccountData{ + Status: ad.Status, + AuthAddr: crypto.Digest(ad.AuthAddr), + TotalAppSchema: ad.TotalAppSchema, + TotalExtraAppPages: ad.TotalExtraAppPages, + TotalAssetParams: ad.TotalAssetParams, + TotalAssets: ad.TotalAssets, + TotalAppParams: ad.TotalAppParams, + TotalAppLocalStates: ad.TotalAppLocalStates, + baseOnlineAccountData: baseOnlineAccountData{ + VoteID: ad.VoteID, + SelectionID: ad.SelectionID, + StateProofID: ad.StateProofID, + VoteFirstValid: ad.VoteFirstValid, + VoteLastValid: ad.VoteLastValid, + VoteKeyDilution: ad.VoteKeyDilution, + }, + } +} + +func unconvertTrimmedLcAccountData(ba baseAccountData) ledgercore.AccountData { + return ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + Status: ba.Status, + AuthAddr: basics.Address(ba.AuthAddr), + TotalAppSchema: ba.TotalAppSchema, + TotalExtraAppPages: ba.TotalExtraAppPages, + TotalAppParams: ba.TotalAppParams, + TotalAppLocalStates: ba.TotalAppLocalStates, + TotalAssetParams: ba.TotalAssetParams, + TotalAssets: ba.TotalAssets, + }, + VotingData: ledgercore.VotingData{ + VoteID: ba.VoteID, + SelectionID: ba.SelectionID, + StateProofID: ba.StateProofID, + VoteFirstValid: ba.VoteFirstValid, + VoteLastValid: ba.VoteLastValid, + VoteKeyDilution: ba.VoteKeyDilution, + }, + } +} + +// EncodeTrimmedLcAccountData encodes ledgercore account data into json. +func EncodeTrimmedLcAccountData(ad ledgercore.AccountData) []byte { + return encodeJSON(convertTrimmedLcAccountData(ad)) +} + +// DecodeTrimmedLcAccountData decodes ledgercore account data from json. +func DecodeTrimmedLcAccountData(data []byte) (ledgercore.AccountData, error) { + var ba baseAccountData + err := DecodeJSON(data, &ba) + if err != nil { + return ledgercore.AccountData{}, err + } + + return unconvertTrimmedLcAccountData(ba), nil +} diff --git a/idb/postgres/internal/encoding/encoding_test.go b/idb/postgres/internal/encoding/encoding_test.go index e02062861..0a3816e85 100644 --- a/idb/postgres/internal/encoding/encoding_test.go +++ b/idb/postgres/internal/encoding/encoding_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" @@ -465,20 +466,21 @@ func TestSpecialAddressesEncoding(t *testing.T) { // Test that encoding of AccountTotals is as expected and that decoding results in the // same object. func TestAccountTotalsEncoding(t *testing.T) { + random := rand.New(rand.NewSource(1)) totals := ledgercore.AccountTotals{ Online: ledgercore.AlgoCount{ - Money: basics.MicroAlgos{Raw: rand.Uint64()}, - RewardUnits: rand.Uint64(), + Money: basics.MicroAlgos{Raw: random.Uint64()}, + RewardUnits: random.Uint64(), }, Offline: ledgercore.AlgoCount{ - Money: basics.MicroAlgos{Raw: rand.Uint64()}, - RewardUnits: rand.Uint64(), + Money: basics.MicroAlgos{Raw: random.Uint64()}, + RewardUnits: random.Uint64(), }, NotParticipating: ledgercore.AlgoCount{ - Money: basics.MicroAlgos{Raw: rand.Uint64()}, - RewardUnits: rand.Uint64(), + Money: basics.MicroAlgos{Raw: random.Uint64()}, + RewardUnits: random.Uint64(), }, - RewardsLevel: rand.Uint64(), + RewardsLevel: random.Uint64(), } buf := EncodeAccountTotals(&totals) @@ -549,3 +551,51 @@ func TestNetworkStateEncoding(t *testing.T) { require.NoError(t, err) assert.Equal(t, network, decodedNetwork) } + +// Test that encoding of ledgercore.AccountData is as expected and that decoding +// results in the same object. +func TestLcAccountDataEncoding(t *testing.T) { + var authAddr basics.Address + authAddr[0] = 6 + + var voteID crypto.OneTimeSignatureVerifier + voteID[0] = 14 + + var selectionID crypto.VRFVerifier + selectionID[0] = 15 + + var stateProofID merklesignature.Verifier + stateProofID[0] = 19 + + ad := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + Status: basics.Online, + AuthAddr: authAddr, + TotalAppSchema: basics.StateSchema{ + NumUint: 7, + NumByteSlice: 8, + }, + TotalExtraAppPages: 9, + TotalAppParams: 10, + TotalAppLocalStates: 11, + TotalAssetParams: 12, + TotalAssets: 13, + }, + VotingData: ledgercore.VotingData{ + VoteID: voteID, + SelectionID: selectionID, + StateProofID: stateProofID, + VoteFirstValid: 16, + VoteLastValid: 17, + VoteKeyDilution: 18, + }, + } + buf := EncodeTrimmedLcAccountData(ad) + + expectedString := `{"onl":1,"sel":"DwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","spend":"BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","stprf":"EwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==","tapl":11,"tapp":10,"tas":13,"tasp":12,"teap":9,"tsch":{"nbs":8,"nui":7},"vote":"DgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","voteFst":16,"voteKD":18,"voteLst":17}` + assert.Equal(t, expectedString, string(buf)) + + decodedAd, err := DecodeTrimmedLcAccountData(buf) + require.NoError(t, err) + assert.Equal(t, ad, decodedAd) +} diff --git a/idb/postgres/internal/encoding/types.go b/idb/postgres/internal/encoding/types.go index d2e4c8987..259be6aac 100644 --- a/idb/postgres/internal/encoding/types.go +++ b/idb/postgres/internal/encoding/types.go @@ -2,6 +2,7 @@ package encoding import ( "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" @@ -106,3 +107,29 @@ type specialAddresses struct { FeeSinkOverride crypto.Digest `codec:"FeeSink"` RewardsPoolOverride crypto.Digest `codec:"RewardsPool"` } + +type baseOnlineAccountData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + VoteID crypto.OneTimeSignatureVerifier `codec:"vote"` + SelectionID crypto.VRFVerifier `codec:"sel"` + StateProofID merklesignature.Verifier `codec:"stprf"` + VoteFirstValid basics.Round `codec:"voteFst"` + VoteLastValid basics.Round `codec:"voteLst"` + VoteKeyDilution uint64 `codec:"voteKD"` +} + +type baseAccountData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Status basics.Status `codec:"onl"` + AuthAddr crypto.Digest `codec:"spend"` + TotalAppSchema basics.StateSchema `codec:"tsch"` + TotalExtraAppPages uint32 `codec:"teap"` + TotalAssetParams uint64 `codec:"tasp"` + TotalAssets uint64 `codec:"tas"` + TotalAppParams uint64 `codec:"tapp"` + TotalAppLocalStates uint64 `codec:"tapl"` + + baseOnlineAccountData +} diff --git a/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator.go b/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator.go index 005f27bc8..fbd275dd6 100644 --- a/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator.go +++ b/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator.go @@ -15,15 +15,15 @@ import ( ) const ( - blockHeaderStmtName = "block_header" - assetCreatorStmtName = "asset_creator" - appCreatorStmtName = "app_creator" - accountStmtName = "account" - assetHoldingsStmtName = "asset_holdings" - assetParamsStmtName = "asset_params" - appParamsStmtName = "app_params" - appLocalStatesStmtName = "app_local_states" - accountTotalsStmtName = "account_totals" + blockHeaderStmtName = "block_header" + assetCreatorStmtName = "asset_creator" + appCreatorStmtName = "app_creator" + accountStmtName = "account" + assetHoldingStmtName = "asset_holding" + assetParamsStmtName = "asset_params" + appParamsStmtName = "app_params" + appLocalStateStmtName = "app_local_state" + accountTotalsStmtName = "account_totals" ) var statements = map[string]string{ @@ -33,13 +33,13 @@ var statements = map[string]string{ appCreatorStmtName: "SELECT creator FROM app WHERE index = $1 AND NOT deleted", accountStmtName: "SELECT microalgos, rewardsbase, rewards_total, account_data " + "FROM account WHERE addr = $1 AND NOT deleted", - assetHoldingsStmtName: "SELECT assetid, amount, frozen FROM account_asset " + - "WHERE addr = $1 AND NOT deleted", - assetParamsStmtName: "SELECT index, params FROM asset " + - "WHERE creator_addr = $1 AND NOT deleted", - appParamsStmtName: "SELECT index, params FROM app WHERE creator = $1 AND NOT deleted", - appLocalStatesStmtName: "SELECT app, localstate FROM account_app " + - "WHERE addr = $1 AND NOT deleted", + assetHoldingStmtName: "SELECT amount, frozen FROM account_asset " + + "WHERE addr = $1 AND assetid = $2 AND NOT deleted", + assetParamsStmtName: "SELECT creator_addr, params FROM asset " + + "WHERE index = $1 AND NOT deleted", + appParamsStmtName: "SELECT creator, params FROM app WHERE index = $1 AND NOT deleted", + appLocalStateStmtName: "SELECT localstate FROM account_app " + + "WHERE addr = $1 AND app = $2 AND NOT deleted", accountTotalsStmtName: `SELECT v FROM metastate WHERE k = '` + schema.AccountTotals + `'`, } @@ -94,7 +94,7 @@ func (l LedgerForEvaluator) LatestBlockHdr() (bookkeeping.BlockHeader, error) { return res, nil } -func (l *LedgerForEvaluator) parseAccountTable(row pgx.Row) (basics.AccountData, bool /*exists*/, error) { +func (l *LedgerForEvaluator) parseAccountTable(row pgx.Row) (ledgercore.AccountData, bool /*exists*/, error) { var microalgos uint64 var rewardsbase uint64 var rewardsTotal uint64 @@ -102,17 +102,17 @@ func (l *LedgerForEvaluator) parseAccountTable(row pgx.Row) (basics.AccountData, err := row.Scan(µalgos, &rewardsbase, &rewardsTotal, &accountData) if err == pgx.ErrNoRows { - return basics.AccountData{}, false, nil + return ledgercore.AccountData{}, false, nil } if err != nil { - return basics.AccountData{}, false, fmt.Errorf("parseAccountTable() scan row err: %w", err) + return ledgercore.AccountData{}, false, fmt.Errorf("parseAccountTable() scan row err: %w", err) } - res := basics.AccountData{} + var res ledgercore.AccountData if accountData != nil { - res, err = encoding.DecodeTrimmedAccountData(accountData) + res, err = encoding.DecodeTrimmedLcAccountData(accountData) if err != nil { - return basics.AccountData{}, false, + return ledgercore.AccountData{}, false, fmt.Errorf("parseAccountTable() decode account data err: %w", err) } } @@ -124,258 +124,300 @@ func (l *LedgerForEvaluator) parseAccountTable(row pgx.Row) (basics.AccountData, return res, true, nil } -func (l *LedgerForEvaluator) parseAccountAssetTable(rows pgx.Rows) (map[basics.AssetIndex]basics.AssetHolding, error) { - defer rows.Close() - var res map[basics.AssetIndex]basics.AssetHolding - - var assetid uint64 - var amount uint64 - var frozen bool - - for rows.Next() { - err := rows.Scan(&assetid, &amount, &frozen) - if err != nil { - return nil, fmt.Errorf("parseAccountAssetTable() scan row err: %w", err) - } - - if res == nil { - res = make(map[basics.AssetIndex]basics.AssetHolding) - } - res[basics.AssetIndex(assetid)] = basics.AssetHolding{ - Amount: amount, - Frozen: frozen, - } +// LookupWithoutRewards is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LookupWithoutRewards(addresses map[basics.Address]struct{}) (map[basics.Address]*ledgercore.AccountData, error) { + addressesArr := make([]basics.Address, 0, len(addresses)) + for address := range addresses { + addressesArr = append(addressesArr, address) } - err := rows.Err() - if err != nil { - return nil, fmt.Errorf("parseAccountAssetTable() scan end err: %w", err) + var batch pgx.Batch + for i := range addressesArr { + batch.Queue(accountStmtName, addressesArr[i][:]) } - return res, nil -} + results := l.tx.SendBatch(context.Background(), &batch) + defer results.Close() -func (l *LedgerForEvaluator) parseAssetTable(rows pgx.Rows) (map[basics.AssetIndex]basics.AssetParams, error) { - defer rows.Close() - var res map[basics.AssetIndex]basics.AssetParams + res := make(map[basics.Address]*ledgercore.AccountData, len(addresses)) + for _, address := range addressesArr { + row := results.QueryRow() - var index uint64 - var params []byte + lcAccountData := new(ledgercore.AccountData) + var exists bool + var err error - for rows.Next() { - err := rows.Scan(&index, ¶ms) + *lcAccountData, exists, err = l.parseAccountTable(row) if err != nil { - return nil, fmt.Errorf("parseAssetTable() scan row err: %w", err) + return nil, fmt.Errorf("LookupWithoutRewards() err: %w", err) } - if res == nil { - res = make(map[basics.AssetIndex]basics.AssetParams) - } - res[basics.AssetIndex(index)], err = encoding.DecodeAssetParams(params) - if err != nil { - return nil, fmt.Errorf("parseAssetTable() decode params err: %w", err) + if exists { + res[address] = lcAccountData + } else { + res[address] = nil } } - err := rows.Err() + err := results.Close() if err != nil { - return nil, fmt.Errorf("parseAssetTable() scan end err: %w", err) + return nil, fmt.Errorf("LookupWithoutRewards() close results err: %w", err) } return res, nil } -func (l *LedgerForEvaluator) parseAppTable(rows pgx.Rows) (map[basics.AppIndex]basics.AppParams, error) { - defer rows.Close() - var res map[basics.AppIndex]basics.AppParams - - var index uint64 - var params []byte - - for rows.Next() { - err := rows.Scan(&index, ¶ms) - if err != nil { - return nil, fmt.Errorf("parseAppTable() scan row err: %w", err) - } +func (l *LedgerForEvaluator) parseAccountAssetTable(row pgx.Row) (basics.AssetHolding, bool /*exists*/, error) { + var amount uint64 + var frozen bool - if res == nil { - res = make(map[basics.AppIndex]basics.AppParams) - } - res[basics.AppIndex(index)], err = encoding.DecodeAppParams(params) - if err != nil { - return nil, fmt.Errorf("parseAppTable() decode params err: %w", err) - } + err := row.Scan(&amount, &frozen) + if err == pgx.ErrNoRows { + return basics.AssetHolding{}, false, nil } - - err := rows.Err() if err != nil { - return nil, fmt.Errorf("parseAppTable() scan end err: %w", err) + return basics.AssetHolding{}, false, + fmt.Errorf("parseAccountAssetTable() scan row err: %w", err) } - return res, nil + assetHolding := basics.AssetHolding{ + Amount: amount, + Frozen: frozen, + } + return assetHolding, true, nil } -func (l *LedgerForEvaluator) parseAccountAppTable(rows pgx.Rows) (map[basics.AppIndex]basics.AppLocalState, error) { - defer rows.Close() - var res map[basics.AppIndex]basics.AppLocalState - - var app uint64 - var localstate []byte - - for rows.Next() { - err := rows.Scan(&app, &localstate) - if err != nil { - return nil, fmt.Errorf("parseAccountAppTable() scan row err: %w", err) - } +func (l *LedgerForEvaluator) parseAssetTable(row pgx.Row) (basics.Address /*creator*/, basics.AssetParams, bool /*exists*/, error) { + var creatorAddr []byte + var params []byte - if res == nil { - res = make(map[basics.AppIndex]basics.AppLocalState) - } - res[basics.AppIndex(app)], err = encoding.DecodeAppLocalState(localstate) - if err != nil { - return nil, fmt.Errorf("parseAccountAppTable() decode local state err: %w", err) - } + err := row.Scan(&creatorAddr, ¶ms) + if err == pgx.ErrNoRows { + return basics.Address{}, basics.AssetParams{}, false, nil } + if err != nil { + return basics.Address{}, basics.AssetParams{}, false, + fmt.Errorf("parseAssetTable() scan row err: %w", err) + } + + var creator basics.Address + copy(creator[:], creatorAddr) - err := rows.Err() + assetParams, err := encoding.DecodeAssetParams(params) if err != nil { - return nil, fmt.Errorf("parseAccountAppTable() scan end err: %w", err) + return basics.Address{}, basics.AssetParams{}, false, + fmt.Errorf("parseAssetTable() decode params err: %w", err) } - return res, nil + return creator, assetParams, true, nil } -// Load rows from the account table for the given addresses except the special accounts. -// nil is stored for those accounts that were not found. Uses batching. -func (l *LedgerForEvaluator) loadAccountTable(addresses map[basics.Address]struct{}) (map[basics.Address]*basics.AccountData, error) { - addressesArr := make([]basics.Address, 0, len(addresses)) - for address := range addresses { - addressesArr = append(addressesArr, address) - } +func (l *LedgerForEvaluator) parseAppTable(row pgx.Row) (basics.Address /*creator*/, basics.AppParams, bool /*exists*/, error) { + var creatorAddr []byte + var params []byte - var batch pgx.Batch - for i := range addressesArr { - batch.Queue(accountStmtName, addressesArr[i][:]) + err := row.Scan(&creatorAddr, ¶ms) + if err == pgx.ErrNoRows { + return basics.Address{}, basics.AppParams{}, false, nil + } + if err != nil { + return basics.Address{}, basics.AppParams{}, false, + fmt.Errorf("parseAppTable() scan row err: %w", err) } - results := l.tx.SendBatch(context.Background(), &batch) - defer results.Close() + var creator basics.Address + copy(creator[:], creatorAddr) - res := make(map[basics.Address]*basics.AccountData, len(addresses)) - for _, address := range addressesArr { - row := results.QueryRow() + appParams, err := encoding.DecodeAppParams(params) + if err != nil { + return basics.Address{}, basics.AppParams{}, false, + fmt.Errorf("parseAppTable() decode params err: %w", err) + } - accountData := new(basics.AccountData) - var exists bool - var err error + return creator, appParams, true, nil +} - *accountData, exists, err = l.parseAccountTable(row) - if err != nil { - return nil, fmt.Errorf("loadAccountTable() err: %w", err) - } +func (l *LedgerForEvaluator) parseAccountAppTable(row pgx.Row) (basics.AppLocalState, bool /*exists*/, error) { + var localstate []byte - if exists { - res[address] = accountData - } else { - res[address] = nil - } + err := row.Scan(&localstate) + if err == pgx.ErrNoRows { + return basics.AppLocalState{}, false, nil + } + if err != nil { + return basics.AppLocalState{}, false, + fmt.Errorf("parseAccountAppTable() scan row err: %w", err) } - err := results.Close() + appLocalState, err := encoding.DecodeAppLocalState(localstate) if err != nil { - return nil, fmt.Errorf("loadAccountTable() close results err: %w", err) + return basics.AppLocalState{}, false, + fmt.Errorf("parseAccountAppTable() decode local state err: %w", err) } - return res, nil + return appLocalState, true, nil } -// Load all creatables for the non-nil account data from the provided map into that -// account data. Uses batching. -func (l *LedgerForEvaluator) loadCreatables(accountDataMap *map[basics.Address]*basics.AccountData) error { - var batch pgx.Batch - - existingAddresses := make([]basics.Address, 0, len(*accountDataMap)) - for address, accountData := range *accountDataMap { - if accountData != nil { - existingAddresses = append(existingAddresses, address) +// LookupResources is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LookupResources(input map[basics.Address]map[ledger.Creatable]struct{}) (map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { + // Create request arrays since iterating over maps is non-deterministic. + type AddrID struct { + addr basics.Address + id basics.CreatableIndex + } + // Asset holdings to request. + assetHoldingsReq := make([]AddrID, 0, len(input)) + // Asset params to request. + assetParamsReq := make([]basics.CreatableIndex, 0, len(input)) + // For each asset id, record for which addresses it was requested. + assetParamsToAddresses := make(map[basics.CreatableIndex]map[basics.Address]struct{}) + // App local states to request. + appLocalStatesReq := make([]AddrID, 0, len(input)) + // App params to request. + appParamsReq := make([]basics.CreatableIndex, 0, len(input)) + // For each app id, record for which addresses it was requested. + appParamsToAddresses := make(map[basics.CreatableIndex]map[basics.Address]struct{}) + + for address, creatables := range input { + for creatable := range creatables { + switch creatable.Type { + case basics.AssetCreatable: + assetHoldingsReq = append(assetHoldingsReq, AddrID{address, creatable.Index}) + if addresses, ok := assetParamsToAddresses[creatable.Index]; ok { + addresses[address] = struct{}{} + } else { + assetParamsReq = append(assetParamsReq, creatable.Index) + addresses = make(map[basics.Address]struct{}) + addresses[address] = struct{}{} + assetParamsToAddresses[creatable.Index] = addresses + } + case basics.AppCreatable: + appLocalStatesReq = append(appLocalStatesReq, AddrID{address, creatable.Index}) + if addresses, ok := appParamsToAddresses[creatable.Index]; ok { + addresses[address] = struct{}{} + } else { + appParamsReq = append(appParamsReq, creatable.Index) + addresses = make(map[basics.Address]struct{}) + addresses[address] = struct{}{} + appParamsToAddresses[creatable.Index] = addresses + } + default: + return nil, fmt.Errorf( + "LookupResources() unknown creatable type %d", creatable.Type) + } } } - for i := range existingAddresses { - batch.Queue(assetHoldingsStmtName, existingAddresses[i][:]) + // Prepare a batch of sql queries. + var batch pgx.Batch + for i := range assetHoldingsReq { + batch.Queue( + assetHoldingStmtName, assetHoldingsReq[i].addr[:], assetHoldingsReq[i].id) } - for i := range existingAddresses { - batch.Queue(assetParamsStmtName, existingAddresses[i][:]) + for _, cidx := range assetParamsReq { + batch.Queue(assetParamsStmtName, cidx) } - for i := range existingAddresses { - batch.Queue(appParamsStmtName, existingAddresses[i][:]) + for _, cidx := range appParamsReq { + batch.Queue(appParamsStmtName, cidx) } - for i := range existingAddresses { - batch.Queue(appLocalStatesStmtName, existingAddresses[i][:]) + for i := range appLocalStatesReq { + batch.Queue( + appLocalStateStmtName, appLocalStatesReq[i].addr[:], appLocalStatesReq[i].id) } + // Execute the sql queries. results := l.tx.SendBatch(context.Background(), &batch) defer results.Close() - for _, address := range existingAddresses { - rows, err := results.Query() - if err != nil { - return fmt.Errorf("loadCreatables() query asset holdings err: %w", err) + // Initialize the result `res` with the same structure as `input`. + res := make( + map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, len(input)) + for address, creatables := range input { + creatablesOutput := + make(map[ledger.Creatable]ledgercore.AccountResource, len(creatables)) + res[address] = creatablesOutput + for creatable := range creatables { + creatablesOutput[creatable] = ledgercore.AccountResource{} } - (*accountDataMap)[address].Assets, err = l.parseAccountAssetTable(rows) + } + + // Parse sql query results in the same order the queries were made. + for _, addrID := range assetHoldingsReq { + row := results.QueryRow() + assetHolding, exists, err := l.parseAccountAssetTable(row) if err != nil { - return fmt.Errorf("loadCreatables() err: %w", err) + return nil, fmt.Errorf("LookupResources() err: %w", err) + } + if exists { + creatable := ledger.Creatable{ + Index: addrID.id, + Type: basics.AssetCreatable, + } + resource := res[addrID.addr][creatable] + resource.AssetHolding = new(basics.AssetHolding) + *resource.AssetHolding = assetHolding + res[addrID.addr][creatable] = resource } } - for _, address := range existingAddresses { - rows, err := results.Query() + for _, cidx := range assetParamsReq { + row := results.QueryRow() + creator, assetParams, exists, err := l.parseAssetTable(row) if err != nil { - return fmt.Errorf("loadCreatables() query asset params err: %w", err) + return nil, fmt.Errorf("LookupResources() err: %w", err) } - (*accountDataMap)[address].AssetParams, err = l.parseAssetTable(rows) - if err != nil { - return fmt.Errorf("loadCreatables() err: %w", err) + if exists { + if _, ok := assetParamsToAddresses[cidx][creator]; ok { + creatable := ledger.Creatable{ + Index: cidx, + Type: basics.AssetCreatable, + } + resource := res[creator][creatable] + resource.AssetParams = new(basics.AssetParams) + *resource.AssetParams = assetParams + res[creator][creatable] = resource + } } } - for _, address := range existingAddresses { - rows, err := results.Query() + for _, cidx := range appParamsReq { + row := results.QueryRow() + creator, appParams, exists, err := l.parseAppTable(row) if err != nil { - return fmt.Errorf("loadCreatables() query app params err: %w", err) + return nil, fmt.Errorf("LookupResources() err: %w", err) } - (*accountDataMap)[address].AppParams, err = l.parseAppTable(rows) - if err != nil { - return fmt.Errorf("loadCreatables() err: %w", err) + if exists { + if _, ok := appParamsToAddresses[cidx][creator]; ok { + creatable := ledger.Creatable{ + Index: cidx, + Type: basics.AppCreatable, + } + resource := res[creator][creatable] + resource.AppParams = new(basics.AppParams) + *resource.AppParams = appParams + res[creator][creatable] = resource + } } } - for _, address := range existingAddresses { - rows, err := results.Query() + for _, addrID := range appLocalStatesReq { + row := results.QueryRow() + appLocalState, exists, err := l.parseAccountAppTable(row) if err != nil { - return fmt.Errorf("loadCreatables() query app local states err: %w", err) + return nil, fmt.Errorf("LookupResources() err: %w", err) } - (*accountDataMap)[address].AppLocalStates, err = l.parseAccountAppTable(rows) - if err != nil { - return fmt.Errorf("loadCreatables() err: %w", err) + if exists { + creatable := ledger.Creatable{ + Index: addrID.id, + Type: basics.AppCreatable, + } + resource := res[addrID.addr][creatable] + resource.AppLocalState = new(basics.AppLocalState) + *resource.AppLocalState = appLocalState + res[addrID.addr][creatable] = resource } } err := results.Close() if err != nil { - return fmt.Errorf("loadCreatables() close results err: %w", err) - } - - return nil -} - -// LookupWithoutRewards is part of go-algorand's indexerLedgerForEval interface. -func (l LedgerForEvaluator) LookupWithoutRewards(addresses map[basics.Address]struct{}) (map[basics.Address]*basics.AccountData, error) { - res, err := l.loadAccountTable(addresses) - if err != nil { - return nil, fmt.Errorf("loadAccounts() err: %w", err) - } - - err = l.loadCreatables(&res) - if err != nil { - return nil, fmt.Errorf("loadAccounts() err: %w", err) + return nil, fmt.Errorf("LookupResources() close results err: %w", err) } return res, nil diff --git a/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator_test.go b/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator_test.go index 57e195fe1..cdccf7611 100644 --- a/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator_test.go +++ b/idb/postgres/internal/ledger_for_evaluator/ledger_for_evaluator_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" @@ -26,17 +27,8 @@ import ( var readonlyRepeatableRead = pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly} -func setupPostgres(t *testing.T) (*pgxpool.Pool, func()) { - db, _, shutdownFunc := pgtest.SetupPostgres(t) - - _, err := db.Exec(context.Background(), schema.SetupPostgresSql) - require.NoError(t, err) - - return db, shutdownFunc -} - func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -65,7 +57,7 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { } func TestLedgerForEvaluatorAccountTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := `INSERT INTO account @@ -76,26 +68,33 @@ func TestLedgerForEvaluatorAccountTableBasic(t *testing.T) { voteID[0] = 2 var selectionID crypto.VRFVerifier selectionID[0] = 3 - accountDataWritten := basics.AccountData{ - Status: basics.Online, - VoteID: voteID, - SelectionID: selectionID, - VoteFirstValid: basics.Round(4), - VoteLastValid: basics.Round(5), - VoteKeyDilution: 6, - AuthAddr: test.AccountA, + var stateProofID merklesignature.Verifier + stateProofID[0] = 10 + accountDataFull := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 4}, + RewardsBase: 5, + RewardedMicroAlgos: basics.MicroAlgos{Raw: 6}, + AuthAddr: test.AccountA, + }, + VotingData: ledgercore.VotingData{ + VoteID: voteID, + SelectionID: selectionID, + StateProofID: stateProofID, + VoteFirstValid: basics.Round(7), + VoteLastValid: basics.Round(8), + VoteKeyDilution: 9, + }, } - accountDataFull := accountDataWritten - accountDataFull.MicroAlgos = basics.MicroAlgos{Raw: 2} - accountDataFull.RewardsBase = 3 - accountDataFull.RewardedMicroAlgos = basics.MicroAlgos{Raw: 4} + accountDataWritten := encoding.TrimLcAccountData(accountDataFull) _, err := db.Exec( context.Background(), query, test.AccountB[:], accountDataFull.MicroAlgos.Raw, accountDataFull.RewardsBase, accountDataFull.RewardedMicroAlgos.Raw, - encoding.EncodeTrimmedAccountData(accountDataWritten)) + encoding.EncodeTrimmedLcAccountData(accountDataWritten)) require.NoError(t, err) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) @@ -116,57 +115,69 @@ func TestLedgerForEvaluatorAccountTableBasic(t *testing.T) { assert.Equal(t, accountDataFull, *accountDataRet) } -func insertAccountData(db *pgxpool.Pool, account basics.Address, createdat uint64, deleted bool, data basics.AccountData) error { - // This could be 'upsertAccountStmtName' - query := - "INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, " + - "created_at, account_data) " + - "VALUES ($1, $2, $3, $4, $5, $6, $7)" +func insertDeletedAccount(db *pgxpool.Pool, address basics.Address) error { + query := `INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, + created_at, account_data) + VALUES ($1, 0, 0, 0, true, 0, 'null'::jsonb)` + + _, err := db.Exec( + context.Background(), query, address[:]) + return err +} + +func insertAccount(db *pgxpool.Pool, address basics.Address, data ledgercore.AccountData) error { + query := `INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, + created_at, account_data) + VALUES ($1, $2, $3, $4, false, 0, $5)` + _, err := db.Exec( context.Background(), query, - account[:], data.MicroAlgos.Raw, data.RewardsBase, data.RewardedMicroAlgos.Raw, deleted, createdat, - encoding.EncodeTrimmedAccountData(data)) + address[:], data.MicroAlgos.Raw, data.RewardsBase, data.RewardedMicroAlgos.Raw, + encoding.EncodeTrimmedLcAccountData(data)) return err } // TestLedgerForEvaluatorAccountTableBasicSingleAccount a table driven single account test. func TestLedgerForEvaluatorAccountTableSingleAccount(t *testing.T) { tests := []struct { - name string - createdAt uint64 - deleted bool - data basics.AccountData - err string + name string + deleted bool + data ledgercore.AccountData + err string }{ { name: "small balance", - data: basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 1}, + data: ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: 1}, + }, }, }, { name: "max balance", - data: basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: math.MaxInt64}, + data: ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: math.MaxInt64}, + }, }, }, { name: "over max balance", - data: basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: math.MaxUint64}, + data: ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: math.MaxUint64}, + }, }, - err: fmt.Sprintf("%d is greater than maximum value for Int8", uint64(math.MaxUint64)), + err: fmt.Sprintf( + "%d is greater than maximum value for Int8", uint64(math.MaxUint64)), }, { name: "deleted", deleted: true, - data: basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: math.MaxInt64}, - }, }, } - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() for i, testcase := range tests { @@ -184,7 +195,12 @@ func TestLedgerForEvaluatorAccountTableSingleAccount(t *testing.T) { return false } - err := insertAccountData(db, addr, tc.createdAt, tc.deleted, tc.data) + var err error + if tc.deleted { + err = insertDeletedAccount(db, addr) + } else { + err = insertAccount(db, addr, tc.data) + } if checkError(err) { return } @@ -225,7 +241,7 @@ func TestLedgerForEvaluatorAccountTableSingleAccount(t *testing.T) { } func TestLedgerForEvaluatorAccountTableDeleted(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -233,12 +249,14 @@ func TestLedgerForEvaluatorAccountTableDeleted(t *testing.T) { "created_at, account_data) " + "VALUES ($1, 2, 3, 4, true, 0, $2)" - accountData := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, + accountData := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: 5}, + }, } _, err := db.Exec( context.Background(), query, test.AccountB[:], - encoding.EncodeTrimmedAccountData(accountData)) + encoding.EncodeTrimmedLcAccountData(accountData)) require.NoError(t, err) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) @@ -258,7 +276,7 @@ func TestLedgerForEvaluatorAccountTableDeleted(t *testing.T) { } func TestLedgerForEvaluatorAccountTableMissingAccount(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) @@ -277,27 +295,34 @@ func TestLedgerForEvaluatorAccountTableMissingAccount(t *testing.T) { assert.Nil(t, accountDataRet) } -func TestLedgerForEvaluatorAccountAssetTable(t *testing.T) { - db, shutdownFunc := setupPostgres(t) - defer shutdownFunc() +func insertDeletedAccountAsset(t *testing.T, db *pgxpool.Pool, addr basics.Address, assetid uint64) { + query := + "INSERT INTO account_asset (addr, assetid, amount, frozen, deleted, created_at) " + + "VALUES ($1, $2, 0, false, true, 0)" - query := `INSERT INTO account - (addr, microalgos, rewardsbase, rewards_total, deleted, created_at, account_data) - VALUES ($1, 0, 0, 0, false, 0, 'null'::jsonb)` - _, err := db.Exec(context.Background(), query, test.AccountA[:]) + _, err := db.Exec( + context.Background(), query, addr[:], assetid) require.NoError(t, err) +} - query = +func insertAccountAsset(t *testing.T, db *pgxpool.Pool, addr basics.Address, assetid uint64, amount uint64, frozen bool) { + query := "INSERT INTO account_asset (addr, assetid, amount, frozen, deleted, created_at) " + - "VALUES ($1, $2, $3, $4, $5, 0)" - _, err = db.Exec(context.Background(), query, test.AccountA[:], 1, 2, false, false) - require.NoError(t, err) - _, err = db.Exec(context.Background(), query, test.AccountA[:], 3, 4, true, false) - require.NoError(t, err) - _, err = db.Exec(context.Background(), query, test.AccountA[:], 5, 6, true, true) // deleted - require.NoError(t, err) - _, err = db.Exec(context.Background(), query, test.AccountB[:], 5, 6, true, false) // wrong account + "VALUES ($1, $2, $3, $4, false, 0)" + + _, err := db.Exec( + context.Background(), query, addr[:], assetid, amount, frozen) require.NoError(t, err) +} + +func TestLedgerForEvaluatorAccountAssetTable(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccountAsset(t, db, test.AccountA, 1, 2, false) + insertAccountAsset(t, db, test.AccountA, 3, 4, true) + insertDeletedAccountAsset(t, db, test.AccountA, 5) + insertAccountAsset(t, db, test.AccountB, 5, 6, true) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) require.NoError(t, err) @@ -307,59 +332,76 @@ func TestLedgerForEvaluatorAccountAssetTable(t *testing.T) { require.NoError(t, err) defer l.Close() - ret, err := - l.LookupWithoutRewards(map[basics.Address]struct{}{test.AccountA: {}}) + ret, err := l.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AssetCreatable}: {}, + {Index: 3, Type: basics.AssetCreatable}: {}, + {Index: 5, Type: basics.AssetCreatable}: {}, + {Index: 8, Type: basics.AssetCreatable}: {}, + }, + test.AccountB: { + {Index: 5, Type: basics.AssetCreatable}: {}, + }, + }) require.NoError(t, err) - accountDataRet := ret[test.AccountA] - require.NotNil(t, accountDataRet) - - accountDataExpected := basics.AccountData{ - Assets: map[basics.AssetIndex]basics.AssetHolding{ - 1: { - Amount: 2, - Frozen: false, + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 2, + Frozen: false, + }, + }, + ledger.Creatable{Index: 3, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 4, + Frozen: true, + }, }, - 3: { - Amount: 4, - Frozen: true, + ledger.Creatable{Index: 5, Type: basics.AssetCreatable}: {}, + ledger.Creatable{Index: 8, Type: basics.AssetCreatable}: {}, + }, + test.AccountB: { + ledger.Creatable{Index: 5, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 6, + Frozen: true, + }, }, }, } - assert.Equal(t, accountDataExpected, *accountDataRet) + assert.Equal(t, expected, ret) } -func TestLedgerForEvaluatorAssetTable(t *testing.T) { - db, shutdownFunc := setupPostgres(t) - defer shutdownFunc() +func insertDeletedAsset(t *testing.T, db *pgxpool.Pool, index uint64, creatorAddr basics.Address) { + query := `INSERT INTO asset (index, creator_addr, params, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, true, 0)` - query := `INSERT INTO account - (addr, microalgos, rewardsbase, rewards_total, deleted, created_at, account_data) - VALUES ($1, 0, 0, 0, false, 0, 'null'::jsonb)` - _, err := db.Exec(context.Background(), query, test.AccountA[:]) + _, err := db.Exec( + context.Background(), query, index, creatorAddr[:]) require.NoError(t, err) +} - query = - "INSERT INTO asset (index, creator_addr, params, deleted, created_at) " + - "VALUES ($1, $2, $3, $4, 0)" - - _, err = db.Exec( - context.Background(), query, 1, test.AccountA[:], - encoding.EncodeAssetParams(basics.AssetParams{Manager: test.AccountB}), - false) - require.NoError(t, err) +func insertAsset(t *testing.T, db *pgxpool.Pool, index uint64, creatorAddr basics.Address, params *basics.AssetParams) { + query := `INSERT INTO asset (index, creator_addr, params, deleted, created_at) + VALUES ($1, $2, $3, false, 0)` - _, err = db.Exec( - context.Background(), query, 2, test.AccountA[:], - encoding.EncodeAssetParams(basics.AssetParams{Manager: test.AccountC}), - false) + _, err := db.Exec( + context.Background(), query, index, creatorAddr[:], + encoding.EncodeAssetParams(*params)) require.NoError(t, err) +} - _, err = db.Exec(context.Background(), query, 3, test.AccountA[:], "{}", true) // deleted - require.NoError(t, err) +func TestLedgerForEvaluatorAssetTable(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() - _, err = db.Exec(context.Background(), query, 4, test.AccountD[:], "{}", false) // wrong account - require.NoError(t, err) + insertAsset(t, db, 1, test.AccountA, &basics.AssetParams{Manager: test.AccountB}) + insertAsset(t, db, 2, test.AccountA, &basics.AssetParams{Total: 10}) + insertDeletedAsset(t, db, 3, test.AccountA) + insertAsset(t, db, 4, test.AccountC, &basics.AssetParams{Total: 12}) + insertAsset(t, db, 5, test.AccountD, &basics.AssetParams{Total: 13}) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) require.NoError(t, err) @@ -369,65 +411,91 @@ func TestLedgerForEvaluatorAssetTable(t *testing.T) { require.NoError(t, err) defer l.Close() - ret, err := - l.LookupWithoutRewards(map[basics.Address]struct{}{test.AccountA: {}}) + ret, err := l.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AssetCreatable}: {}, + {Index: 2, Type: basics.AssetCreatable}: {}, + {Index: 3, Type: basics.AssetCreatable}: {}, + {Index: 4, Type: basics.AssetCreatable}: {}, + {Index: 6, Type: basics.AssetCreatable}: {}, + }, + test.AccountD: { + {Index: 5, Type: basics.AssetCreatable}: {}, + }, + }) require.NoError(t, err) - accountDataRet := ret[test.AccountA] - require.NotNil(t, accountDataRet) - - accountDataExpected := basics.AccountData{ - AssetParams: map[basics.AssetIndex]basics.AssetParams{ - 1: { - Manager: test.AccountB, + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AssetCreatable}: { + AssetParams: &basics.AssetParams{ + Manager: test.AccountB, + }, }, - 2: { - Manager: test.AccountC, + ledger.Creatable{Index: 2, Type: basics.AssetCreatable}: { + AssetParams: &basics.AssetParams{ + Total: 10, + }, + }, + ledger.Creatable{Index: 3, Type: basics.AssetCreatable}: {}, + ledger.Creatable{Index: 4, Type: basics.AssetCreatable}: {}, + ledger.Creatable{Index: 6, Type: basics.AssetCreatable}: {}, + }, + test.AccountD: { + ledger.Creatable{Index: 5, Type: basics.AssetCreatable}: { + AssetParams: &basics.AssetParams{ + Total: 13, + }, }, }, } - assert.Equal(t, accountDataExpected, *accountDataRet) + assert.Equal(t, expected, ret) } -func TestLedgerForEvaluatorAppTable(t *testing.T) { - db, shutdownFunc := setupPostgres(t) - defer shutdownFunc() +func insertDeletedApp(t *testing.T, db *pgxpool.Pool, index uint64, creator basics.Address) { + query := `INSERT INTO app (index, creator, params, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, true, 0)` - query := `INSERT INTO account - (addr, microalgos, rewardsbase, rewards_total, deleted, created_at, account_data) - VALUES ($1, 0, 0, 0, false, 0, 'null'::jsonb)` - _, err := db.Exec(context.Background(), query, test.AccountA[:]) + _, err := db.Exec(context.Background(), query, index, creator[:]) require.NoError(t, err) +} - query = - "INSERT INTO app (index, creator, params, deleted, created_at) " + - "VALUES ($1, $2, $3, $4, 0)" +func insertApp(t *testing.T, db *pgxpool.Pool, index uint64, creator basics.Address, params *basics.AppParams) { + query := `INSERT INTO app (index, creator, params, deleted, created_at) + VALUES ($1, $2, $3, false, 0)` + + _, err := db.Exec( + context.Background(), query, index, creator[:], encoding.EncodeAppParams(*params)) + require.NoError(t, err) +} + +func TestLedgerForEvaluatorAppTable(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() params1 := basics.AppParams{ GlobalState: map[string]basics.TealValue{ string([]byte{0xff}): {}, // try a non-utf8 string }, } - _, err = db.Exec( - context.Background(), query, 1, test.AccountA[:], - encoding.EncodeAppParams(params1), false) - require.NoError(t, err) + insertApp(t, db, 1, test.AccountA, ¶ms1) params2 := basics.AppParams{ - ApprovalProgram: []byte{1, 2, 3}, + ApprovalProgram: []byte{1, 2, 3, 10}, } - _, err = db.Exec( - context.Background(), query, 2, test.AccountA[:], - encoding.EncodeAppParams(params2), false) - require.NoError(t, err) + insertApp(t, db, 2, test.AccountA, ¶ms2) - _, err = db.Exec( - context.Background(), query, 3, test.AccountA[:], "{}", true) // deteled - require.NoError(t, err) + insertDeletedApp(t, db, 3, test.AccountA) - _, err = db.Exec( - context.Background(), query, 4, test.AccountB[:], "{}", false) // wrong account - require.NoError(t, err) + params4 := basics.AppParams{ + ApprovalProgram: []byte{1, 2, 3, 12}, + } + insertApp(t, db, 4, test.AccountB, ¶ms4) + + params5 := basics.AppParams{ + ApprovalProgram: []byte{1, 2, 3, 13}, + } + insertApp(t, db, 5, test.AccountC, ¶ms5) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) require.NoError(t, err) @@ -437,63 +505,86 @@ func TestLedgerForEvaluatorAppTable(t *testing.T) { require.NoError(t, err) defer l.Close() - ret, err := - l.LookupWithoutRewards(map[basics.Address]struct{}{test.AccountA: {}}) + ret, err := l.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AppCreatable}: {}, + {Index: 2, Type: basics.AppCreatable}: {}, + {Index: 3, Type: basics.AppCreatable}: {}, + {Index: 4, Type: basics.AppCreatable}: {}, + {Index: 6, Type: basics.AppCreatable}: {}, + }, + test.AccountC: { + {Index: 5, Type: basics.AppCreatable}: {}, + }, + }) require.NoError(t, err) - accountDataRet := ret[test.AccountA] - require.NotNil(t, accountDataRet) - - accountDataExpected := basics.AccountData{ - AppParams: map[basics.AppIndex]basics.AppParams{ - 1: params1, - 2: params2, + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AppCreatable}: { + AppParams: ¶ms1, + }, + ledger.Creatable{Index: 2, Type: basics.AppCreatable}: { + AppParams: ¶ms2, + }, + ledger.Creatable{Index: 3, Type: basics.AppCreatable}: {}, + ledger.Creatable{Index: 4, Type: basics.AppCreatable}: {}, + ledger.Creatable{Index: 6, Type: basics.AppCreatable}: {}, + }, + test.AccountC: { + ledger.Creatable{Index: 5, Type: basics.AppCreatable}: { + AppParams: ¶ms5, + }, }, } - assert.Equal(t, accountDataExpected, *accountDataRet) + assert.Equal(t, expected, ret) } -func TestLedgerForEvaluatorAccountAppTable(t *testing.T) { - db, shutdownFunc := setupPostgres(t) - defer shutdownFunc() +func insertDeletedAccountApp(t *testing.T, db *pgxpool.Pool, addr basics.Address, app uint64) { + query := `INSERT INTO account_app (addr, app, localstate, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, true, 0)` - query := `INSERT INTO account - (addr, microalgos, rewardsbase, rewards_total, deleted, created_at, account_data) - VALUES ($1, 0, 0, 0, false, 0, 'null'::jsonb)` - _, err := db.Exec(context.Background(), query, test.AccountA[:]) + _, err := db.Exec( + context.Background(), query, addr[:], app) require.NoError(t, err) +} - query = - "INSERT INTO account_app (addr, app, localstate, deleted, created_at) " + - "VALUES ($1, $2, $3, $4, 0)" +func insertAccountApp(t *testing.T, db *pgxpool.Pool, addr basics.Address, app uint64, localstate *basics.AppLocalState) { + query := `INSERT INTO account_app (addr, app, localstate, deleted, created_at) + VALUES ($1, $2, $3, false, 0)` - params1 := basics.AppLocalState{ + _, err := db.Exec( + context.Background(), query, addr[:], app, + encoding.EncodeAppLocalState(*localstate)) + require.NoError(t, err) +} + +func TestLedgerForEvaluatorAccountAppTable(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + stateA1 := basics.AppLocalState{ KeyValue: map[string]basics.TealValue{ string([]byte{0xff}): {}, // try a non-utf8 string }, } - _, err = db.Exec( - context.Background(), query, test.AccountA[:], 1, - encoding.EncodeAppLocalState(params1), false) - require.NoError(t, err) + insertAccountApp(t, db, test.AccountA, 1, &stateA1) - params2 := basics.AppLocalState{ + stateA2 := basics.AppLocalState{ KeyValue: map[string]basics.TealValue{ "abc": {}, }, } - _, err = db.Exec( - context.Background(), query, test.AccountA[:], 2, - encoding.EncodeAppLocalState(params2), false) - require.NoError(t, err) + insertAccountApp(t, db, test.AccountA, 2, &stateA2) - _, err = db.Exec( - context.Background(), query, test.AccountA[:], 3, "{}", true) // deteled - require.NoError(t, err) + insertDeletedAccountApp(t, db, test.AccountA, 3) - _, err = db.Exec( - context.Background(), query, test.AccountB[:], 4, "{}", false) // wrong account - require.NoError(t, err) + stateB3 := basics.AppLocalState{ + KeyValue: map[string]basics.TealValue{ + "abf": {}, + }, + } + insertAccountApp(t, db, test.AccountB, 3, &stateB3) tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) require.NoError(t, err) @@ -503,70 +594,107 @@ func TestLedgerForEvaluatorAccountAppTable(t *testing.T) { require.NoError(t, err) defer l.Close() - ret, err := - l.LookupWithoutRewards(map[basics.Address]struct{}{test.AccountA: {}}) + ret, err := l.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AppCreatable}: {}, + {Index: 2, Type: basics.AppCreatable}: {}, + {Index: 3, Type: basics.AppCreatable}: {}, + {Index: 4, Type: basics.AppCreatable}: {}, + }, + test.AccountB: { + {Index: 3, Type: basics.AppCreatable}: {}, + }, + }) require.NoError(t, err) - accountDataRet := ret[test.AccountA] - require.NotNil(t, accountDataRet) + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AppCreatable}: { + AppLocalState: &stateA1, + }, + ledger.Creatable{Index: 2, Type: basics.AppCreatable}: { + AppLocalState: &stateA2, + }, + ledger.Creatable{Index: 3, Type: basics.AppCreatable}: {}, + ledger.Creatable{Index: 4, Type: basics.AppCreatable}: {}, + }, + test.AccountB: { + ledger.Creatable{Index: 3, Type: basics.AppCreatable}: { + AppLocalState: &stateB3, + }, + }, + } + assert.Equal(t, expected, ret) +} + +func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccountAsset(t, db, test.AccountA, 1, 2, true) + insertAsset(t, db, 1, test.AccountA, &basics.AssetParams{Total: 3}) + insertAccountApp( + t, db, test.AccountA, 4, + &basics.AppLocalState{Schema: basics.StateSchema{NumUint: 5}}) + insertApp(t, db, 4, test.AccountA, &basics.AppParams{ExtraProgramPages: 6}) + + tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) + require.NoError(t, err) + defer tx.Rollback(context.Background()) + + l, err := ledger_for_evaluator.MakeLedgerForEvaluator(tx, basics.Round(0)) + require.NoError(t, err) + defer l.Close() - accountDataExpected := basics.AccountData{ - AppLocalStates: map[basics.AppIndex]basics.AppLocalState{ - 1: params1, - 2: params2, + ret, err := l.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AssetCreatable}: {}, + {Index: 4, Type: basics.AppCreatable}: {}, + }, + }) + require.NoError(t, err) + + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 2, + Frozen: true, + }, + AssetParams: &basics.AssetParams{ + Total: 3, + }, + }, + ledger.Creatable{Index: 4, Type: basics.AppCreatable}: { + AppLocalState: &basics.AppLocalState{ + Schema: basics.StateSchema{ + NumUint: 5, + }, + }, + AppParams: &basics.AppParams{ + ExtraProgramPages: 6, + }, + }, }, } - assert.Equal(t, accountDataExpected, *accountDataRet) + assert.Equal(t, expected, ret) } -// Tests that queuing and reading from a batch when using PreloadAccounts() -// is in the same order. +// Tests that queuing and reading from a batch is in the same order. func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() addAccountQuery := `INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, created_at, account_data) VALUES ($1, 0, 0, 0, false, 0, 'null'::jsonb)` - addAccountAssetQuery := - "INSERT INTO account_asset (addr, assetid, amount, frozen, deleted, created_at) " + - "VALUES ($1, $2, 0, false, false, 0)" - addAssetQuery := - "INSERT INTO asset (index, creator_addr, params, deleted, created_at) " + - "VALUES ($1, $2, '{}', false, 0)" - addAppQuery := - "INSERT INTO app (index, creator, params, deleted, created_at) " + - "VALUES ($1, $2, '{}', false, 0)" - addAccountAppQuery := - "INSERT INTO account_app (addr, app, localstate, deleted, created_at) " + - "VALUES ($1, $2, '{}', false, 0)" addresses := []basics.Address{ test.AccountA, test.AccountB, test.AccountC, test.AccountD, test.AccountE} - seq := []int{4, 9, 3, 6, 5, 1} - for i, address := range addresses { + for _, address := range addresses { _, err := db.Exec(context.Background(), addAccountQuery, address[:]) require.NoError(t, err) - - // Insert all types of creatables. Note that no creatable id is ever repeated. - for j := range seq { - _, err = db.Exec( - context.Background(), addAccountAssetQuery, address[:], i+10*j+100) - require.NoError(t, err) - - _, err = db.Exec( - context.Background(), addAssetQuery, i+10*j+200, address[:]) - require.NoError(t, err) - - _, err = db.Exec( - context.Background(), addAppQuery, i+10*j+300, address[:]) - require.NoError(t, err) - - _, err = db.Exec( - context.Background(), addAccountAppQuery, address[:], i+10*j+400) - require.NoError(t, err) - } } tx, err := db.BeginTx(context.Background(), readonlyRepeatableRead) @@ -587,33 +715,14 @@ func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { ret, err := l.LookupWithoutRewards(addressesMap) require.NoError(t, err) - for i, address := range addresses { + for _, address := range addresses { accountData, _ := ret[address] require.NotNil(t, accountData) - - assert.Equal(t, len(seq), len(accountData.Assets)) - assert.Equal(t, len(seq), len(accountData.AssetParams)) - assert.Equal(t, len(seq), len(accountData.AppParams)) - assert.Equal(t, len(seq), len(accountData.AppLocalStates)) - - for j := range seq { - _, ok := accountData.Assets[basics.AssetIndex(i+10*j+100)] - assert.True(t, ok) - - _, ok = accountData.AssetParams[basics.AssetIndex(i+10*j+200)] - assert.True(t, ok) - - _, ok = accountData.AppParams[basics.AppIndex(i+10*j+300)] - assert.True(t, ok) - - _, ok = accountData.AppLocalStates[basics.AppIndex(i+10*j+400)] - assert.True(t, ok) - } } } func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -645,7 +754,7 @@ func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { } func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -673,7 +782,7 @@ func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { } func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() creatorsMap := map[basics.AssetIndex]basics.Address{ @@ -729,7 +838,7 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { } func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -761,7 +870,7 @@ func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { } func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() query := @@ -789,7 +898,7 @@ func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { } func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() creatorsMap := map[basics.AppIndex]basics.Address{ @@ -845,7 +954,7 @@ func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { } func TestLedgerForEvaluatorAccountTotals(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() accountTotals := ledgercore.AccountTotals{ diff --git a/idb/postgres/internal/migrations/convert_account_data/m.go b/idb/postgres/internal/migrations/convert_account_data/m.go new file mode 100644 index 000000000..e7cb13021 --- /dev/null +++ b/idb/postgres/internal/migrations/convert_account_data/m.go @@ -0,0 +1,188 @@ +package convertaccountdata + +import ( + "context" + "fmt" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + + "github.com/jackc/pgx/v4" + + "github.com/algorand/indexer/idb/postgres/internal/encoding" +) + +type aad struct { + address basics.Address + trimmedAccountData basics.AccountData +} + +func getAccounts(tx pgx.Tx, batchSize uint, lastAddress *basics.Address) ([]aad, error) { + var rows pgx.Rows + var err error + if lastAddress == nil { + query := + `SELECT addr, account_data FROM account WHERE NOT deleted ORDER BY addr LIMIT $1` + rows, err = tx.Query(context.Background(), query, batchSize) + } else { + query := `SELECT addr, account_data FROM account WHERE NOT deleted AND addr > $1 + ORDER BY addr LIMIT $2` + rows, err = tx.Query(context.Background(), query, (*lastAddress)[:], batchSize) + } + if err != nil { + return nil, fmt.Errorf("getAccounts() query err: %w", err) + } + + res := make([]aad, 0, batchSize) + for rows.Next() { + var addr []byte + var accountData []byte + err = rows.Scan(&addr, &accountData) + if err != nil { + return nil, fmt.Errorf("getAccounts() scan err: %w", err) + } + + res = append(res, aad{}) + e := &res[len(res)-1] + copy(e.address[:], addr) + e.trimmedAccountData, err = encoding.DecodeTrimmedAccountData(accountData) + if err != nil { + return nil, fmt.Errorf("getAccounts() decode err: %w", err) + } + } + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("getAccounts() rows error err: %w", err) + } + + return res, nil +} + +func computeLcAccountData(tx pgx.Tx, accounts []aad) ([]ledgercore.AccountData, error) { + res := make([]ledgercore.AccountData, 0, len(accounts)) + for i := range accounts { + res = append(res, ledgercore.ToAccountData(accounts[i].trimmedAccountData)) + } + + var batch pgx.Batch + for i := range accounts { + batch.Queue( + "SELECT COUNT(*) FROM account_asset WHERE NOT deleted AND addr = $1", + accounts[i].address[:]) + } + for i := range accounts { + batch.Queue( + "SELECT COUNT(*) FROM asset WHERE NOT deleted AND creator_addr = $1", + accounts[i].address[:]) + } + for i := range accounts { + batch.Queue( + "SELECT COUNT(*) FROM app WHERE NOT deleted AND creator = $1", + accounts[i].address[:]) + } + for i := range accounts { + batch.Queue( + "SELECT COUNT(*) FROM account_app WHERE NOT deleted AND addr = $1", + accounts[i].address[:]) + } + + results := tx.SendBatch(context.Background(), &batch) + defer results.Close() + + for i := range accounts { + err := results.QueryRow().Scan(&res[i].TotalAssets) + if err != nil { + return nil, fmt.Errorf("computeLcAccountData() scan total assets err: %w", err) + } + } + for i := range accounts { + err := results.QueryRow().Scan(&res[i].TotalAssetParams) + if err != nil { + return nil, fmt.Errorf("computeLcAccountData() scan total asset params err: %w", err) + } + } + for i := range accounts { + err := results.QueryRow().Scan(&res[i].TotalAppParams) + if err != nil { + return nil, fmt.Errorf("computeLcAccountData() scan total app params err: %w", err) + } + } + for i := range accounts { + err := results.QueryRow().Scan(&res[i].TotalAppLocalStates) + if err != nil { + return nil, fmt.Errorf("computeLcAccountData() scan total app local states err: %w", err) + } + } + + err := results.Close() + if err != nil { + return nil, fmt.Errorf("computeLcAccountData() close results err: %w", err) + } + + return res, nil +} + +func writeLcAccountData(tx pgx.Tx, accounts []aad, lcAccountData []ledgercore.AccountData) error { + var batch pgx.Batch + for i := range accounts { + query := "UPDATE account SET account_data = $1 WHERE addr = $2" + batch.Queue( + query, encoding.EncodeTrimmedLcAccountData(lcAccountData[i]), + accounts[i].address[:]) + } + + results := tx.SendBatch(context.Background(), &batch) + // Clean the results off the connection's queue. Without this, weird things happen. + for i := 0; i < batch.Len(); i++ { + _, err := results.Exec() + if err != nil { + results.Close() + return fmt.Errorf("writeLcAccountData() exec err: %w", err) + } + } + err := results.Close() + if err != nil { + return fmt.Errorf("writeLcAccountData() close results err: %w", err) + } + + return nil +} + +func processAccounts(tx pgx.Tx, accounts []aad) error { + lcAccountData, err := computeLcAccountData(tx, accounts) + if err != nil { + return fmt.Errorf("processAccounts() err: %w", err) + } + + err = writeLcAccountData(tx, accounts, lcAccountData) + if err != nil { + return fmt.Errorf("processAccounts() err: %w", err) + } + + return nil +} + +// RunMigration executes the migration core functionality. +func RunMigration(tx pgx.Tx, batchSize uint) error { + accounts, err := getAccounts(tx, batchSize, nil) + if err != nil { + return fmt.Errorf("RunMigration() err: %w", err) + } + err = processAccounts(tx, accounts) + if err != nil { + return fmt.Errorf("RunMigration() err: %w", err) + } + + for uint(len(accounts)) >= batchSize { + accounts, err = getAccounts(tx, batchSize, &accounts[len(accounts)-1].address) + if err != nil { + return fmt.Errorf("RunMigration() err: %w", err) + } + err = processAccounts(tx, accounts) + if err != nil { + return fmt.Errorf("RunMigration() err: %w", err) + } + } + + return nil +} diff --git a/idb/postgres/internal/migrations/convert_account_data/m_test.go b/idb/postgres/internal/migrations/convert_account_data/m_test.go new file mode 100644 index 000000000..ba0665ba2 --- /dev/null +++ b/idb/postgres/internal/migrations/convert_account_data/m_test.go @@ -0,0 +1,270 @@ +package convertaccountdata_test + +import ( + "context" + "fmt" + "testing" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + + "github.com/algorand/indexer/idb/postgres/internal/encoding" + cad "github.com/algorand/indexer/idb/postgres/internal/migrations/convert_account_data" + pgtest "github.com/algorand/indexer/idb/postgres/internal/testing" + pgutil "github.com/algorand/indexer/idb/postgres/internal/util" +) + +func makeAddress(i int) basics.Address { + var address basics.Address + address[0] = byte(i) + return address +} + +func insertAccount(t *testing.T, db *pgxpool.Pool, address basics.Address, trimmedAccountData *basics.AccountData) { + query := `INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, + created_at, account_data) VALUES ($1, 0, 0, 0, false, 0, $2)` + _, err := db.Exec( + context.Background(), query, address[:], + encoding.EncodeTrimmedAccountData(*trimmedAccountData)) + require.NoError(t, err) +} + +func insertDeletedAccount(t *testing.T, db *pgxpool.Pool, address basics.Address) { + query := `INSERT INTO account (addr, microalgos, rewardsbase, rewards_total, deleted, + created_at, account_data) VALUES ($1, 0, 0, 0, true, 0, 'null'::jsonb)` + _, err := db.Exec(context.Background(), query, address[:]) + require.NoError(t, err) +} + +func checkAccount(t *testing.T, db *pgxpool.Pool, address basics.Address, accountData *ledgercore.AccountData) { + query := "SELECT account_data FROM account WHERE addr = $1" + row := db.QueryRow(context.Background(), query, address[:]) + + var buf []byte + err := row.Scan(&buf) + require.NoError(t, err) + + ret, err := encoding.DecodeTrimmedLcAccountData(buf) + require.NoError(t, err) + + assert.Equal(t, accountData, &ret) +} + +func checkDeletedAccount(t *testing.T, db *pgxpool.Pool, address basics.Address) { + query := "SELECT account_data FROM account WHERE addr = $1" + row := db.QueryRow(context.Background(), query, address[:]) + + var buf []byte + err := row.Scan(&buf) + require.NoError(t, err) + + assert.Equal(t, []byte("null"), buf) +} + +func insertAccountAsset(t *testing.T, db *pgxpool.Pool, address basics.Address, assetid uint64, deleted bool) { + query := `INSERT INTO account_asset (addr, assetid, amount, frozen, deleted, + created_at) VALUES ($1, $2, 0, false, $3, 0)` + _, err := db.Exec(context.Background(), query, address[:], assetid, deleted) + require.NoError(t, err) +} + +func insertAsset(t *testing.T, db *pgxpool.Pool, assetid uint64, address basics.Address, deleted bool) { + query := `INSERT INTO asset (index, creator_addr, params, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, $3, 0)` + _, err := db.Exec(context.Background(), query, assetid, address[:], deleted) + require.NoError(t, err) +} + +func insertApp(t *testing.T, db *pgxpool.Pool, appid uint64, address basics.Address, deleted bool) { + query := `INSERT INTO app (index, creator, params, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, $3, 0)` + _, err := db.Exec(context.Background(), query, appid, address[:], deleted) + require.NoError(t, err) +} + +func insertAccountApp(t *testing.T, db *pgxpool.Pool, address basics.Address, appid uint64, deleted bool) { + query := `INSERT INTO account_app (addr, app, localstate, deleted, created_at) + VALUES ($1, $2, 'null'::jsonb, $3, 0)` + _, err := db.Exec(context.Background(), query, address[:], appid, deleted) + require.NoError(t, err) +} + +func TestBasic(t *testing.T) { + for _, i := range []int{1, 2, 3, 4} { + t.Run(fmt.Sprint(i), func(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccount(t, db, makeAddress(1), &basics.AccountData{VoteKeyDilution: 1}) + insertDeletedAccount(t, db, makeAddress(2)) + insertAccount(t, db, makeAddress(3), &basics.AccountData{VoteKeyDilution: 3}) + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 1) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + checkAccount( + t, db, makeAddress(1), + &ledgercore.AccountData{VotingData: ledgercore.VotingData{VoteKeyDilution: 1}}) + checkDeletedAccount(t, db, makeAddress(2)) + checkAccount( + t, db, makeAddress(3), + &ledgercore.AccountData{VotingData: ledgercore.VotingData{VoteKeyDilution: 3}}) + }) + } +} + +func TestAccountAssetCount(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccount(t, db, makeAddress(1), &basics.AccountData{VoteKeyDilution: 1}) + for i := uint64(2); i < 10; i++ { + insertAccountAsset(t, db, makeAddress(1), i, i%2 == 0) + } + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 1) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + expected := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + TotalAssets: 4, + }, + VotingData: ledgercore.VotingData{ + VoteKeyDilution: 1, + }, + } + checkAccount(t, db, makeAddress(1), &expected) +} + +func TestAssetCount(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccount(t, db, makeAddress(1), &basics.AccountData{VoteKeyDilution: 1}) + for i := uint64(2); i < 10; i++ { + insertAsset(t, db, i, makeAddress(1), i%2 == 0) + } + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 1) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + expected := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + TotalAssetParams: 4, + }, + VotingData: ledgercore.VotingData{ + VoteKeyDilution: 1, + }, + } + checkAccount(t, db, makeAddress(1), &expected) +} + +func TestAppCount(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccount(t, db, makeAddress(1), &basics.AccountData{VoteKeyDilution: 1}) + for i := uint64(2); i < 10; i++ { + insertApp(t, db, i, makeAddress(1), i%2 == 0) + } + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 1) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + expected := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + TotalAppParams: 4, + }, + VotingData: ledgercore.VotingData{ + VoteKeyDilution: 1, + }, + } + checkAccount(t, db, makeAddress(1), &expected) +} + +func TestAccountAppCount(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + insertAccount(t, db, makeAddress(1), &basics.AccountData{VoteKeyDilution: 1}) + for i := uint64(2); i < 10; i++ { + insertAccountApp(t, db, makeAddress(1), i, i%2 == 0) + } + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 1) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + expected := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + TotalAppLocalStates: 4, + }, + VotingData: ledgercore.VotingData{ + VoteKeyDilution: 1, + }, + } + checkAccount(t, db, makeAddress(1), &expected) +} + +func TestAllResourcesMultipleAccounts(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + numAccounts := 14 + + for i := 0; i < numAccounts; i++ { + insertAccount(t, db, makeAddress(i), &basics.AccountData{VoteKeyDilution: uint64(i)}) + for j := uint64(20); j < 30; j++ { + insertAccountAsset(t, db, makeAddress(i), j, j%2 == 0) + } + for j := uint64(30); j < 50; j++ { + insertAsset(t, db, uint64(i)*1000+j, makeAddress(i), j%2 == 0) + } + for j := uint64(50); j < 80; j++ { + insertApp(t, db, uint64(i)*1000+j, makeAddress(i), j%2 == 0) + } + for j := uint64(80); j < 120; j++ { + insertAccountApp(t, db, makeAddress(i), j, j%2 == 0) + } + } + + f := func(tx pgx.Tx) error { + return cad.RunMigration(tx, 5) + } + err := pgutil.TxWithRetry(db, pgx.TxOptions{IsoLevel: pgx.Serializable}, f, nil) + require.NoError(t, err) + + for i := 0; i < numAccounts; i++ { + expected := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + TotalAssets: 5, + TotalAssetParams: 10, + TotalAppParams: 15, + TotalAppLocalStates: 20, + }, + VotingData: ledgercore.VotingData{ + VoteKeyDilution: uint64(i), + }, + } + checkAccount(t, db, makeAddress(i), &expected) + } +} diff --git a/idb/postgres/internal/schema/setup_postgres.sql b/idb/postgres/internal/schema/setup_postgres.sql index fb3d63824..9337c0e10 100644 --- a/idb/postgres/internal/schema/setup_postgres.sql +++ b/idb/postgres/internal/schema/setup_postgres.sql @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS account ( created_at bigint NOT NULL, -- round that the account is first used closed_at bigint, -- round that the account was last closed keytype varchar(8), -- "sig", "msig", "lsig", or NULL if unknown - account_data jsonb NOT NULL -- trimmed AccountData that excludes the fields above and the four creatable maps; SQL 'NOT NULL' is held though the json string will be "null" iff account is deleted + account_data jsonb NOT NULL -- trimmed ledgercore.AccountData that excludes the fields above; SQL 'NOT NULL' is held though the json string will be "null" iff account is deleted ); -- data.basics.AccountData Assets[asset id] AssetHolding{} diff --git a/idb/postgres/internal/schema/setup_postgres_sql.go b/idb/postgres/internal/schema/setup_postgres_sql.go index acad7851b..3d5fb0a2a 100644 --- a/idb/postgres/internal/schema/setup_postgres_sql.go +++ b/idb/postgres/internal/schema/setup_postgres_sql.go @@ -53,7 +53,7 @@ CREATE TABLE IF NOT EXISTS account ( created_at bigint NOT NULL, -- round that the account is first used closed_at bigint, -- round that the account was last closed keytype varchar(8), -- "sig", "msig", "lsig", or NULL if unknown - account_data jsonb NOT NULL -- trimmed AccountData that excludes the fields above and the four creatable maps; SQL 'NOT NULL' is held though the json string will be "null" iff account is deleted + account_data jsonb NOT NULL -- trimmed ledgercore.AccountData that excludes the fields above; SQL 'NOT NULL' is held though the json string will be "null" iff account is deleted ); -- data.basics.AccountData Assets[asset id] AssetHolding{} diff --git a/idb/postgres/internal/testing/testing.go b/idb/postgres/internal/testing/testing.go index 433225353..34f049ad5 100644 --- a/idb/postgres/internal/testing/testing.go +++ b/idb/postgres/internal/testing/testing.go @@ -10,6 +10,8 @@ import ( "github.com/orlangure/gnomock" "github.com/orlangure/gnomock/preset/postgres" "github.com/stretchr/testify/require" + + "github.com/algorand/indexer/idb/postgres/internal/schema" ) var testpg = flag.String( @@ -61,3 +63,14 @@ func SetupPostgres(t *testing.T) (*pgxpool.Pool, string, func()) { return db, connStr, shutdownFunc } + +// SetupPostgresWithSchema is equivalent to SetupPostgres() but also creates the +// indexer schema. +func SetupPostgresWithSchema(t *testing.T) (*pgxpool.Pool, string, func()) { + db, connStr, shutdownFunc := SetupPostgres(t) + + _, err := db.Exec(context.Background(), schema.SetupPostgresSql) + require.NoError(t, err) + + return db, connStr, shutdownFunc +} diff --git a/idb/postgres/internal/writer/writer.go b/idb/postgres/internal/writer/writer.go index 0ba5e491f..9d61efae6 100644 --- a/idb/postgres/internal/writer/writer.go +++ b/idb/postgres/internal/writer/writer.go @@ -178,36 +178,7 @@ type optionalSigTypeDelta struct { value sigTypeDelta } -func writeAccount(round basics.Round, address basics.Address, accountData basics.AccountData, sigtypeDelta optionalSigTypeDelta, batch *pgx.Batch) { - // Update `asset` table. - for assetid, params := range accountData.AssetParams { - batch.Queue( - upsertAssetStmtName, - uint64(assetid), address[:], encoding.EncodeAssetParams(params), uint64(round)) - } - - // Update `account_asset` table. - for assetid, holding := range accountData.Assets { - batch.Queue( - upsertAccountAssetStmtName, - address[:], uint64(assetid), strconv.FormatUint(holding.Amount, 10), - holding.Frozen, uint64(round)) - } - - // Update `app` table. - for appid, params := range accountData.AppParams { - batch.Queue( - upsertAppStmtName, - uint64(appid), address[:], encoding.EncodeAppParams(params), uint64(round)) - } - - // Update `account_app` table. - for appid, state := range accountData.AppLocalStates { - batch.Queue( - upsertAccountAppStmtName, - address[:], uint64(appid), encoding.EncodeAppLocalState(state), uint64(round)) - } - +func writeAccount(round basics.Round, address basics.Address, accountData ledgercore.AccountData, sigtypeDelta optionalSigTypeDelta, batch *pgx.Batch) { sigtypeFunc := func(delta sigTypeDelta) *idb.SigType { if !delta.present { return nil @@ -218,7 +189,6 @@ func writeAccount(round basics.Round, address basics.Address, accountData basics return res } - // Update `account` table. if accountData.IsZero() { // Delete account. if sigtypeDelta.present { @@ -231,7 +201,7 @@ func writeAccount(round basics.Round, address basics.Address, accountData basics } else { // Update account. accountDataJSON := - encoding.EncodeTrimmedAccountData(encoding.TrimAccountData(accountData)) + encoding.EncodeTrimmedLcAccountData(encoding.TrimLcAccountData(accountData)) if sigtypeDelta.present { batch.Queue( @@ -249,53 +219,75 @@ func writeAccount(round basics.Round, address basics.Address, accountData basics } } -func writeAccounts(round basics.Round, accountDeltas ledgercore.AccountDeltas, sigtypeDeltas map[basics.Address]sigTypeDelta, batch *pgx.Batch) { - // Update `account` table. - for i := 0; i < accountDeltas.Len(); i++ { - address, accountData := accountDeltas.GetByIdx(i) - - var sigtypeDelta optionalSigTypeDelta - sigtypeDelta.value, sigtypeDelta.present = sigtypeDeltas[address] - - writeAccount(round, address, accountData, sigtypeDelta, batch) +func writeAssetResource(round basics.Round, resource *ledgercore.AssetResourceRecord, batch *pgx.Batch) { + if resource.Params.Deleted { + batch.Queue(deleteAssetStmtName, resource.Aidx, resource.Addr[:], round) + } else { + if resource.Params.Params != nil { + batch.Queue( + upsertAssetStmtName, resource.Aidx, resource.Addr[:], + encoding.EncodeAssetParams(*resource.Params.Params), round) + } } -} -func writeDeletedCreatables(round basics.Round, creatables map[basics.CreatableIndex]ledgercore.ModifiedCreatable, batch *pgx.Batch) { - for index, creatable := range creatables { - // If deleted. - if !creatable.Created { - creator := new(basics.Address) - *creator = creatable.Creator - - if creatable.Ctype == basics.AssetCreatable { - batch.Queue(deleteAssetStmtName, uint64(index), creator[:], uint64(round)) - } else { - batch.Queue(deleteAppStmtName, uint64(index), creator[:], uint64(round)) - } + if resource.Holding.Deleted { + batch.Queue(deleteAccountAssetStmtName, resource.Addr[:], resource.Aidx, round) + } else { + if resource.Holding.Holding != nil { + batch.Queue( + upsertAccountAssetStmtName, resource.Addr[:], resource.Aidx, + strconv.FormatUint(resource.Holding.Holding.Amount, 10), + resource.Holding.Holding.Frozen, round) } } } -func writeDeletedAssetHoldings(round basics.Round, modifiedAssetHoldings map[ledgercore.AccountAsset]bool, batch *pgx.Batch) { - for aa, created := range modifiedAssetHoldings { - if !created { - address := new(basics.Address) - *address = aa.Address +func writeAppResource(round basics.Round, resource *ledgercore.AppResourceRecord, batch *pgx.Batch) { + if resource.Params.Deleted { + batch.Queue(deleteAppStmtName, resource.Aidx, resource.Addr[:], round) + } else { + if resource.Params.Params != nil { + batch.Queue( + upsertAppStmtName, resource.Aidx, resource.Addr[:], + encoding.EncodeAppParams(*resource.Params.Params), round) + } + } + if resource.State.Deleted { + batch.Queue(deleteAccountAppStmtName, resource.Addr[:], resource.Aidx, round) + } else { + if resource.State.LocalState != nil { batch.Queue( - deleteAccountAssetStmtName, address[:], uint64(aa.Asset), uint64(round)) + upsertAccountAppStmtName, resource.Addr[:], resource.Aidx, + encoding.EncodeAppLocalState(*resource.State.LocalState), round) } } } -func writeDeletedAppLocalStates(round basics.Round, modifiedAppLocalStates map[ledgercore.AccountApp]bool, batch *pgx.Batch) { - for aa, created := range modifiedAppLocalStates { - if !created { - address := new(basics.Address) - *address = aa.Address +func writeAccountDeltas(round basics.Round, accountDeltas *ledgercore.AccountDeltas, sigtypeDeltas map[basics.Address]sigTypeDelta, batch *pgx.Batch) { + // Update `account` table. + for i := 0; i < accountDeltas.Len(); i++ { + address, accountData := accountDeltas.GetByIdx(i) - batch.Queue(deleteAccountAppStmtName, address[:], uint64(aa.App), uint64(round)) + var sigtypeDelta optionalSigTypeDelta + sigtypeDelta.value, sigtypeDelta.present = sigtypeDeltas[address] + + writeAccount(round, address, accountData, sigtypeDelta, batch) + } + + // Update `asset` and `account_asset` tables. + { + assetResources := accountDeltas.GetAllAssetResources() + for i := range assetResources { + writeAssetResource(round, &assetResources[i], batch) + } + } + + // Update `app` and `account_app` tables. + { + appResources := accountDeltas.GetAllAppResources() + for i := range appResources { + writeAppResource(round, &appResources[i], batch) } } } @@ -345,11 +337,8 @@ func (w *Writer) AddBlock(block *bookkeeping.Block, modifiedTxns []transactions. if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - writeAccounts(block.Round(), delta.Accts, sigTypeDeltas, &batch) + writeAccountDeltas(block.Round(), &delta.Accts, sigTypeDeltas, &batch) } - writeDeletedCreatables(block.Round(), delta.Creatables, &batch) - writeDeletedAssetHoldings(block.Round(), delta.ModifiedAssetHoldings, &batch) - writeDeletedAppLocalStates(block.Round(), delta.ModifiedAppLocalStates, &batch) batch.Queue(updateAccountTotalsStmtName, encoding.EncodeAccountTotals(&delta.Totals)) results := w.tx.SendBatch(context.Background(), &batch) diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index 70d57441e..3b0902c0e 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -29,15 +29,6 @@ import ( var serializable = pgx.TxOptions{IsoLevel: pgx.Serializable} -func setupPostgres(t *testing.T) (*pgxpool.Pool, func()) { - db, _, shutdownFunc := pgtest.SetupPostgres(t) - - _, err := db.Exec(context.Background(), schema.SetupPostgresSql) - require.NoError(t, err) - - return db, shutdownFunc -} - // makeTx is a helper to simplify calling TxWithRetry func makeTx(db *pgxpool.Pool, f func(tx pgx.Tx) error) error { return pgutil.TxWithRetry(db, serializable, f, nil) @@ -105,7 +96,7 @@ func txnParticipationQuery(db *pgxpool.Pool, query string) ([]txnParticipationRo } func TestWriterBlockHeaderTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -146,7 +137,7 @@ func TestWriterBlockHeaderTableBasic(t *testing.T) { } func TestWriterSpecialAccounts(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := test.MakeGenesisBlock() @@ -178,7 +169,7 @@ func TestWriterSpecialAccounts(t *testing.T) { } func TestWriterTxnTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := bookkeeping.Block{ @@ -267,7 +258,7 @@ func TestWriterTxnTableBasic(t *testing.T) { // Test that asset close amount is written even if it is missing in the apply data // in the block (it is present in the "modified transactions"). func TestWriterTxnTableAssetCloseAmount(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := bookkeeping.Block{ @@ -412,7 +403,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { for _, testcase := range tests { t.Run(testcase.name, func(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := makeBlockFunc() @@ -436,7 +427,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { // Create a new account and then delete it. func TestWriterAccountTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var voteID crypto.OneTimeSignatureVerifier @@ -452,17 +443,21 @@ func TestWriterAccountTableBasic(t *testing.T) { block.BlockHeader.Round = 4 var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, basics.AccountData{ - Status: basics.Online, - MicroAlgos: basics.MicroAlgos{Raw: 5}, - RewardsBase: 6, - RewardedMicroAlgos: basics.MicroAlgos{Raw: 7}, - VoteID: voteID, - SelectionID: selectionID, - VoteFirstValid: 7, - VoteLastValid: 8, - VoteKeyDilution: 9, - AuthAddr: authAddr, + delta.Accts.Upsert(test.AccountA, ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 5}, + RewardsBase: 6, + RewardedMicroAlgos: basics.MicroAlgos{Raw: 7}, + AuthAddr: authAddr, + }, + VotingData: ledgercore.VotingData{ + VoteID: voteID, + SelectionID: selectionID, + VoteFirstValid: 7, + VoteLastValid: 8, + VoteKeyDilution: 9, + }, }) f := func(tx pgx.Tx) error { @@ -510,16 +505,9 @@ func TestWriterAccountTableBasic(t *testing.T) { assert.Nil(t, closedAt) assert.Nil(t, keytype) { - accountDataRead, err := encoding.DecodeTrimmedAccountData(accountData) + accountDataRead, err := encoding.DecodeTrimmedLcAccountData(accountData) require.NoError(t, err) - - assert.Equal(t, expectedAccountData.Status, accountDataRead.Status) - assert.Equal(t, expectedAccountData.VoteID, accountDataRead.VoteID) - assert.Equal(t, expectedAccountData.SelectionID, accountDataRead.SelectionID) - assert.Equal(t, expectedAccountData.VoteFirstValid, accountDataRead.VoteFirstValid) - assert.Equal(t, expectedAccountData.VoteLastValid, accountDataRead.VoteLastValid) - assert.Equal(t, expectedAccountData.VoteKeyDilution, accountDataRead.VoteKeyDilution) - assert.Equal(t, expectedAccountData.AuthAddr, accountDataRead.AuthAddr) + assert.Equal(t, encoding.TrimLcAccountData(expectedAccountData), accountDataRead) } assert.False(t, rows.Next()) @@ -528,7 +516,7 @@ func TestWriterAccountTableBasic(t *testing.T) { // Now delete this account. block.BlockHeader.Round++ delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, basics.AccountData{}) + delta.Accts.Upsert(test.AccountA, ledgercore.AccountData{}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -554,9 +542,9 @@ func TestWriterAccountTableBasic(t *testing.T) { assert.Nil(t, keytype) assert.Equal(t, []byte("null"), accountData) { - accountData, err := encoding.DecodeTrimmedAccountData(accountData) + accountData, err := encoding.DecodeTrimmedLcAccountData(accountData) require.NoError(t, err) - assert.Equal(t, basics.AccountData{}, accountData) + assert.Equal(t, ledgercore.AccountData{}, accountData) } assert.False(t, rows.Next()) @@ -565,14 +553,14 @@ func TestWriterAccountTableBasic(t *testing.T) { // Simulate the scenario where an account is created and deleted in the same round. func TestWriterAccountTableCreateDeleteSameRound(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block block.BlockHeader.Round = 4 var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, basics.AccountData{}) + delta.Accts.Upsert(test.AccountA, ledgercore.AccountData{}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -617,9 +605,9 @@ func TestWriterAccountTableCreateDeleteSameRound(t *testing.T) { assert.Nil(t, keytype) assert.Equal(t, []byte("null"), accountData) { - accountData, err := encoding.DecodeTrimmedAccountData(accountData) + accountData, err := encoding.DecodeTrimmedLcAccountData(accountData) require.NoError(t, err) - assert.Equal(t, basics.AccountData{}, accountData) + assert.Equal(t, ledgercore.AccountData{}, accountData) } assert.False(t, rows.Next()) @@ -627,7 +615,7 @@ func TestWriterAccountTableCreateDeleteSameRound(t *testing.T) { } func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := bookkeeping.Block{ @@ -651,8 +639,10 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { require.NoError(t, err) var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, + delta.Accts.Upsert(test.AccountA, ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: 5}, + }, }) f := func(tx pgx.Tx) error { @@ -678,7 +668,7 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { // Now delete this account. block.BlockHeader.Round = basics.Round(5) delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, basics.AccountData{}) + delta.Accts.Upsert(test.AccountA, ledgercore.AccountData{}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -690,7 +680,7 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { } func TestWriterAccountAssetTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -701,14 +691,10 @@ func TestWriterAccountAssetTableBasic(t *testing.T) { Amount: 4, Frozen: true, } - accountData := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, - Assets: map[basics.AssetIndex]basics.AssetHolding{ - assetID: assetHolding, - }, - } var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{}, + ledgercore.AssetHoldingDelta{Holding: &assetHolding}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -753,12 +739,10 @@ func TestWriterAccountAssetTableBasic(t *testing.T) { // Now delete the asset. block.BlockHeader.Round++ - delta.ModifiedAssetHoldings = map[ledgercore.AccountAsset]bool{ - {Address: test.AccountA, Asset: assetID}: false, - } - accountData.Assets = nil delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{}, + ledgercore.AssetHoldingDelta{Deleted: true}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -786,18 +770,17 @@ func TestWriterAccountAssetTableBasic(t *testing.T) { // Simulate a scenario where an asset holding is added and deleted in the same round. func TestWriterAccountAssetTableCreateDeleteSameRound(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block block.BlockHeader.Round = basics.Round(1) assetID := basics.AssetIndex(3) - delta := ledgercore.StateDelta{ - ModifiedAssetHoldings: map[ledgercore.AccountAsset]bool{ - {Address: test.AccountA, Asset: assetID}: false, - }, - } + var delta ledgercore.StateDelta + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{}, + ledgercore.AssetHoldingDelta{Deleted: true}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -834,7 +817,7 @@ func TestWriterAccountAssetTableCreateDeleteSameRound(t *testing.T) { } func TestWriterAccountAssetTableLargeAmount(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -845,12 +828,9 @@ func TestWriterAccountAssetTableLargeAmount(t *testing.T) { Amount: math.MaxUint64, } var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, - Assets: map[basics.AssetIndex]basics.AssetHolding{ - assetID: assetHolding, - }, - }) + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{}, + ledgercore.AssetHoldingDelta{Holding: &assetHolding}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -874,7 +854,7 @@ func TestWriterAccountAssetTableLargeAmount(t *testing.T) { } func TestWriterAssetTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -885,14 +865,10 @@ func TestWriterAssetTableBasic(t *testing.T) { Total: 99999, Manager: test.AccountB, } - accountData := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, - AssetParams: map[basics.AssetIndex]basics.AssetParams{ - assetID: assetParams, - }, - } var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{Params: &assetParams}, + ledgercore.AssetHoldingDelta{}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -939,16 +915,10 @@ func TestWriterAssetTableBasic(t *testing.T) { // Now delete the asset. block.BlockHeader.Round++ - delta.Creatables = map[basics.CreatableIndex]ledgercore.ModifiedCreatable{ - basics.CreatableIndex(assetID): { - Ctype: basics.AssetCreatable, - Created: false, - Creator: test.AccountA, - }, - } - accountData.AssetParams = nil delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{Deleted: true}, + ledgercore.AssetHoldingDelta{}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -980,22 +950,17 @@ func TestWriterAssetTableBasic(t *testing.T) { // Simulate a scenario where an asset is added and deleted in the same round. func TestWriterAssetTableCreateDeleteSameRound(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block block.BlockHeader.Round = basics.Round(1) assetID := basics.AssetIndex(3) - delta := ledgercore.StateDelta{ - Creatables: map[basics.CreatableIndex]ledgercore.ModifiedCreatable{ - basics.CreatableIndex(assetID): { - Ctype: basics.AssetCreatable, - Created: false, - Creator: test.AccountA, - }, - }, - } + var delta ledgercore.StateDelta + delta.Accts.UpsertAssetResource( + test.AccountA, assetID, ledgercore.AssetParamsDelta{Deleted: true}, + ledgercore.AssetHoldingDelta{}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -1035,7 +1000,7 @@ func TestWriterAssetTableCreateDeleteSameRound(t *testing.T) { } func TestWriterAppTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -1050,14 +1015,10 @@ func TestWriterAppTableBasic(t *testing.T) { }, }, } - accountData := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, - AppParams: map[basics.AppIndex]basics.AppParams{ - appID: appParams, - }, - } var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{Params: &appParams}, + ledgercore.AppLocalStateDelta{}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -1104,16 +1065,10 @@ func TestWriterAppTableBasic(t *testing.T) { // Now delete the app. block.BlockHeader.Round++ - delta.Creatables = map[basics.CreatableIndex]ledgercore.ModifiedCreatable{ - basics.CreatableIndex(appID): { - Ctype: basics.AppCreatable, - Created: false, - Creator: test.AccountA, - }, - } - accountData.AppParams = nil delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{Deleted: true}, + ledgercore.AppLocalStateDelta{}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -1145,22 +1100,17 @@ func TestWriterAppTableBasic(t *testing.T) { // Simulate a scenario where an app is added and deleted in the same round. func TestWriterAppTableCreateDeleteSameRound(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block block.BlockHeader.Round = basics.Round(1) appID := basics.AppIndex(3) - delta := ledgercore.StateDelta{ - Creatables: map[basics.CreatableIndex]ledgercore.ModifiedCreatable{ - basics.CreatableIndex(appID): { - Ctype: basics.AppCreatable, - Created: false, - Creator: test.AccountA, - }, - }, - } + var delta ledgercore.StateDelta + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{Deleted: true}, + ledgercore.AppLocalStateDelta{}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -1201,7 +1151,7 @@ func TestWriterAppTableCreateDeleteSameRound(t *testing.T) { } func TestWriterAccountAppTableBasic(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block @@ -1215,14 +1165,10 @@ func TestWriterAccountAppTableBasic(t *testing.T) { }, }, } - accountData := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: 5}, - AppLocalStates: map[basics.AppIndex]basics.AppLocalState{ - appID: appLocalState, - }, - } var delta ledgercore.StateDelta - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{}, + ledgercore.AppLocalStateDelta{LocalState: &appLocalState}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -1269,12 +1215,10 @@ func TestWriterAccountAppTableBasic(t *testing.T) { // Now delete the app. block.BlockHeader.Round++ - delta.ModifiedAppLocalStates = map[ledgercore.AccountApp]bool{ - {Address: test.AccountA, App: appID}: false, - } - accountData.AppLocalStates = nil delta.Accts = ledgercore.AccountDeltas{} - delta.Accts.Upsert(test.AccountA, accountData) + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{}, + ledgercore.AppLocalStateDelta{Deleted: true}) err = pgutil.TxWithRetry(db, serializable, f, nil) require.NoError(t, err) @@ -1306,18 +1250,17 @@ func TestWriterAccountAppTableBasic(t *testing.T) { // Simulate a scenario where an account app is added and deleted in the same round. func TestWriterAccountAppTableCreateDeleteSameRound(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() var block bookkeeping.Block block.BlockHeader.Round = basics.Round(1) appID := basics.AppIndex(3) - delta := ledgercore.StateDelta{ - ModifiedAppLocalStates: map[ledgercore.AccountApp]bool{ - {Address: test.AccountA, App: appID}: false, - }, - } + var delta ledgercore.StateDelta + delta.Accts.UpsertAppResource( + test.AccountA, appID, ledgercore.AppParamsDelta{}, + ledgercore.AppLocalStateDelta{Deleted: true}) f := func(tx pgx.Tx) error { w, err := writer.MakeWriter(tx) @@ -1357,7 +1300,7 @@ func TestWriterAccountAppTableCreateDeleteSameRound(t *testing.T) { } func TestAddBlockInvalidInnerAsset(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() callWithBadInner := test.MakeCreateAppTxn(test.AccountA) @@ -1391,7 +1334,7 @@ func TestAddBlockInvalidInnerAsset(t *testing.T) { } func TestWriterAddBlockInnerTxnsAssetCreate(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() // App call with inner txns, should be intra 0, 1, 2, 3, 4 @@ -1529,7 +1472,7 @@ func TestWriterAddBlockInnerTxnsAssetCreate(t *testing.T) { } func TestWriterAccountTotals(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() // Set empty account totals. @@ -1567,7 +1510,7 @@ func TestWriterAccountTotals(t *testing.T) { } func TestWriterAddBlock0(t *testing.T) { - db, shutdownFunc := setupPostgres(t) + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() block := test.MakeGenesisBlock() diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 8a3d20784..6fc9b44d5 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -21,7 +21,6 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/protocol" "github.com/jackc/pgconn" "github.com/jackc/pgerrcode" @@ -179,75 +178,60 @@ func (db *IndexerDb) init(opts idb.IndexerDbOptions) (chan struct{}, error) { return db.runAvailableMigrations() } -// Returns all addresses referenced in `block`. -func getBlockAddresses(block *bookkeeping.Block) map[basics.Address]struct{} { - // Reserve a reasonable memory size for the map. - res := make(map[basics.Address]struct{}, len(block.Payset)+2) +// Preload asset and app creators. +func prepareCreators(l *ledger_for_evaluator.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { + assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) - res[block.FeeSink] = struct{}{} - res[block.RewardsPool] = struct{}{} - for _, stib := range block.Payset { - addFunc := func(address basics.Address) { - res[address] = struct{}{} - } - accounting.GetTransactionParticipants(&stib.SignedTxnWithAD, true, addFunc) + assets, err := l.GetAssetCreator(assetsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) + } + apps, err := l.GetAppCreator(appsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) } - return res + return assets, apps, nil } -func prepareEvalResources(l *ledger_for_evaluator.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { - addresses := getBlockAddresses(block) - assets := make(map[basics.AssetIndex]struct{}) - apps := make(map[basics.AppIndex]struct{}) +// Preload account data and account resources. +func prepareAccountsResources(l *ledger_for_evaluator.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { + addressesReq, resourcesReq := + accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) - for _, stib := range block.Payset { - switch stib.Txn.Type { - case protocol.AssetConfigTx: - if stib.Txn.ConfigAsset != 0 { - assets[stib.Txn.ConfigAsset] = struct{}{} - } - case protocol.AssetTransferTx: - if stib.Txn.XferAsset != 0 { - assets[stib.Txn.XferAsset] = struct{}{} - } - case protocol.AssetFreezeTx: - if stib.Txn.FreezeAsset != 0 { - assets[stib.Txn.FreezeAsset] = struct{}{} - } - case protocol.ApplicationCallTx: - if stib.Txn.ApplicationID != 0 { - apps[stib.Txn.ApplicationID] = struct{}{} - } - } + accounts, err := l.LookupWithoutRewards(addressesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) } - - res := ledger.EvalForIndexerResources{ - Accounts: nil, - Creators: make(map[ledger.Creatable]ledger.FoundAddress), + resources, err := l.LookupResources(resourcesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) } - assetCreators, err := l.GetAssetCreator(assets) + return accounts, resources, nil +} + +// Preload all resources (account data, account resources, asset/app creators) for the +// evaluator. +func prepareEvalResources(l *ledger_for_evaluator.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { + assetCreators, appCreators, err := prepareCreators(l, block.Payset) if err != nil { return ledger.EvalForIndexerResources{}, fmt.Errorf("prepareEvalResources() err: %w", err) } + + res := ledger.EvalForIndexerResources{ + Accounts: nil, + Resources: nil, + Creators: make(map[ledger.Creatable]ledger.FoundAddress), + } + for index, foundAddress := range assetCreators { creatable := ledger.Creatable{ Index: basics.CreatableIndex(index), Type: basics.AssetCreatable, } res.Creators[creatable] = foundAddress - - if foundAddress.Exists { - addresses[foundAddress.Address] = struct{}{} - } - } - - appCreators, err := l.GetAppCreator(apps) - if err != nil { - return ledger.EvalForIndexerResources{}, - fmt.Errorf("prepareEvalResources() err: %w", err) } for index, foundAddress := range appCreators { creatable := ledger.Creatable{ @@ -255,13 +239,9 @@ func prepareEvalResources(l *ledger_for_evaluator.LedgerForEvaluator, block *boo Type: basics.AppCreatable, } res.Creators[creatable] = foundAddress - - if foundAddress.Exists { - addresses[foundAddress.Address] = struct{}{} - } } - res.Accounts, err = l.LookupWithoutRewards(addresses) + res.Accounts, res.Resources, err = prepareAccountsResources(l, block.Payset, assetCreators, appCreators) if err != nil { return ledger.EvalForIndexerResources{}, fmt.Errorf("prepareEvalResources() err: %w", err) @@ -447,15 +427,16 @@ func (db *IndexerDb) LoadGenesis(genesis bookkeeping.Genesis) error { if len(alloc.State.AssetParams) > 0 || len(alloc.State.Assets) > 0 { return fmt.Errorf("LoadGenesis() genesis account[%d] has unhandled asset", ai) } + accountData := ledgercore.ToAccountData(alloc.State) _, err = tx.Exec( context.Background(), setAccountStatementName, addr[:], alloc.State.MicroAlgos.Raw, - encoding.EncodeTrimmedAccountData(encoding.TrimAccountData(alloc.State)), 0) + encoding.EncodeTrimmedLcAccountData(encoding.TrimLcAccountData(accountData)), 0) if err != nil { return fmt.Errorf("LoadGenesis() error setting genesis account[%d], %w", ai, err) } - totals.AddAccount(proto, alloc.State, &ot) + totals.AddAccount(proto, accountData, &ot) } err = db.setMetastate( @@ -1126,37 +1107,22 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) { var localStateClosedBytes []byte var localStateDeletedBytes []byte - var err error - - if req.opts.IncludeAssetHoldings && req.opts.IncludeAssetParams { - err = req.rows.Scan( - &addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr, - &holdingAssetids, &holdingAmount, &holdingFrozen, &holdingCreatedBytes, &holdingClosedBytes, &holdingDeletedBytes, - &assetParamsIds, &assetParamsStr, &assetParamsCreatedBytes, &assetParamsClosedBytes, &assetParamsDeletedBytes, - &appParamIndexes, &appParams, &appCreatedBytes, &appClosedBytes, &appDeletedBytes, &localStateAppIds, &localStates, - &localStateCreatedBytes, &localStateClosedBytes, &localStateDeletedBytes, - ) - } else if req.opts.IncludeAssetHoldings { - err = req.rows.Scan( - &addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr, - &holdingAssetids, &holdingAmount, &holdingFrozen, &holdingCreatedBytes, &holdingClosedBytes, &holdingDeletedBytes, - &appParamIndexes, &appParams, &appCreatedBytes, &appClosedBytes, &appDeletedBytes, &localStateAppIds, &localStates, - &localStateCreatedBytes, &localStateClosedBytes, &localStateDeletedBytes, - ) - } else if req.opts.IncludeAssetParams { - err = req.rows.Scan( - &addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr, - &assetParamsIds, &assetParamsStr, &assetParamsCreatedBytes, &assetParamsClosedBytes, &assetParamsDeletedBytes, - &appParamIndexes, &appParams, &appCreatedBytes, &appClosedBytes, &appDeletedBytes, &localStateAppIds, &localStates, - &localStateCreatedBytes, &localStateClosedBytes, &localStateDeletedBytes, - ) - } else { - err = req.rows.Scan( - &addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr, - &appParamIndexes, &appParams, &appCreatedBytes, &appClosedBytes, &appDeletedBytes, &localStateAppIds, &localStates, - &localStateCreatedBytes, &localStateClosedBytes, &localStateDeletedBytes, - ) + // build list of columns to scan using include options like buildAccountQuery + cols := []interface{}{&addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr} + if req.opts.IncludeAssetHoldings { + cols = append(cols, &holdingAssetids, &holdingAmount, &holdingFrozen, &holdingCreatedBytes, &holdingClosedBytes, &holdingDeletedBytes) + } + if req.opts.IncludeAssetParams { + cols = append(cols, &assetParamsIds, &assetParamsStr, &assetParamsCreatedBytes, &assetParamsClosedBytes, &assetParamsDeletedBytes) + } + if req.opts.IncludeAppParams { + cols = append(cols, &appParamIndexes, &appParams, &appCreatedBytes, &appClosedBytes, &appDeletedBytes) } + if req.opts.IncludeAppLocalState { + cols = append(cols, &localStateAppIds, &localStates, &localStateCreatedBytes, &localStateClosedBytes, &localStateDeletedBytes) + } + + err := req.rows.Scan(cols...) if err != nil { err = fmt.Errorf("account scan err %v", err) req.out <- idb.AccountRow{Error: err} @@ -1182,8 +1148,8 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) { } { - var ad basics.AccountData - ad, err = encoding.DecodeTrimmedAccountData(accountDataJSONStr) + var ad ledgercore.AccountData + ad, err = encoding.DecodeTrimmedLcAccountData(accountDataJSONStr) if err != nil { err = fmt.Errorf("account decode err (%s) %v", accountDataJSONStr, err) req.out <- idb.AccountRow{Error: err} @@ -1229,6 +1195,11 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) { if ad.TotalExtraAppPages != 0 { account.AppsTotalExtraPages = uint64Ptr(uint64(ad.TotalExtraAppPages)) } + + account.TotalAppsOptedIn = ad.TotalAppLocalStates + account.TotalCreatedApps = ad.TotalAppParams + account.TotalAssetsOptedIn = ad.TotalAssets + account.TotalCreatedAssets = ad.TotalAssetParams } if account.Status == "NotParticipating" { @@ -1324,7 +1295,7 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) { OptedOutAtRound: holdingClosed[i], OptedInAtRound: holdingCreated[i], Deleted: holdingDeleted[i], - } // TODO: set Creator to asset creator addr string + } av = append(av, tah) } account.Assets = new([]models.AssetHolding) @@ -1713,8 +1684,19 @@ func (db *IndexerDb) GetAccounts(ctx context.Context, opts idb.AccountQueryOptio return out, round } + // Enforce max combined # of app & asset resources per account limit, if set + if opts.MaxResources != 0 { + err = db.checkAccountResourceLimit(ctx, tx, opts) + if err != nil { + out <- idb.AccountRow{Error: err} + close(out) + tx.Rollback(ctx) + return out, round + } + } + // Construct query for fetching accounts... - query, whereArgs := db.buildAccountQuery(opts) + query, whereArgs := db.buildAccountQuery(opts, false) req := &getAccountsRequest{ opts: opts, blockheader: blockheader, @@ -1738,9 +1720,114 @@ func (db *IndexerDb) GetAccounts(ctx context.Context, opts idb.AccountQueryOptio return out, round } -func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query string, whereArgs []interface{}) { +func (db *IndexerDb) checkAccountResourceLimit(ctx context.Context, tx pgx.Tx, opts idb.AccountQueryOptions) error { + // skip check if no resources are requested + if !opts.IncludeAssetHoldings && !opts.IncludeAssetParams && !opts.IncludeAppLocalState && !opts.IncludeAppParams { + return nil + } + + // make a copy of the filters requested + o := opts + var countOnly bool + + if opts.IncludeDeleted { + // if IncludeDeleted is set, need to construct a query (preserving filters) to count deleted values that would be returned from + // asset, app, account_asset, account_app + countOnly = true + } else { + // if IncludeDeleted is not set, query AccountData with no resources (preserving filters), to read ad.TotalX counts inside + o.IncludeAssetHoldings = false + o.IncludeAssetParams = false + o.IncludeAppLocalState = false + o.IncludeAppParams = false + } + + query, whereArgs := db.buildAccountQuery(o, countOnly) + rows, err := tx.Query(ctx, query, whereArgs...) + if err != nil { + return fmt.Errorf("account limit query %#v err %v", query, err) + } + defer rows.Close() + for rows.Next() { + var addr []byte + var microalgos uint64 + var rewardstotal uint64 + var createdat sql.NullInt64 + var closedat sql.NullInt64 + var deleted sql.NullBool + var rewardsbase uint64 + var keytype *string + var accountDataJSONStr []byte + var holdingCount, assetCount, appCount, lsCount uint64 + cols := []interface{}{&addr, µalgos, &rewardstotal, &createdat, &closedat, &deleted, &rewardsbase, &keytype, &accountDataJSONStr} + if countOnly { + if o.IncludeAssetHoldings { + cols = append(cols, &holdingCount) + } + if o.IncludeAssetParams { + cols = append(cols, &assetCount) + } + if o.IncludeAppParams { + cols = append(cols, &appCount) + } + if o.IncludeAppLocalState { + cols = append(cols, &lsCount) + } + } + err := rows.Scan(cols...) + if err != nil { + return fmt.Errorf("account limit scan err %v", err) + } + + var ad ledgercore.AccountData + ad, err = encoding.DecodeTrimmedLcAccountData(accountDataJSONStr) + if err != nil { + return fmt.Errorf("account limit decode err (%s) %v", accountDataJSONStr, err) + } + + // check limit against filters (only count what would be returned) + var resultCount, totalAssets, totalAssetParams, totalAppLocalStates, totalAppParams uint64 + if countOnly { + totalAssets = holdingCount + totalAssetParams = assetCount + totalAppLocalStates = lsCount + totalAppParams = appCount + } else { + totalAssets = ad.TotalAssets + totalAssetParams = ad.TotalAssetParams + totalAppLocalStates = ad.TotalAppLocalStates + totalAppParams = ad.TotalAppParams + } + if opts.IncludeAssetHoldings { + resultCount += totalAssets + } + if opts.IncludeAssetParams { + resultCount += totalAssetParams + } + if opts.IncludeAppLocalState { + resultCount += totalAppLocalStates + } + if opts.IncludeAppParams { + resultCount += totalAppParams + } + if resultCount > opts.MaxResources { + var aaddr basics.Address + copy(aaddr[:], addr) + return idb.MaxAPIResourcesPerAccountError{ + Address: aaddr, + TotalAppLocalStates: totalAppLocalStates, + TotalAppParams: totalAppParams, + TotalAssets: totalAssets, + TotalAssetParams: totalAssetParams, + } + } + } + return nil +} + +func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions, countOnly bool) (query string, whereArgs []interface{}) { // Construct query for fetching accounts... - const maxWhereParts = 14 + const maxWhereParts = 9 whereParts := make([]string, 0, maxWhereParts) whereArgs = make([]interface{}, 0, maxWhereParts) partNumber := 1 @@ -1790,7 +1877,7 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query stri partNumber++ } if !opts.IncludeDeleted { - whereParts = append(whereParts, "coalesce(a.deleted, false) = false") + whereParts = append(whereParts, "NOT a.deleted") } if len(opts.EqualToAuthAddr) > 0 { whereParts = append(whereParts, fmt.Sprintf("a.account_data ->> 'spend' = $%d", partNumber)) @@ -1814,42 +1901,91 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query stri if opts.Limit != 0 { query += fmt.Sprintf(" LIMIT %d", opts.Limit) } - // TODO: asset holdings and asset params are optional, but practically always used. Either make them actually always on, or make app-global and app-local clauses also optional (they are currently always on). + withClauses = append(withClauses, "qaccounts AS ("+query+")") query = "WITH " + strings.Join(withClauses, ", ") - if opts.IncludeDeleted { - if opts.IncludeAssetHoldings { - query += `, qaa AS (SELECT xa.addr, json_agg(aa.assetid) as haid, json_agg(aa.amount) as hamt, json_agg(aa.frozen) as hf, json_agg(aa.created_at) as holding_created_at, json_agg(aa.closed_at) as holding_closed_at, json_agg(coalesce(aa.deleted, false)) as holding_deleted FROM account_asset aa JOIN qaccounts xa ON aa.addr = xa.addr GROUP BY 1)` + + // build nested selects for querying app/asset data associated with an address + if opts.IncludeAssetHoldings { + var where, selectCols string + if !opts.IncludeDeleted { + where = ` WHERE NOT aa.deleted` } - if opts.IncludeAssetParams { - query += `, qap AS (SELECT ya.addr, json_agg(ap.index) as paid, json_agg(ap.params) as pp, json_agg(ap.created_at) as asset_created_at, json_agg(ap.closed_at) as asset_closed_at, json_agg(ap.deleted) as asset_deleted FROM asset ap JOIN qaccounts ya ON ap.creator_addr = ya.addr GROUP BY 1)` + if countOnly { + selectCols = `count(*) as holding_count` + } else { + selectCols = `json_agg(aa.assetid) as haid, json_agg(aa.amount) as hamt, json_agg(aa.frozen) as hf, json_agg(aa.created_at) as holding_created_at, json_agg(aa.closed_at) as holding_closed_at, json_agg(aa.deleted) as holding_deleted` } - // app - query += `, qapp AS (SELECT app.creator as addr, json_agg(app.index) as papps, json_agg(app.params) as ppa, json_agg(app.created_at) as app_created_at, json_agg(app.closed_at) as app_closed_at, json_agg(app.deleted) as app_deleted FROM app JOIN qaccounts ON qaccounts.addr = app.creator GROUP BY 1)` - // app localstate - query += `, qls AS (SELECT la.addr, json_agg(la.app) as lsapps, json_agg(la.localstate) as lsls, json_agg(la.created_at) as ls_created_at, json_agg(la.closed_at) as ls_closed_at, json_agg(la.deleted) as ls_deleted FROM account_app la JOIN qaccounts ON qaccounts.addr = la.addr GROUP BY 1)` - } else { - if opts.IncludeAssetHoldings { - query += `, qaa AS (SELECT xa.addr, json_agg(aa.assetid) as haid, json_agg(aa.amount) as hamt, json_agg(aa.frozen) as hf, json_agg(aa.created_at) as holding_created_at, json_agg(aa.closed_at) as holding_closed_at, json_agg(coalesce(aa.deleted, false)) as holding_deleted FROM account_asset aa JOIN qaccounts xa ON aa.addr = xa.addr WHERE coalesce(aa.deleted, false) = false GROUP BY 1)` + query += `, qaa AS (SELECT xa.addr, ` + selectCols + ` FROM account_asset aa JOIN qaccounts xa ON aa.addr = xa.addr` + where + ` GROUP BY 1)` + } + if opts.IncludeAssetParams { + var where, selectCols string + if !opts.IncludeDeleted { + where = ` WHERE NOT ap.deleted` } - if opts.IncludeAssetParams { - query += `, qap AS (SELECT ya.addr, json_agg(ap.index) as paid, json_agg(ap.params) as pp, json_agg(ap.created_at) as asset_created_at, json_agg(ap.closed_at) as asset_closed_at, json_agg(ap.deleted) as asset_deleted FROM asset ap JOIN qaccounts ya ON ap.creator_addr = ya.addr WHERE coalesce(ap.deleted, false) = false GROUP BY 1)` + if countOnly { + selectCols = `count(*) as asset_count` + } else { + selectCols = `json_agg(ap.index) as paid, json_agg(ap.params) as pp, json_agg(ap.created_at) as asset_created_at, json_agg(ap.closed_at) as asset_closed_at, json_agg(ap.deleted) as asset_deleted` + } + query += `, qap AS (SELECT ya.addr, ` + selectCols + ` FROM asset ap JOIN qaccounts ya ON ap.creator_addr = ya.addr` + where + ` GROUP BY 1)` + } + if opts.IncludeAppParams { + var where, selectCols string + if !opts.IncludeDeleted { + where = ` WHERE NOT app.deleted` + } + if countOnly { + selectCols = `count(*) as app_count` + } else { + selectCols = `json_agg(app.index) as papps, json_agg(app.params) as ppa, json_agg(app.created_at) as app_created_at, json_agg(app.closed_at) as app_closed_at, json_agg(app.deleted) as app_deleted` } - // app - query += `, qapp AS (SELECT app.creator as addr, json_agg(app.index) as papps, json_agg(app.params) as ppa, json_agg(app.created_at) as app_created_at, json_agg(app.closed_at) as app_closed_at, json_agg(app.deleted) as app_deleted FROM app JOIN qaccounts ON qaccounts.addr = app.creator WHERE coalesce(app.deleted, false) = false GROUP BY 1)` - // app localstate - query += `, qls AS (SELECT la.addr, json_agg(la.app) as lsapps, json_agg(la.localstate) as lsls, json_agg(la.created_at) as ls_created_at, json_agg(la.closed_at) as ls_closed_at, json_agg(la.deleted) as ls_deleted FROM account_app la JOIN qaccounts ON qaccounts.addr = la.addr WHERE coalesce(la.deleted, false) = false GROUP BY 1)` + query += `, qapp AS (SELECT app.creator as addr, ` + selectCols + ` FROM app JOIN qaccounts ON qaccounts.addr = app.creator` + where + ` GROUP BY 1)` + } + if opts.IncludeAppLocalState { + var where, selectCols string + if !opts.IncludeDeleted { + where = ` WHERE NOT la.deleted` + } + if countOnly { + selectCols = `count(*) as app_count` + } else { + selectCols = `json_agg(la.app) as lsapps, json_agg(la.localstate) as lsls, json_agg(la.created_at) as ls_created_at, json_agg(la.closed_at) as ls_closed_at, json_agg(la.deleted) as ls_deleted` + } + query += `, qls AS (SELECT la.addr, ` + selectCols + ` FROM account_app la JOIN qaccounts ON qaccounts.addr = la.addr` + where + ` GROUP BY 1)` } // query results query += ` SELECT za.addr, za.microalgos, za.rewards_total, za.created_at, za.closed_at, za.deleted, za.rewardsbase, za.keytype, za.account_data` if opts.IncludeAssetHoldings { - query += `, qaa.haid, qaa.hamt, qaa.hf, qaa.holding_created_at, qaa.holding_closed_at, qaa.holding_deleted` + if countOnly { + query += `, qaa.holding_count` + } else { + query += `, qaa.haid, qaa.hamt, qaa.hf, qaa.holding_created_at, qaa.holding_closed_at, qaa.holding_deleted` + } } if opts.IncludeAssetParams { - query += `, qap.paid, qap.pp, qap.asset_created_at, qap.asset_closed_at, qap.asset_deleted` + if countOnly { + query += `, qap.asset_count` + } else { + query += `, qap.paid, qap.pp, qap.asset_created_at, qap.asset_closed_at, qap.asset_deleted` + } + } + if opts.IncludeAppParams { + if countOnly { + query += `, qapp.app_count` + } else { + query += `, qapp.papps, qapp.ppa, qapp.app_created_at, qapp.app_closed_at, qapp.app_deleted` + } + } + if opts.IncludeAppLocalState { + if countOnly { + query += `, qls.ls_count` + } else { + query += `, qls.lsapps, qls.lsls, qls.ls_created_at, qls.ls_closed_at, qls.ls_deleted` + } } - query += `, qapp.papps, qapp.ppa, qapp.app_created_at, qapp.app_closed_at, qapp.app_deleted, qls.lsapps, qls.lsls, qls.ls_created_at, qls.ls_closed_at, qls.ls_deleted FROM qaccounts za` + query += ` FROM qaccounts za` // join everything together if opts.IncludeAssetHoldings { @@ -1858,7 +1994,13 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query stri if opts.IncludeAssetParams { query += ` LEFT JOIN qap ON za.addr = qap.addr` } - query += " LEFT JOIN qapp ON za.addr = qapp.addr LEFT JOIN qls ON qls.addr = za.addr ORDER BY za.addr ASC;" + if opts.IncludeAppParams { + query += ` LEFT JOIN qapp ON za.addr = qapp.addr` + } + if opts.IncludeAppLocalState { + query += ` LEFT JOIN qls ON za.addr = qls.addr` + } + query += ` ORDER BY za.addr ASC;` return query, whereArgs } @@ -1901,7 +2043,7 @@ func (db *IndexerDb) Assets(ctx context.Context, filter idb.AssetsQuery) (<-chan partNumber++ } if !filter.IncludeDeleted { - whereParts = append(whereParts, "coalesce(a.deleted, false) = false") + whereParts = append(whereParts, "NOT a.deleted") } if len(whereParts) > 0 { whereStr := strings.Join(whereParts, " AND ") @@ -1993,6 +2135,16 @@ func (db *IndexerDb) AssetBalances(ctx context.Context, abq idb.AssetBalanceQuer whereArgs = append(whereArgs, abq.AssetID) partNumber++ } + if abq.AssetIDGT != 0 { + whereParts = append(whereParts, fmt.Sprintf("aa.assetid > $%d", partNumber)) + whereArgs = append(whereArgs, abq.AssetIDGT) + partNumber++ + } + if abq.Address != nil { + whereParts = append(whereParts, fmt.Sprintf("aa.addr = $%d", partNumber)) + whereArgs = append(whereArgs, abq.Address) + partNumber++ + } if abq.AmountGT != nil { whereParts = append(whereParts, fmt.Sprintf("aa.amount > $%d", partNumber)) whereArgs = append(whereArgs, *abq.AmountGT) @@ -2009,13 +2161,14 @@ func (db *IndexerDb) AssetBalances(ctx context.Context, abq idb.AssetBalanceQuer partNumber++ } if !abq.IncludeDeleted { - whereParts = append(whereParts, "coalesce(aa.deleted, false) = false") + whereParts = append(whereParts, "NOT aa.deleted") } query := `SELECT addr, assetid, amount, frozen, created_at, closed_at, deleted FROM account_asset aa` if len(whereParts) > 0 { query += " WHERE " + strings.Join(whereParts, " AND ") } - query += " ORDER BY addr ASC" + query += " ORDER BY addr, assetid ASC" + if abq.Limit > 0 { query += fmt.Sprintf(" LIMIT %d", abq.Limit) } @@ -2085,40 +2238,40 @@ func (db *IndexerDb) yieldAssetBalanceThread(rows pgx.Rows, out chan<- idb.Asset } // Applications is part of idb.IndexerDB -func (db *IndexerDb) Applications(ctx context.Context, filter *models.SearchForApplicationsParams) (<-chan idb.ApplicationRow, uint64) { +func (db *IndexerDb) Applications(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.ApplicationRow, uint64) { out := make(chan idb.ApplicationRow, 1) - if filter == nil { - out <- idb.ApplicationRow{Error: fmt.Errorf("no arguments provided to application search")} - close(out) - return out, 0 - } query := `SELECT index, creator, params, created_at, closed_at, deleted FROM app ` - const maxWhereParts = 30 + const maxWhereParts = 4 whereParts := make([]string, 0, maxWhereParts) whereArgs := make([]interface{}, 0, maxWhereParts) partNumber := 1 - if filter.ApplicationId != nil { + if filter.ApplicationID != 0 { whereParts = append(whereParts, fmt.Sprintf("index = $%d", partNumber)) - whereArgs = append(whereArgs, *filter.ApplicationId) + whereArgs = append(whereArgs, filter.ApplicationID) + partNumber++ + } + if filter.Address != nil { + whereParts = append(whereParts, fmt.Sprintf("creator = $%d", partNumber)) + whereArgs = append(whereArgs, filter.Address) partNumber++ } - if filter.Next != nil { + if filter.ApplicationIDGreaterThan != 0 { whereParts = append(whereParts, fmt.Sprintf("index > $%d", partNumber)) - whereArgs = append(whereArgs, *filter.Next) + whereArgs = append(whereArgs, filter.ApplicationIDGreaterThan) partNumber++ } - if filter.IncludeAll == nil || !(*filter.IncludeAll) { - whereParts = append(whereParts, "coalesce(deleted, false) = false") + if !filter.IncludeDeleted { + whereParts = append(whereParts, "NOT deleted") } if len(whereParts) > 0 { whereStr := strings.Join(whereParts, " AND ") query += " WHERE " + whereStr } query += " ORDER BY 1" - if filter.Limit != nil { - query += fmt.Sprintf(" LIMIT %d", *filter.Limit) + if filter.Limit != 0 { + query += fmt.Sprintf(" LIMIT %d", filter.Limit) } tx, err := db.db.BeginTx(ctx, readonlyRepeatableRead) @@ -2174,7 +2327,7 @@ func (db *IndexerDb) yieldApplicationsThread(rows pgx.Rows, out chan idb.Applica rec.Application.Deleted = deleted ap, err := encoding.DecodeAppParams(paramsjson) if err != nil { - rec.Error = fmt.Errorf("app=%d json err, %v", index, err) + rec.Error = fmt.Errorf("app=%d json err: %w", index, err) out <- rec break } @@ -2208,6 +2361,113 @@ func (db *IndexerDb) yieldApplicationsThread(rows pgx.Rows, out chan idb.Applica } } +// AppLocalState is part of idb.IndexerDB +func (db *IndexerDb) AppLocalState(ctx context.Context, filter idb.ApplicationQuery) (<-chan idb.AppLocalStateRow, uint64) { + out := make(chan idb.AppLocalStateRow, 1) + + query := `SELECT app, addr, localstate, created_at, closed_at, deleted FROM account_app ` + + const maxWhereParts = 4 + whereParts := make([]string, 0, maxWhereParts) + whereArgs := make([]interface{}, 0, maxWhereParts) + partNumber := 1 + if filter.ApplicationID != 0 { + whereParts = append(whereParts, fmt.Sprintf("app = $%d", partNumber)) + whereArgs = append(whereArgs, filter.ApplicationID) + partNumber++ + } + if filter.Address != nil { + whereParts = append(whereParts, fmt.Sprintf("addr = $%d", partNumber)) + whereArgs = append(whereArgs, filter.Address) + partNumber++ + } + if filter.ApplicationIDGreaterThan != 0 { + whereParts = append(whereParts, fmt.Sprintf("app > $%d", partNumber)) + whereArgs = append(whereArgs, filter.ApplicationIDGreaterThan) + partNumber++ + } + if !filter.IncludeDeleted { + whereParts = append(whereParts, "NOT deleted") + } + if len(whereParts) > 0 { + whereStr := strings.Join(whereParts, " AND ") + query += " WHERE " + whereStr + } + query += " ORDER BY 1" + if filter.Limit != 0 { + query += fmt.Sprintf(" LIMIT %d", filter.Limit) + } + + tx, err := db.db.BeginTx(ctx, readonlyRepeatableRead) + if err != nil { + out <- idb.AppLocalStateRow{Error: err} + close(out) + return out, 0 + } + + round, err := db.getMaxRoundAccounted(ctx, tx) + if err != nil { + out <- idb.AppLocalStateRow{Error: err} + close(out) + tx.Rollback(ctx) + return out, round + } + + rows, err := tx.Query(ctx, query, whereArgs...) + if err != nil { + out <- idb.AppLocalStateRow{Error: err} + close(out) + tx.Rollback(ctx) + return out, round + } + + go func() { + db.yieldAppLocalStateThread(rows, out) + close(out) + tx.Rollback(ctx) + }() + return out, round +} + +func (db *IndexerDb) yieldAppLocalStateThread(rows pgx.Rows, out chan idb.AppLocalStateRow) { + defer rows.Close() + + for rows.Next() { + var index uint64 + var address []byte + var statejson []byte + var created *uint64 + var closed *uint64 + var deleted *bool + err := rows.Scan(&index, &address, &statejson, &created, &closed, &deleted) + if err != nil { + out <- idb.AppLocalStateRow{Error: err} + break + } + var rec idb.AppLocalStateRow + rec.AppLocalState.Id = index + rec.AppLocalState.OptedInAtRound = created + rec.AppLocalState.ClosedOutAtRound = closed + rec.AppLocalState.Deleted = deleted + + ls, err := encoding.DecodeAppLocalState(statejson) + if err != nil { + rec.Error = fmt.Errorf("app=%d json err: %w", index, err) + out <- rec + break + } + rec.AppLocalState.Schema = models.ApplicationStateSchema{ + NumByteSlice: ls.Schema.NumByteSlice, + NumUint: ls.Schema.NumUint, + } + rec.AppLocalState.KeyValue = tealKeyValueToModel(ls.KeyValue) + out <- rec + } + if err := rows.Err(); err != nil { + out <- idb.AppLocalStateRow{Error: err} + } +} + // Health is part of idb.IndexerDB func (db *IndexerDb) Health(ctx context.Context) (idb.Health, error) { migrationRequired := false @@ -2279,41 +2539,32 @@ func (db *IndexerDb) GetSpecialAccounts(ctx context.Context) (transactions.Speci return accounts, nil } -// GetAccountData returns account data for the given addresses. For accounts that are -// not found, empty AccountData is returned. This function is only used for debugging. -func (db *IndexerDb) GetAccountData(addresses []basics.Address) (map[basics.Address]basics.AccountData, error) { +// GetAccountState returns account data and account resources for the given input. +// For accounts that are not found, empty AccountData is returned. +// This function is only used for debugging. +func (db *IndexerDb) GetAccountState(addressesReq map[basics.Address]struct{}, resourcesReq map[basics.Address]map[ledger.Creatable]struct{}) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { tx, err := db.db.BeginTx(context.Background(), readonlyRepeatableRead) if err != nil { - return nil, fmt.Errorf("GetAccountData() begin tx err: %w", err) + return nil, nil, fmt.Errorf("GetAccountState() begin tx err: %w", err) } defer tx.Rollback(context.Background()) l, err := ledger_for_evaluator.MakeLedgerForEvaluator(tx, basics.Round(0)) if err != nil { - return nil, fmt.Errorf("GetAccountData() err: %w", err) + return nil, nil, fmt.Errorf("GetAccountState() err: %w", err) } defer l.Close() - addressesMap := make(map[basics.Address]struct{}, len(addresses)) - for _, address := range addresses { - addressesMap[address] = struct{}{} - } - - accountDataMap, err := l.LookupWithoutRewards(addressesMap) + accounts, err := l.LookupWithoutRewards(addressesReq) if err != nil { - return nil, fmt.Errorf("GetAccountData() err: %w", err) + return nil, nil, fmt.Errorf("GetAccountState() err: %w", err) } - - res := make(map[basics.Address]basics.AccountData, len(accountDataMap)) - for address, accountData := range accountDataMap { - if accountData == nil { - res[address] = basics.AccountData{} - } else { - res[address] = *accountData - } + resources, err := l.LookupResources(resourcesReq) + if err != nil { + return nil, nil, fmt.Errorf("GetAccountState() err: %w", err) } - return res, nil + return accounts, resources, nil } // GetNetworkState is part of idb.IndexerDB diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index 5fbf67877..deff7e28a 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -407,7 +407,7 @@ func TestRekeyBasic(t *testing.T) { err = row.Scan(&accountDataStr) assert.NoError(t, err, "querying account data") - ad, err := encoding.DecodeTrimmedAccountData(accountDataStr) + ad, err := encoding.DecodeTrimmedLcAccountData(accountDataStr) require.NoError(t, err, "failed to parse account data json") assert.Equal(t, test.AccountB, ad.AuthAddr) } @@ -444,7 +444,7 @@ func TestRekeyToItself(t *testing.T) { err = row.Scan(&accountDataStr) assert.NoError(t, err, "querying account data") - ad, err := encoding.DecodeTrimmedAccountData(accountDataStr) + ad, err := encoding.DecodeTrimmedLcAccountData(accountDataStr) require.NoError(t, err, "failed to parse account data json") assert.Equal(t, basics.Address{}, ad.AuthAddr) } @@ -479,7 +479,7 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { err = row.Scan(&accountDataStr) assert.NoError(t, err, "querying account data") - ad, err := encoding.DecodeTrimmedAccountData(accountDataStr) + ad, err := encoding.DecodeTrimmedLcAccountData(accountDataStr) require.NoError(t, err, "failed to parse account data json") assert.Equal(t, test.AccountC, ad.AuthAddr) } @@ -860,10 +860,10 @@ func TestAppExtraPages(t *testing.T) { require.NoError(t, err) require.Equal(t, uint32(1), ap.ExtraProgramPages) - var filter generated.SearchForApplicationsParams + var filter idb.ApplicationQuery var aidx uint64 = uint64(index) - filter.ApplicationId = &aidx - appRows, _ := db.Applications(context.Background(), &filter) + filter.ApplicationID = aidx + appRows, _ := db.Applications(context.Background(), filter) num := 0 for row := range appRows { require.NoError(t, row.Error) @@ -873,7 +873,7 @@ func TestAppExtraPages(t *testing.T) { } require.Equal(t, 1, num) - rows, _ := db.GetAccounts(context.Background(), idb.AccountQueryOptions{EqualToAddress: test.AccountA[:]}) + rows, _ := db.GetAccounts(context.Background(), idb.AccountQueryOptions{EqualToAddress: test.AccountA[:], IncludeAppParams: true}) num = 0 var createdApps *[]generated.Application for row := range rows { @@ -881,6 +881,7 @@ func TestAppExtraPages(t *testing.T) { num++ require.NotNil(t, row.Account.AppsTotalExtraPages, "we should have this field") require.Equal(t, uint64(1), *row.Account.AppsTotalExtraPages) + require.Equal(t, uint64(1), row.Account.TotalCreatedApps) createdApps = row.Account.CreatedApps } require.Equal(t, 1, num) @@ -978,6 +979,7 @@ func TestLargeAssetAmount(t *testing.T) { require.NoError(t, row.Error) require.NotNil(t, row.Account.Assets) require.Equal(t, 1, len(*row.Account.Assets)) + require.Equal(t, uint64(1), row.Account.TotalAssetsOptedIn) assert.Equal(t, uint64(math.MaxUint64), (*row.Account.Assets)[0].Amount) } } @@ -1143,6 +1145,7 @@ func TestNonDisplayableUTF8(t *testing.T) { require.NoError(t, acct.Error) require.NotNil(t, acct.Account.CreatedAssets) require.Len(t, *acct.Account.CreatedAssets, 1) + require.Equal(t, uint64(1), acct.Account.TotalCreatedAssets) asset := (*acct.Account.CreatedAssets)[0] if testcase.ExpectedAssetName == "" { @@ -1502,12 +1505,11 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { err = db.AddBlock(&block) require.NoError(t, err) - yes := true - opts := generated.SearchForApplicationsParams{ - ApplicationId: &appid, - IncludeAll: &yes, + opts := idb.ApplicationQuery{ + ApplicationID: appid, + IncludeDeleted: true, } - rowsCh, _ := db.Applications(context.Background(), &opts) + rowsCh, _ := db.Applications(context.Background(), opts) row, ok := <-rowsCh require.True(t, ok) @@ -1538,8 +1540,9 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { require.NoError(t, err) opts := idb.AccountQueryOptions{ - EqualToAddress: test.AccountB[:], - IncludeDeleted: true, + EqualToAddress: test.AccountB[:], + IncludeDeleted: true, + IncludeAppLocalState: true, } rowsCh, _ := db.GetAccounts(context.Background(), opts) @@ -1557,6 +1560,24 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { assert.Equal(t, uint64(1), *localState.OptedInAtRound) require.NotNil(t, localState.ClosedOutAtRound) assert.Equal(t, uint64(1), *localState.ClosedOutAtRound) + require.Equal(t, uint64(0), row.Account.TotalAppsOptedIn) + + q := idb.ApplicationQuery{ + ApplicationID: appid, + IncludeDeleted: true, + } + lsRows, _ := db.AppLocalState(context.Background(), q) + lsRow, ok := <-lsRows + require.True(t, ok) + require.NoError(t, lsRow.Error) + ls := lsRow.AppLocalState + require.Equal(t, appid, ls.Id) + require.NotNil(t, ls.Deleted) + assert.True(t, *ls.Deleted) + require.NotNil(t, ls.OptedInAtRound) + assert.Equal(t, uint64(1), *ls.OptedInAtRound) + require.NotNil(t, ls.ClosedOutAtRound) + assert.Equal(t, uint64(1), *ls.ClosedOutAtRound) } // TestSearchForInnerTransactionReturnsRootTransaction checks that the parent diff --git a/idb/postgres/postgres_migrations.go b/idb/postgres/postgres_migrations.go index 2618f1698..53b162e3d 100644 --- a/idb/postgres/postgres_migrations.go +++ b/idb/postgres/postgres_migrations.go @@ -14,6 +14,7 @@ import ( "github.com/algorand/indexer/idb" "github.com/algorand/indexer/idb/migration" "github.com/algorand/indexer/idb/postgres/internal/encoding" + cad "github.com/algorand/indexer/idb/postgres/internal/migrations/convert_account_data" "github.com/algorand/indexer/idb/postgres/internal/schema" "github.com/algorand/indexer/idb/postgres/internal/types" ) @@ -48,6 +49,7 @@ func init() { {upgradeNotSupported, true, "change import state format"}, {upgradeNotSupported, true, "notify the user that upgrade is not supported"}, {dropTxnBytesColumn, true, "drop txnbytes column"}, + {convertAccountData, true, "convert account.account_data column"}, } } @@ -86,20 +88,6 @@ func needsMigration(state types.MigrationState) bool { return state.NextMigration < len(migrations) } -// upsertMigrationState updates the migration state, and optionally increments -// the next counter with an existing transaction. -// If `tx` is nil, use a normal query. -//lint:ignore U1000 this function might be used in a future migration -func upsertMigrationState(db *IndexerDb, tx pgx.Tx, state *types.MigrationState) error { - migrationStateJSON := encoding.EncodeMigrationState(state) - err := db.setMetastate(tx, schema.MigrationMetastateKey, string(migrationStateJSON)) - if err != nil { - return fmt.Errorf("upsertMigrationState() err: %w", err) - } - - return nil -} - // Returns an error object and a channel that gets closed when blocking migrations // finish running successfully. func (db *IndexerDb) runAvailableMigrations() (chan struct{}, error) { @@ -171,6 +159,17 @@ func (db *IndexerDb) getMigrationState(ctx context.Context, tx pgx.Tx) (types.Mi return state, nil } +// If `tx` is nil, use a normal query. +func (db *IndexerDb) setMigrationState(tx pgx.Tx, state *types.MigrationState) error { + err := db.setMetastate( + tx, schema.MigrationMetastateKey, string(encoding.EncodeMigrationState(state))) + if err != nil { + return fmt.Errorf("setMigrationState() err: %w", err) + } + + return nil +} + // sqlMigration executes a sql statements as the entire migration. //lint:ignore U1000 this function might be used in a future migration func sqlMigration(db *IndexerDb, state *types.MigrationState, sqlLines []string) error { @@ -224,3 +223,29 @@ func dropTxnBytesColumn(db *IndexerDb, migrationState *types.MigrationState) err return sqlMigration( db, migrationState, []string{"ALTER TABLE txn DROP COLUMN txnbytes"}) } + +func convertAccountData(db *IndexerDb, migrationState *types.MigrationState) error { + newMigrationState := *migrationState + newMigrationState.NextMigration++ + + f := func(tx pgx.Tx) error { + err := cad.RunMigration(tx, 10000) + if err != nil { + return fmt.Errorf("convertAccountData() err: %w", err) + } + + err = db.setMigrationState(tx, &newMigrationState) + if err != nil { + return fmt.Errorf("convertAccountData() err: %w", err) + } + + return nil + } + err := db.txWithRetry(serializable, f) + if err != nil { + return fmt.Errorf("convertAccountData() err: %w", err) + } + + *migrationState = newMigrationState + return nil +} diff --git a/idb/postgres/postgres_migrations_test.go b/idb/postgres/postgres_migrations_test.go new file mode 100644 index 000000000..3a0a98e79 --- /dev/null +++ b/idb/postgres/postgres_migrations_test.go @@ -0,0 +1,34 @@ +package postgres + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pgtest "github.com/algorand/indexer/idb/postgres/internal/testing" + "github.com/algorand/indexer/idb/postgres/internal/types" +) + +func TestConvertAccountDataIncrementsMigrationNumber(t *testing.T) { + pdb, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + db := IndexerDb{db: pdb} + defer db.Close() + + migrationState := types.MigrationState{ + NextMigration: 5, + } + err := db.setMigrationState(nil, &migrationState) + require.NoError(t, err) + + err = convertAccountData(&db, &migrationState) + require.NoError(t, err) + + migrationState, err = db.getMigrationState(context.Background(), nil) + require.NoError(t, err) + + assert.Equal(t, types.MigrationState{NextMigration: 6}, migrationState) +} diff --git a/idb/postgres/postgres_rand_test.go b/idb/postgres/postgres_rand_test.go index 078a5b631..e4fd44be8 100644 --- a/idb/postgres/postgres_rand_test.go +++ b/idb/postgres/postgres_rand_test.go @@ -8,26 +8,144 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" ledgerforevaluator "github.com/algorand/indexer/idb/postgres/internal/ledger_for_evaluator" "github.com/algorand/indexer/idb/postgres/internal/writer" "github.com/algorand/indexer/util/test" "github.com/jackc/pgx/v4" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func generateAddress(t *testing.T) basics.Address { + var res basics.Address + + _, err := rand.Read(res[:]) + require.NoError(t, err) + + return res +} + +func generateAccountData() ledgercore.AccountData { + // Return empty account data with probability 50%. + if rand.Uint32()%2 == 0 { + return ledgercore.AccountData{} + } + + const numCreatables = 20 + + res := ledgercore.AccountData{ + AccountBaseData: ledgercore.AccountBaseData{ + MicroAlgos: basics.MicroAlgos{Raw: uint64(rand.Int63())}, + }, + } + + return res +} + +// Write random account data for many random accounts, then read it and compare. +// Tests in particular that batch writing and reading is done in the same order +// and that there are no problems around passing account address pointers to the postgres +// driver which could be the same pointer if we are not careful. +func TestWriteReadAccountData(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + addresses := make(map[basics.Address]struct{}) + var delta ledgercore.StateDelta + for i := 0; i < 1000; i++ { + address := generateAddress(t) + + addresses[address] = struct{}{} + delta.Accts.Upsert(address, generateAccountData()) + } + + f := func(tx pgx.Tx) error { + w, err := writer.MakeWriter(tx) + require.NoError(t, err) + + err = w.AddBlock(&bookkeeping.Block{}, transactions.Payset{}, delta) + require.NoError(t, err) + + w.Close() + return nil + } + err := db.txWithRetry(serializable, f) + require.NoError(t, err) + + tx, err := db.db.BeginTx(context.Background(), serializable) + require.NoError(t, err) + defer tx.Rollback(context.Background()) + + l, err := ledgerforevaluator.MakeLedgerForEvaluator(tx, basics.Round(0)) + require.NoError(t, err) + defer l.Close() + + ret, err := l.LookupWithoutRewards(addresses) + require.NoError(t, err) + + for address := range addresses { + expected, ok := delta.Accts.GetData(address) + require.True(t, ok) + + ret, ok := ret[address] + require.True(t, ok) + + if ret == nil { + require.True(t, expected.IsZero()) + } else { + require.Equal(t, &expected, ret) + } + } +} + func generateAssetParams() basics.AssetParams { return basics.AssetParams{ Total: rand.Uint64(), } } +func generateAssetParamsDelta() ledgercore.AssetParamsDelta { + var res ledgercore.AssetParamsDelta + + r := rand.Uint32() % 3 + switch r { + case 0: + res.Deleted = true + case 1: + res.Params = new(basics.AssetParams) + *res.Params = generateAssetParams() + case 2: + // do nothing + } + + return res +} + func generateAssetHolding() basics.AssetHolding { return basics.AssetHolding{ Amount: rand.Uint64(), } } +func generateAssetHoldingDelta() ledgercore.AssetHoldingDelta { + var res ledgercore.AssetHoldingDelta + + r := rand.Uint32() % 3 + switch r { + case 0: + res.Deleted = true + case 1: + res.Holding = new(basics.AssetHolding) + *res.Holding = generateAssetHolding() + case 2: + // do nothing + } + + return res +} + func generateAppParams(t *testing.T) basics.AppParams { p := make([]byte, 100) _, err := rand.Read(p) @@ -38,6 +156,23 @@ func generateAppParams(t *testing.T) basics.AppParams { } } +func generateAppParamsDelta(t *testing.T) ledgercore.AppParamsDelta { + var res ledgercore.AppParamsDelta + + r := rand.Uint32() % 3 + switch r { + case 0: + res.Deleted = true + case 1: + res.Params = new(basics.AppParams) + *res.Params = generateAppParams(t) + case 2: + // do nothing + } + + return res +} + func generateAppLocalState(t *testing.T) basics.AppLocalState { k := make([]byte, 100) _, err := rand.Read(k) @@ -56,61 +191,61 @@ func generateAppLocalState(t *testing.T) basics.AppLocalState { } } -func generateAccountData(t *testing.T) basics.AccountData { - // Return empty account data with probability 50%. - if rand.Uint32()%2 == 0 { - return basics.AccountData{} - } - - const numCreatables = 20 - - res := basics.AccountData{ - MicroAlgos: basics.MicroAlgos{Raw: uint64(rand.Int63())}, - AssetParams: make(map[basics.AssetIndex]basics.AssetParams), - Assets: make(map[basics.AssetIndex]basics.AssetHolding), - AppLocalStates: make(map[basics.AppIndex]basics.AppLocalState), - AppParams: make(map[basics.AppIndex]basics.AppParams), - } +func generateAppLocalStateDelta(t *testing.T) ledgercore.AppLocalStateDelta { + var res ledgercore.AppLocalStateDelta - for i := 0; i < numCreatables; i++ { - { - index := basics.AssetIndex(rand.Int63()) - res.AssetParams[index] = generateAssetParams() - } - { - index := basics.AssetIndex(rand.Int63()) - res.Assets[index] = generateAssetHolding() - } - { - index := basics.AppIndex(rand.Int63()) - res.AppLocalStates[index] = generateAppLocalState(t) - } - { - index := basics.AppIndex(rand.Int63()) - res.AppParams[index] = generateAppParams(t) - } + r := rand.Uint32() % 3 + switch r { + case 0: + res.Deleted = true + case 1: + res.LocalState = new(basics.AppLocalState) + *res.LocalState = generateAppLocalState(t) + case 2: + // do nothing } return res } -// Write random account data for many random accounts, then read it and compare. +// Write random assets and apps, then read it and compare. // Tests in particular that batch writing and reading is done in the same order // and that there are no problems around passing account address pointers to the postgres // driver which could be the same pointer if we are not careful. -func TestWriteReadAccountData(t *testing.T) { +func TestWriteReadResources(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - addresses := make(map[basics.Address]struct{}) + resources := make(map[basics.Address]map[ledger.Creatable]struct{}) var delta ledgercore.StateDelta for i := 0; i < 1000; i++ { - var address basics.Address - _, err := rand.Read(address[:]) - require.NoError(t, err) + address := generateAddress(t) + assetIndex := basics.AssetIndex(rand.Int63()) + appIndex := basics.AppIndex(rand.Int63()) - addresses[address] = struct{}{} - delta.Accts.Upsert(address, generateAccountData(t)) + { + c := make(map[ledger.Creatable]struct{}) + resources[address] = c + + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(assetIndex), + Type: basics.AssetCreatable, + } + c[creatable] = struct{}{} + + creatable = ledger.Creatable{ + Index: basics.CreatableIndex(appIndex), + Type: basics.AppCreatable, + } + c[creatable] = struct{}{} + } + + delta.Accts.UpsertAssetResource( + address, assetIndex, generateAssetParamsDelta(), + generateAssetHoldingDelta()) + delta.Accts.UpsertAppResource( + address, appIndex, generateAppParamsDelta(t), + generateAppLocalStateDelta(t)) } f := func(tx pgx.Tx) error { @@ -134,20 +269,35 @@ func TestWriteReadAccountData(t *testing.T) { require.NoError(t, err) defer l.Close() - ret, err := l.LookupWithoutRewards(addresses) + ret, err := l.LookupResources(resources) require.NoError(t, err) - for address := range addresses { - expected, ok := delta.Accts.Get(address) - require.True(t, ok) - + for address, creatables := range resources { ret, ok := ret[address] require.True(t, ok) - if ret == nil { - require.True(t, expected.IsZero()) - } else { - require.Equal(t, &expected, ret) + for creatable := range creatables { + ret, ok := ret[creatable] + require.True(t, ok) + + switch creatable.Type { + case basics.AssetCreatable: + assetParamsDelta, _ := + delta.Accts.GetAssetParams(address, basics.AssetIndex(creatable.Index)) + assert.Equal(t, assetParamsDelta.Params, ret.AssetParams) + + assetHoldingDelta, _ := + delta.Accts.GetAssetHolding(address, basics.AssetIndex(creatable.Index)) + assert.Equal(t, assetHoldingDelta.Holding, ret.AssetHolding) + case basics.AppCreatable: + appParamsDelta, _ := + delta.Accts.GetAppParams(address, basics.AppIndex(creatable.Index)) + assert.Equal(t, appParamsDelta.Params, ret.AppParams) + + appLocalStateDelta, _ := + delta.Accts.GetAppLocalState(address, basics.AppIndex(creatable.Index)) + assert.Equal(t, appLocalStateDelta.LocalState, ret.AppLocalState) + } } } } diff --git a/misc/Dockerfile b/misc/Dockerfile index bdff1f0c7..fff8a9985 100644 --- a/misc/Dockerfile +++ b/misc/Dockerfile @@ -6,7 +6,7 @@ RUN echo "Go image: $GO_IMAGE" # Misc dependencies ENV HOME /opt ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y apt-utils curl git git-core bsdmainutils python3 python3-pip make bash libtool libboost-all-dev libffi-dev +RUN apt-get update && apt-get install -y apt-utils curl git git-core bsdmainutils python3 python3-pip make bash libtool libboost-math-dev libffi-dev # Install algod nightly binaries to the path RUN mkdir -p /opt/algorand/{bin,data} diff --git a/misc/parity/reports/algod2indexer_dropped.yml b/misc/parity/reports/algod2indexer_dropped.yml index aa395e102..3186ec820 100644 --- a/misc/parity/reports/algod2indexer_dropped.yml +++ b/misc/parity/reports/algod2indexer_dropped.yml @@ -26,6 +26,9 @@ DryrunTxnResult: ErrorResponse: - INDEXER: null - ALGOD: '{"description":"An error respo...' +AccountsErrorResponse: +- INDEXER: null +- ALGOD: '{"description":"An error respo...' ParticipationKey: - INDEXER: null - ALGOD: '{"description":"Represents a p...' diff --git a/misc/parity/reports/algod2indexer_full.yml b/misc/parity/reports/algod2indexer_full.yml index 8f7f3e03a..65b27c8b4 100644 --- a/misc/parity/reports/algod2indexer_full.yml +++ b/misc/parity/reports/algod2indexer_full.yml @@ -100,6 +100,9 @@ DryrunTxnResult: ErrorResponse: - INDEXER: null - ALGOD: '{"description":"An error respo...' +AccountsErrorResponse: +- INDEXER: null +- ALGOD: '{"description":"An error respo...' HealthCheck: - INDEXER: '{"description":"A health check...' - ALGOD: null diff --git a/test/common.sh b/test/common.sh index d72f09423..0e1a7c31a 100755 --- a/test/common.sh +++ b/test/common.sh @@ -492,7 +492,6 @@ function create_delete_tests() { '{ "amount": 0, "asset-id": 135, - "creator": "", "deleted": true, "is-frozen": false, "opted-in-at-round": 25, diff --git a/third_party/go-algorand b/third_party/go-algorand index b56953f44..09b6c38e1 160000 --- a/third_party/go-algorand +++ b/third_party/go-algorand @@ -1 +1 @@ -Subproject commit b56953f449670398ee3ff850ed847b6e17d4e6af +Subproject commit 09b6c38e12622e2bac3ed7f2013873e716f80003