Skip to content

Commit

Permalink
Add support for unlimited assets. (#900)
Browse files Browse the repository at this point in the history
Co-authored-by: John Lee <[email protected]>
Co-authored-by: Brice Rising <[email protected]>
Co-authored-by: Tolik Zinovyev <[email protected]>
Co-authored-by: Barbara Poon <[email protected]>
Co-authored-by: Will Winder <[email protected]>
  • Loading branch information
6 people authored Mar 10, 2022
1 parent b303af1 commit 31eec10
Show file tree
Hide file tree
Showing 46 changed files with 5,648 additions and 1,683 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ commands:
name: Install python and other python dependencies
command: |
sudo apt update
sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-all-dev libffi-dev
sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
pip3 install -r misc/requirements.txt
- run:
Expand Down
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u
GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n')
GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version)

COVERPKG := $(shell go list ./... | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' | paste -s -d, - )

# Used for e2e test
export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )

Expand Down Expand Up @@ -45,7 +47,7 @@ fakepackage: go-algorand
misc/release.py --host-only --outdir $(PKG_DIR) --fake-release

test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
go test ./... -coverprofile=coverage.txt -covermode=atomic
go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic

lint: go-algorand
golint -set_exit_status ./...
Expand Down
41 changes: 27 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,20 +178,33 @@ If the maximum number of connections/active queries is reached, subsequent conne

Settings can be provided from the command line, a configuration file, or an environment variable

| Command Line Flag (long) | (short) | Config File | Environment Variable |
| ------------------------ | ------- | -------------------------- | ---------------------------------- |
| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
| pidfile | | pidfile | INDEXER_PIDFILE |
| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR |
| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS |
| algod-token | | algod-token | INDEXER_ALGOD_TOKEN |
| genesis | g | genesis | INDEXER_GENESIS |
| server | S | server-address | INDEXER_SERVER_ADDRESS |
| no-algod | | no-algod | INDEXER_NO_ALGOD |
| token | t | api-token | INDEXER_API_TOKEN |
| dev-mode | | dev-mode | INDEXER_DEV_MODE |
| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE |
| max-conn | | max-conn | INDEXER_MAX_CONN |
| Command Line Flag (long) | (short) | Config File | Environment Variable |
|-------------------------------|---------|-------------------------------|---------------------------------------|
| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
| pidfile | | pidfile | INDEXER_PIDFILE |
| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR |
| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS |
| algod-token | | algod-token | INDEXER_ALGOD_TOKEN |
| genesis | g | genesis | INDEXER_GENESIS |
| server | S | server-address | INDEXER_SERVER_ADDRESS |
| no-algod | | no-algod | INDEXER_NO_ALGOD |
| token | t | api-token | INDEXER_API_TOKEN |
| dev-mode | | dev-mode | INDEXER_DEV_MODE |
| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE |
| max-conn | | max-conn | INDEXER_MAX_CONN |
| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT |
| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT |
| max-api-resources-per-account | | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT |
| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT |
| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT |
| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT |
| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT |
| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT |
| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT |
| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT |
| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT |
| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT |
| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT |

## Command line

Expand Down
202 changes: 202 additions & 0 deletions accounting/eval_preload.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
package accounting

import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/transactions"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/protocol"
)

// Add requests for asset and app creators to `assetsReq` and `appsReq` for the given
// transaction.
func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[basics.AssetIndex]struct{}, appsReq map[basics.AppIndex]struct{}) {
txn := &stxnad.Txn

switch txn.Type {
case protocol.AssetConfigTx:
fields := &txn.AssetConfigTxnFields
if fields.ConfigAsset != 0 {
assetsReq[fields.ConfigAsset] = struct{}{}
}
case protocol.AssetTransferTx:
fields := &txn.AssetTransferTxnFields
if fields.XferAsset != 0 {
assetsReq[fields.XferAsset] = struct{}{}
}
case protocol.AssetFreezeTx:
fields := &txn.AssetFreezeTxnFields
if fields.FreezeAsset != 0 {
assetsReq[fields.FreezeAsset] = struct{}{}
}
case protocol.ApplicationCallTx:
fields := &txn.ApplicationCallTxnFields
if fields.ApplicationID != 0 {
appsReq[fields.ApplicationID] = struct{}{}
}
for _, index := range fields.ForeignApps {
appsReq[index] = struct{}{}
}
for _, index := range fields.ForeignAssets {
assetsReq[index] = struct{}{}
}
}

for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
addToCreatorsRequest(&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetsReq, appsReq)
}
}

// MakePreloadCreatorsRequest makes a request for preloading creators in the batch mode.
func MakePreloadCreatorsRequest(payset transactions.Payset) (map[basics.AssetIndex]struct{}, map[basics.AppIndex]struct{}) {
assetsReq := make(map[basics.AssetIndex]struct{}, len(payset))
appsReq := make(map[basics.AppIndex]struct{}, len(payset))

for i := range payset {
addToCreatorsRequest(&payset[i].SignedTxnWithAD, assetsReq, appsReq)
}

return assetsReq, appsReq
}

// Add requests for account data and account resources to `addressesReq` and
// `resourcesReq` respectively for the given transaction.
func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress, addressesReq map[basics.Address]struct{}, resourcesReq map[basics.Address]map[ledger.Creatable]struct{}) {
setResourcesReq := func(addr basics.Address, creatable ledger.Creatable) {
c, ok := resourcesReq[addr]
if !ok {
c = make(map[ledger.Creatable]struct{})
resourcesReq[addr] = c
}
c[creatable] = struct{}{}
}

txn := &stxnad.Txn

addressesReq[txn.Sender] = struct{}{}

switch txn.Type {
case protocol.PaymentTx:
fields := &txn.PaymentTxnFields
addressesReq[fields.Receiver] = struct{}{}
// Close address is optional.
if !fields.CloseRemainderTo.IsZero() {
addressesReq[fields.CloseRemainderTo] = struct{}{}
}
case protocol.AssetConfigTx:
fields := &txn.AssetConfigTxnFields
if fields.ConfigAsset == 0 {
if stxnad.ApplyData.ConfigAsset != 0 {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(stxnad.ApplyData.ConfigAsset),
Type: basics.AssetCreatable,
}
setResourcesReq(txn.Sender, creatable)
}
} else {
if creator := assetCreators[fields.ConfigAsset]; creator.Exists {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(fields.ConfigAsset),
Type: basics.AssetCreatable,
}
addressesReq[creator.Address] = struct{}{}
setResourcesReq(creator.Address, creatable)
}
}
case protocol.AssetTransferTx:
fields := &txn.AssetTransferTxnFields
creatable := ledger.Creatable{
Index: basics.CreatableIndex(fields.XferAsset),
Type: basics.AssetCreatable,
}
if creator := assetCreators[fields.XferAsset]; creator.Exists {
setResourcesReq(creator.Address, creatable)
}
source := txn.Sender
// If asset sender is non-zero, it is a clawback transaction. Otherwise,
// the transaction sender address is used.
if !fields.AssetSender.IsZero() {
source = fields.AssetSender
}
addressesReq[source] = struct{}{}
setResourcesReq(source, creatable)
addressesReq[fields.AssetReceiver] = struct{}{}
setResourcesReq(fields.AssetReceiver, creatable)
// Asset close address is optional.
if !fields.AssetCloseTo.IsZero() {
addressesReq[fields.AssetCloseTo] = struct{}{}
setResourcesReq(fields.AssetCloseTo, creatable)
}
case protocol.AssetFreezeTx:
fields := &txn.AssetFreezeTxnFields
creatable := ledger.Creatable{
Index: basics.CreatableIndex(fields.FreezeAsset),
Type: basics.AssetCreatable,
}
if creator := assetCreators[fields.FreezeAsset]; creator.Exists {
setResourcesReq(creator.Address, creatable)
}
setResourcesReq(fields.FreezeAccount, creatable)
case protocol.ApplicationCallTx:
fields := &txn.ApplicationCallTxnFields
if fields.ApplicationID == 0 {
if stxnad.ApplyData.ApplicationID != 0 {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(stxnad.ApplyData.ApplicationID),
Type: basics.AppCreatable,
}
setResourcesReq(txn.Sender, creatable)
}
} else {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(fields.ApplicationID),
Type: basics.AppCreatable,
}
if creator := appCreators[fields.ApplicationID]; creator.Exists {
addressesReq[creator.Address] = struct{}{}
setResourcesReq(creator.Address, creatable)
}
setResourcesReq(txn.Sender, creatable)
}
for _, address := range fields.Accounts {
addressesReq[address] = struct{}{}
}
for _, index := range fields.ForeignApps {
if creator := appCreators[index]; creator.Exists {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(index),
Type: basics.AppCreatable,
}
setResourcesReq(creator.Address, creatable)
}
}
for _, index := range fields.ForeignAssets {
if creator := assetCreators[index]; creator.Exists {
creatable := ledger.Creatable{
Index: basics.CreatableIndex(index),
Type: basics.AssetCreatable,
}
setResourcesReq(creator.Address, creatable)
}
}
}

for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
addToAccountsResourcesRequest(
&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetCreators, appCreators,
addressesReq, resourcesReq)
}
}

// MakePreloadAccountsResourcesRequest makes a request for preloading account data and
// account resources in the batch mode.
func MakePreloadAccountsResourcesRequest(payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}) {
addressesReq := make(map[basics.Address]struct{}, len(payset))
resourcesReq := make(map[basics.Address]map[ledger.Creatable]struct{}, len(payset))

for i := range payset {
addToAccountsResourcesRequest(
&payset[i].SignedTxnWithAD, assetCreators, appCreators, addressesReq, resourcesReq)
}

return addressesReq, resourcesReq
}
49 changes: 45 additions & 4 deletions api/converter_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ func signedTxnWithAdToTransaction(stxn *transactions.SignedTxnWithAD, extra rowD
return txn, nil
}

func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
func (si *ServerImplementation) assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
creator, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
if len(errorArr) != 0 {
return idb.AssetsQuery{}, errors.New(errUnableToParseAddress)
Expand All @@ -548,21 +548,45 @@ func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.Assets
Unit: strOrDefault(params.Unit),
Query: "",
IncludeDeleted: boolOrDefault(params.IncludeAll),
Limit: min(uintOrDefaultValue(params.Limit, defaultAssetsLimit), maxAssetsLimit),
Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultAssetsLimit), si.opts.MaxAssetsLimit),
}

return query, nil
}

func transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
func (si *ServerImplementation) appParamsToApplicationQuery(params generated.SearchForApplicationsParams) (idb.ApplicationQuery, error) {
addr, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
if len(errorArr) != 0 {
return idb.ApplicationQuery{}, errors.New(errUnableToParseAddress)
}

var appGreaterThan uint64 = 0
if params.Next != nil {
agt, err := strconv.ParseUint(*params.Next, 10, 64)
if err != nil {
return idb.ApplicationQuery{}, fmt.Errorf("%s: %v", errUnableToParseNext, err)
}
appGreaterThan = agt
}

return idb.ApplicationQuery{
ApplicationID: uintOrDefault(params.ApplicationId),
ApplicationIDGreaterThan: appGreaterThan,
Address: addr,
IncludeDeleted: boolOrDefault(params.IncludeAll),
Limit: min(uintOrDefaultValue(params.Limit, si.opts.DefaultApplicationsLimit), si.opts.MaxApplicationsLimit),
}, nil
}

func (si *ServerImplementation) transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
var errorArr = make([]string, 0)

// Integer
filter.MaxRound = uintOrDefault(params.MaxRound)
filter.MinRound = uintOrDefault(params.MinRound)
filter.AssetID = uintOrDefault(params.AssetId)
filter.ApplicationID = uintOrDefault(params.ApplicationId)
filter.Limit = min(uintOrDefaultValue(params.Limit, defaultTransactionsLimit), maxTransactionsLimit)
filter.Limit = min(uintOrDefaultValue(params.Limit, si.opts.DefaultTransactionsLimit), si.opts.MaxTransactionsLimit)

// filter Algos or Asset but not both.
if filter.AssetID != 0 {
Expand Down Expand Up @@ -610,3 +634,20 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions

return
}

func (si *ServerImplementation) maxAccountsErrorToAccountsErrorResponse(maxErr idb.MaxAPIResourcesPerAccountError) generated.ErrorResponse {
addr := maxErr.Address.String()
max := uint64(si.opts.MaxAPIResourcesPerAccount)
extraData := map[string]interface{}{
"max-results": max,
"address": addr,
"total-assets-opted-in": maxErr.TotalAssets,
"total-created-assets": maxErr.TotalAssetParams,
"total-apps-opted-in": maxErr.TotalAppLocalStates,
"total-created-apps": maxErr.TotalAppParams,
}
return generated.ErrorResponse{
Message: "Result limit exceeded",
Data: &extraData,
}
}
2 changes: 1 addition & 1 deletion api/disabled_parameters.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ func GetDefaultDisabledMapConfigForPostgres() *DisabledMapConfig {
get("/v2/accounts", []string{"currency-greater-than", "currency-less-than"})
get("/v2/accounts/{account-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "rekey-to"})
get("/v2/assets", []string{"name", "unit"})
get("/v2/assets/{asset-id}/balances", []string{"round", "currency-greater-than", "currency-less-than"})
get("/v2/assets/{asset-id}/balances", []string{"currency-greater-than", "currency-less-than"})
get("/v2/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to", "application-id"})
get("/v2/assets/{asset-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to"})

Expand Down
Loading

0 comments on commit 31eec10

Please sign in to comment.