Feature: Add support for unlimited assets.

.circleci/config.yml

@@ -83,7 +83,7 @@ commands:
           name: Install python and other python dependencies
           command: |
             sudo apt update
-            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-all-dev libffi-dev
+            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
             pip3 install -r misc/requirements.txt
 
       - run:

Makefile

@@ -11,6 +11,8 @@ GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u
 GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n')
 GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version)
 
+COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' |  paste -s -d, - )
+
 # Used for e2e test
 export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )
 
@@ -45,7 +47,7 @@ fakepackage: go-algorand
 	misc/release.py --host-only --outdir $(PKG_DIR) --fake-release
 
 test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
-	go test ./... -coverprofile=coverage.txt -covermode=atomic
+	go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic
 
 lint: go-algorand
 	golint -set_exit_status ./...

README.md

@@ -178,20 +178,33 @@ If the maximum number of connections/active queries is reached, subsequent conne
 
 Settings can be provided from the command line, a configuration file, or an environment variable
 
-| Command Line Flag (long) | (short) | Config File                | Environment Variable               |
-| ------------------------ | ------- | -------------------------- | ---------------------------------- |
-| postgres                 | P       | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
-| pidfile                  |         | pidfile                    | INDEXER_PIDFILE                    |
-| algod                    | d       | algod-data-dir             | INDEXER_ALGOD_DATA_DIR             |
-| algod-net                |         | algod-address              | INDEXER_ALGOD_ADDRESS              |
-| algod-token              |         | algod-token                | INDEXER_ALGOD_TOKEN                |
-| genesis                  | g       | genesis                    | INDEXER_GENESIS                    |
-| server                   | S       | server-address             | INDEXER_SERVER_ADDRESS             |
-| no-algod                 |         | no-algod                   | INDEXER_NO_ALGOD                   |
-| token                    | t       | api-token                  | INDEXER_API_TOKEN                  |
-| dev-mode                 |         | dev-mode                   | INDEXER_DEV_MODE                   |
-| metrics-mode             |         | metrics-mode               | INDEXER_METRICS_MODE               |
-| max-conn                 |         | max-conn                   | INDEXER_MAX_CONN                   |
+| Command Line Flag (long)      | (short) | Config File                   | Environment Variable                  |
+|-------------------------------|---------|-------------------------------|---------------------------------------|
+| postgres                      | P       | postgres-connection-string    | INDEXER_POSTGRES_CONNECTION_STRING    |
+| pidfile                       |         | pidfile                       | INDEXER_PIDFILE                       |
+| algod                         | d       | algod-data-dir                | INDEXER_ALGOD_DATA_DIR                |
+| algod-net                     |         | algod-address                 | INDEXER_ALGOD_ADDRESS                 |
+| algod-token                   |         | algod-token                   | INDEXER_ALGOD_TOKEN                   |
+| genesis                       | g       | genesis                       | INDEXER_GENESIS                       |
+| server                        | S       | server-address                | INDEXER_SERVER_ADDRESS                |
+| no-algod                      |         | no-algod                      | INDEXER_NO_ALGOD                      |
+| token                         | t       | api-token                     | INDEXER_API_TOKEN                     |
+| dev-mode                      |         | dev-mode                      | INDEXER_DEV_MODE                      |
+| metrics-mode                  |         | metrics-mode                  | INDEXER_METRICS_MODE                  |
+| max-conn                      |         | max-conn                      | INDEXER_MAX_CONN                      |
+| write-timeout                 |         | write-timeout                 | INDEXER_WRITE_TIMEOUT                 |
+| read-timeout                  |         | read-timeout                  | INDEXER_READ_TIMEOUT                  |
+| max-api-resources-per-account |         | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT |
+| max-transactions-limit        |         | max-transactions-limit        | INDEXER_MAX_TRANSACTIONS_LIMIT        |
+| default-transactions-limit    |         | default-transactions-limit    | INDEXER_DEFAULT_TRANSACTIONS_LIMIT    |
+| max-accounts-limit            |         | max-accounts-limit            | INDEXER_MAX_ACCOUNTS_LIMIT            |
+| default-accounts-limit        |         | default-accounts-limit        | INDEXER_DEFAULT_ACCOUNTS_LIMIT        |
+| max-assets-limit              |         | max-assets-limit              | INDEXER_MAX_ASSETS_LIMIT              |
+| default-assets-limit          |         | default-assets-limit          | INDEXER_DEFAULT_ASSETS_LIMIT          |
+| max-balances-limit            |         | max-balances-limit            | INDEXER_MAX_BALANCES_LIMIT            |
+| default-balances-limit        |         | default-balances-limit        | INDEXER_DEFAULT_BALANCES_LIMIT        |
+| max-applications-limit        |         | max-applications-limit        | INDEXER_MAX_APPLICATIONS_LIMIT        |
+| default-applications-limit    |         | default-applications-limit    | INDEXER_DEFAULT_APPLICATIONS_LIMIT    |
 
 ## Command line
 

accounting/eval_preload.go

@@ -0,0 +1,202 @@
+package accounting
+
+import (
+	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/transactions"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/protocol"
+)
+
+// Add requests for asset and app creators to `assetsReq` and `appsReq` for the given
+// transaction.
+func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[basics.AssetIndex]struct{}, appsReq map[basics.AppIndex]struct{}) {
+	txn := &stxnad.Txn
+
+	switch txn.Type {
+	case protocol.AssetConfigTx:
+		fields := &txn.AssetConfigTxnFields
+		if fields.ConfigAsset != 0 {
+			assetsReq[fields.ConfigAsset] = struct{}{}
+		}
+	case protocol.AssetTransferTx:
+		fields := &txn.AssetTransferTxnFields
+		if fields.XferAsset != 0 {
+			assetsReq[fields.XferAsset] = struct{}{}
+		}
+	case protocol.AssetFreezeTx:
+		fields := &txn.AssetFreezeTxnFields
+		if fields.FreezeAsset != 0 {
+			assetsReq[fields.FreezeAsset] = struct{}{}
+		}
+	case protocol.ApplicationCallTx:
+		fields := &txn.ApplicationCallTxnFields
+		if fields.ApplicationID != 0 {
+			appsReq[fields.ApplicationID] = struct{}{}
+		}
+		for _, index := range fields.ForeignApps {
+			appsReq[index] = struct{}{}
+		}
+		for _, index := range fields.ForeignAssets {
+			assetsReq[index] = struct{}{}
+		}
+	}
+
+	for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
+		addToCreatorsRequest(&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetsReq, appsReq)
+	}
+}
+
+// MakePreloadCreatorsRequest makes a request for preloading creators in the batch mode.
+func MakePreloadCreatorsRequest(payset transactions.Payset) (map[basics.AssetIndex]struct{}, map[basics.AppIndex]struct{}) {
+	assetsReq := make(map[basics.AssetIndex]struct{}, len(payset))
+	appsReq := make(map[basics.AppIndex]struct{}, len(payset))
+
+	for i := range payset {
+		addToCreatorsRequest(&payset[i].SignedTxnWithAD, assetsReq, appsReq)
+	}
+
+	return assetsReq, appsReq
+}
+
+// Add requests for account data and account resources to `addressesReq` and
+// `resourcesReq` respectively for the given transaction.
+func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress, addressesReq map[basics.Address]struct{}, resourcesReq map[basics.Address]map[ledger.Creatable]struct{}) {
+	setResourcesReq := func(addr basics.Address, creatable ledger.Creatable) {
+		c, ok := resourcesReq[addr]
+		if !ok {
+			c = make(map[ledger.Creatable]struct{})
+			resourcesReq[addr] = c
+		}
+		c[creatable] = struct{}{}
+	}
+
+	txn := &stxnad.Txn
+
+	addressesReq[txn.Sender] = struct{}{}
+
+	switch txn.Type {
+	case protocol.PaymentTx:
+		fields := &txn.PaymentTxnFields
+		addressesReq[fields.Receiver] = struct{}{}
+		// Close address is optional.
+		if !fields.CloseRemainderTo.IsZero() {
+			addressesReq[fields.CloseRemainderTo] = struct{}{}
+		}
+	case protocol.AssetConfigTx:
+		fields := &txn.AssetConfigTxnFields
+		if fields.ConfigAsset == 0 {
+			if stxnad.ApplyData.ConfigAsset != 0 {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(stxnad.ApplyData.ConfigAsset),
+					Type:  basics.AssetCreatable,
+				}
+				setResourcesReq(txn.Sender, creatable)
+			}
+		} else {
+			if creator := assetCreators[fields.ConfigAsset]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(fields.ConfigAsset),
+					Type:  basics.AssetCreatable,
+				}
+				addressesReq[creator.Address] = struct{}{}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+	case protocol.AssetTransferTx:
+		fields := &txn.AssetTransferTxnFields
+		creatable := ledger.Creatable{
+			Index: basics.CreatableIndex(fields.XferAsset),
+			Type:  basics.AssetCreatable,
+		}
+		if creator := assetCreators[fields.XferAsset]; creator.Exists {
+			setResourcesReq(creator.Address, creatable)
+		}
+		source := txn.Sender
+		// If asset sender is non-zero, it is a clawback transaction. Otherwise,
+		// the transaction sender address is used.
+		if !fields.AssetSender.IsZero() {
+			source = fields.AssetSender
+		}
+		addressesReq[source] = struct{}{}
+		setResourcesReq(source, creatable)
+		addressesReq[fields.AssetReceiver] = struct{}{}
+		setResourcesReq(fields.AssetReceiver, creatable)
+		// Asset close address is optional.
+		if !fields.AssetCloseTo.IsZero() {
+			addressesReq[fields.AssetCloseTo] = struct{}{}
+			setResourcesReq(fields.AssetCloseTo, creatable)
+		}
+	case protocol.AssetFreezeTx:
+		fields := &txn.AssetFreezeTxnFields
+		creatable := ledger.Creatable{
+			Index: basics.CreatableIndex(fields.FreezeAsset),
+			Type:  basics.AssetCreatable,
+		}
+		if creator := assetCreators[fields.FreezeAsset]; creator.Exists {
+			setResourcesReq(creator.Address, creatable)
+		}
+		setResourcesReq(fields.FreezeAccount, creatable)
+	case protocol.ApplicationCallTx:
+		fields := &txn.ApplicationCallTxnFields
+		if fields.ApplicationID == 0 {
+			if stxnad.ApplyData.ApplicationID != 0 {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(stxnad.ApplyData.ApplicationID),
+					Type:  basics.AppCreatable,
+				}
+				setResourcesReq(txn.Sender, creatable)
+			}
+		} else {
+			creatable := ledger.Creatable{
+				Index: basics.CreatableIndex(fields.ApplicationID),
+				Type:  basics.AppCreatable,
+			}
+			if creator := appCreators[fields.ApplicationID]; creator.Exists {
+				addressesReq[creator.Address] = struct{}{}
+				setResourcesReq(creator.Address, creatable)
+			}
+			setResourcesReq(txn.Sender, creatable)
+		}
+		for _, address := range fields.Accounts {
+			addressesReq[address] = struct{}{}
+		}
+		for _, index := range fields.ForeignApps {
+			if creator := appCreators[index]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(index),
+					Type:  basics.AppCreatable,
+				}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+		for _, index := range fields.ForeignAssets {
+			if creator := assetCreators[index]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(index),
+					Type:  basics.AssetCreatable,
+				}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+	}
+
+	for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
+		addToAccountsResourcesRequest(
+			&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetCreators, appCreators,
+			addressesReq, resourcesReq)
+	}
+}
+
+// MakePreloadAccountsResourcesRequest makes a request for preloading account data and
+// account resources in the batch mode.
+func MakePreloadAccountsResourcesRequest(payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}) {
+	addressesReq := make(map[basics.Address]struct{}, len(payset))
+	resourcesReq := make(map[basics.Address]map[ledger.Creatable]struct{}, len(payset))
+
+	for i := range payset {
+		addToAccountsResourcesRequest(
+			&payset[i].SignedTxnWithAD, assetCreators, appCreators, addressesReq, resourcesReq)
+	}
+
+	return addressesReq, resourcesReq
+}

api/converter_utils.go

@@ -525,7 +525,7 @@ func signedTxnWithAdToTransaction(stxn *transactions.SignedTxnWithAD, extra rowD
 	return txn, nil
 }
 
-func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
+func (si *ServerImplementation) assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
 	creator, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
 	if len(errorArr) != 0 {
 		return idb.AssetsQuery{}, errors.New(errUnableToParseAddress)
@@ -548,21 +548,45 @@ func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.Assets
 		Unit:               strOrDefault(params.Unit),
 		Query:              "",
 		IncludeDeleted:     boolOrDefault(params.IncludeAll),
-		Limit:              min(uintOrDefaultValue(params.Limit, defaultAssetsLimit), maxAssetsLimit),
+		Limit:              min(uintOrDefaultValue(params.Limit, si.opts.DefaultAssetsLimit), si.opts.MaxAssetsLimit),
 	}
 
 	return query, nil
 }
 
-func transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
+func (si *ServerImplementation) appParamsToApplicationQuery(params generated.SearchForApplicationsParams) (idb.ApplicationQuery, error) {
+	addr, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
+	if len(errorArr) != 0 {
+		return idb.ApplicationQuery{}, errors.New(errUnableToParseAddress)
+	}
+
+	var appGreaterThan uint64 = 0
+	if params.Next != nil {
+		agt, err := strconv.ParseUint(*params.Next, 10, 64)
+		if err != nil {
+			return idb.ApplicationQuery{}, fmt.Errorf("%s: %v", errUnableToParseNext, err)
+		}
+		appGreaterThan = agt
+	}
+
+	return idb.ApplicationQuery{
+		ApplicationID:            uintOrDefault(params.ApplicationId),
+		ApplicationIDGreaterThan: appGreaterThan,
+		Address:                  addr,
+		IncludeDeleted:           boolOrDefault(params.IncludeAll),
+		Limit:                    min(uintOrDefaultValue(params.Limit, si.opts.DefaultApplicationsLimit), si.opts.MaxApplicationsLimit),
+	}, nil
+}
+
+func (si *ServerImplementation) transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
 	var errorArr = make([]string, 0)
 
 	// Integer
 	filter.MaxRound = uintOrDefault(params.MaxRound)
 	filter.MinRound = uintOrDefault(params.MinRound)
 	filter.AssetID = uintOrDefault(params.AssetId)
 	filter.ApplicationID = uintOrDefault(params.ApplicationId)
-	filter.Limit = min(uintOrDefaultValue(params.Limit, defaultTransactionsLimit), maxTransactionsLimit)
+	filter.Limit = min(uintOrDefaultValue(params.Limit, si.opts.DefaultTransactionsLimit), si.opts.MaxTransactionsLimit)
 
 	// filter Algos or Asset but not both.
 	if filter.AssetID != 0 {
@@ -610,3 +634,20 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions
 
 	return
 }
+
+func (si *ServerImplementation) maxAccountsErrorToAccountsErrorResponse(maxErr idb.MaxAPIResourcesPerAccountError) generated.ErrorResponse {
+	addr := maxErr.Address.String()
+	max := uint64(si.opts.MaxAPIResourcesPerAccount)
+	extraData := map[string]interface{}{
+		"max-results":           max,
+		"address":               addr,
+		"total-assets-opted-in": maxErr.TotalAssets,
+		"total-created-assets":  maxErr.TotalAssetParams,
+		"total-apps-opted-in":   maxErr.TotalAppLocalStates,
+		"total-created-apps":    maxErr.TotalAppParams,
+	}
+	return generated.ErrorResponse{
+		Message: "Result limit exceeded",
+		Data:    &extraData,
+	}
+}

api/disabled_parameters.go

@@ -320,7 +320,7 @@ func GetDefaultDisabledMapConfigForPostgres() *DisabledMapConfig {
 	get("/v2/accounts", []string{"currency-greater-than", "currency-less-than"})
 	get("/v2/accounts/{account-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "rekey-to"})
 	get("/v2/assets", []string{"name", "unit"})
-	get("/v2/assets/{asset-id}/balances", []string{"round", "currency-greater-than", "currency-less-than"})
+	get("/v2/assets/{asset-id}/balances", []string{"currency-greater-than", "currency-less-than"})
 	get("/v2/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to", "application-id"})
 	get("/v2/assets/{asset-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "currency-greater-than", "currency-less-than", "address-role", "exclude-close-to", "rekey-to"})