From fa3014eefe4e00acd74ea20b8a3c3ee4208efb58 Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Tue, 14 Jun 2022 14:38:09 -0300 Subject: [PATCH 01/12] poc --- types/query/filtered_pagination.go | 116 +++++++++++++++++++++++++++++ x/authz/keeper/grpc_query.go | 34 +++------ 2 files changed, 125 insertions(+), 25 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 0f755c55e788..245c32cc70ac 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/cosmos/cosmos-sdk/store/types" + proto "github.com/gogo/protobuf/proto" ) // FilteredPaginate does pagination of all the results in the PrefixStore based on the @@ -114,3 +115,118 @@ func FilteredPaginate( return res, nil } + +func GenericFilteredPaginated[V proto.Message]( + prefixStore types.KVStore, + pageRequest *PageRequest, + onResult func(key []byte, value V) (V, error), +) ([]V, *PageResponse, error) { + // if the PageRequest is nil, use default PageRequest + if pageRequest == nil { + pageRequest = &PageRequest{} + } + + offset := pageRequest.Offset + key := pageRequest.Key + limit := pageRequest.Limit + countTotal := pageRequest.CountTotal + reverse := pageRequest.Reverse + results := []V{} + + if offset > 0 && key != nil { + return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") + } + + if limit == 0 { + limit = DefaultLimit + + // count total results when the limit is zero/not supplied + countTotal = true + } + + if len(key) != 0 { + iterator := getIterator(prefixStore, key, reverse) + defer iterator.Close() + + var numHits uint64 + var nextKey []byte + + for ; iterator.Valid(); iterator.Next() { + if numHits == limit { + nextKey = iterator.Key() + break + } + + if iterator.Error() != nil { + return nil, nil, iterator.Error() + } + + var protoMsg *V + err := proto.Unmarshal(iterator.Value(), *protoMsg) + if err != nil { + return nil, nil, err + } + + val, err := onResult(iterator.Key(), *protoMsg) + if err != nil { + return nil, nil, err + } + + if protoMsg != nil { + results = append(results, val) + numHits++ + } + } + + return nil, &PageResponse{ + NextKey: nextKey, + }, nil + } + + iterator := getIterator(prefixStore, nil, reverse) + defer iterator.Close() + + end := offset + limit + + var numHits uint64 + var nextKey []byte + + for ; iterator.Valid(); iterator.Next() { + if iterator.Error() != nil { + return nil, nil, iterator.Error() + } + + var protoMsg *V + err := proto.Unmarshal(iterator.Value(), *protoMsg) + if err != nil { + return nil, nil, err + } + + val, err := onResult(iterator.Key(), *protoMsg) + if err != nil { + return nil, nil, err + } + + if protoMsg != nil { + results = append(results, val) + numHits++ + } + + if numHits == end+1 { + if nextKey == nil { + nextKey = iterator.Key() + } + + if !countTotal { + break + } + } + } + + res := &PageResponse{NextKey: nextKey} + if countTotal { + res.Total = numHits + } + + return results, res, nil +} diff --git a/x/authz/keeper/grpc_query.go b/x/authz/keeper/grpc_query.go index 1156ac23521d..d7a86b26da21 100644 --- a/x/authz/keeper/grpc_query.go +++ b/x/authz/keeper/grpc_query.go @@ -6,8 +6,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - proto "github.com/gogo/protobuf/proto" - "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/store/prefix" @@ -64,34 +62,20 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz key := grantStoreKey(grantee, granter, "") grantsStore := prefix.NewStore(store, key) - var authorizations []*authz.Grant - pageRes, err := query.FilteredPaginate(grantsStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { - auth, err := unmarshalAuthorization(k.cdc, value) + authorizations, pageRes, err := query.GenericFilteredPaginated(grantsStore, req.Pagination, func(key []byte, value *authz.Grant) (*authz.Grant, error) { + auth, err := value.GetAuthorization() if err != nil { - return false, err + return nil, err } - auth1, err := auth.GetAuthorization() + authorizationAny, err := codectypes.NewAnyWithValue(auth) if err != nil { - return false, err - } - - if accumulate { - msg, ok := auth1.(proto.Message) - if !ok { - return false, status.Errorf(codes.Internal, "can't protomarshal %T", msg) - } - - authorizationAny, err := codectypes.NewAnyWithValue(msg) - if err != nil { - return false, status.Errorf(codes.Internal, err.Error()) - } - authorizations = append(authorizations, &authz.Grant{ - Authorization: authorizationAny, - Expiration: auth.Expiration, - }) + return nil, status.Errorf(codes.Internal, err.Error()) } - return true, nil + return &authz.Grant{ + Authorization: authorizationAny, + Expiration: value.Expiration, + }, nil }) if err != nil { return nil, err From cb28171529a679e023d9582d50ee03d9573ff9ee Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Tue, 14 Jun 2022 17:31:51 -0300 Subject: [PATCH 02/12] progress --- types/query/filtered_pagination.go | 32 +++++++----- x/authz/keeper/grpc_query.go | 84 +++++++++++------------------- x/authz/keeper/grpc_query_test.go | 3 +- 3 files changed, 51 insertions(+), 68 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 245c32cc70ac..ca4668c20060 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -3,8 +3,8 @@ package query import ( "fmt" + "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/store/types" - proto "github.com/gogo/protobuf/proto" ) // FilteredPaginate does pagination of all the results in the PrefixStore based on the @@ -116,11 +116,12 @@ func FilteredPaginate( return res, nil } -func GenericFilteredPaginated[V proto.Message]( +func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( + cdc codec.BinaryCodec, prefixStore types.KVStore, pageRequest *PageRequest, - onResult func(key []byte, value V) (V, error), -) ([]V, *PageResponse, error) { + onResult func(key []byte, value V) (F, error), +) ([]F, *PageResponse, error) { // if the PageRequest is nil, use default PageRequest if pageRequest == nil { pageRequest = &PageRequest{} @@ -131,7 +132,7 @@ func GenericFilteredPaginated[V proto.Message]( limit := pageRequest.Limit countTotal := pageRequest.CountTotal reverse := pageRequest.Reverse - results := []V{} + results := []F{} if offset > 0 && key != nil { return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") @@ -161,18 +162,18 @@ func GenericFilteredPaginated[V proto.Message]( return nil, nil, iterator.Error() } - var protoMsg *V - err := proto.Unmarshal(iterator.Value(), *protoMsg) + protoMsg := any(new(V)) + err := cdc.Unmarshal(iterator.Value(), protoMsg.(codec.ProtoMarshaler)) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), *protoMsg) + val, err := onResult(iterator.Key(), *(protoMsg.(*V))) if err != nil { return nil, nil, err } - if protoMsg != nil { + if any(val) != nil { results = append(results, val) numHits++ } @@ -196,19 +197,22 @@ func GenericFilteredPaginated[V proto.Message]( return nil, nil, iterator.Error() } - var protoMsg *V - err := proto.Unmarshal(iterator.Value(), *protoMsg) + protoMsg := any(new(V)) + err := cdc.Unmarshal(iterator.Value(), protoMsg.(codec.ProtoMarshaler)) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), *protoMsg) + val, err := onResult(iterator.Key(), *(protoMsg.(*V))) if err != nil { return nil, nil, err } - if protoMsg != nil { - results = append(results, val) + if any(val) != nil { + // Previously this was the "accumulate" flag + if numHits >= offset && numHits < end { + results = append(results, val) + } numHits++ } diff --git a/x/authz/keeper/grpc_query.go b/x/authz/keeper/grpc_query.go index d7a86b26da21..e1a54d6dd619 100644 --- a/x/authz/keeper/grpc_query.go +++ b/x/authz/keeper/grpc_query.go @@ -6,7 +6,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/store/prefix" sdk "github.com/cosmos/cosmos-sdk/types" @@ -62,19 +61,19 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz key := grantStoreKey(grantee, granter, "") grantsStore := prefix.NewStore(store, key) - authorizations, pageRes, err := query.GenericFilteredPaginated(grantsStore, req.Pagination, func(key []byte, value *authz.Grant) (*authz.Grant, error) { - auth, err := value.GetAuthorization() + authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, grantsStore, req.Pagination, func(key []byte, auth authz.Grant) (*authz.Grant, error) { + auth1, err := auth.GetAuthorization() if err != nil { return nil, err } - authorizationAny, err := codectypes.NewAnyWithValue(auth) + authorizationAny, err := codectypes.NewAnyWithValue(auth1) if err != nil { return nil, status.Errorf(codes.Internal, err.Error()) } return &authz.Grant{ Authorization: authorizationAny, - Expiration: value.Expiration, + Expiration: auth.Expiration, }, nil }) if err != nil { @@ -102,34 +101,27 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe store := ctx.KVStore(k.storeKey) authzStore := prefix.NewStore(store, grantStoreKey(nil, granter, "")) - var grants []*authz.GrantAuthorization - pageRes, err := query.FilteredPaginate(authzStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { - auth, err := unmarshalAuthorization(k.cdc, value) + grants, pageRes, err := query.GenericFilteredPaginate(k.cdc, authzStore, req.Pagination, func(key []byte, auth authz.Grant) (*authz.GrantAuthorization, error) { + auth1, err := auth.GetAuthorization() if err != nil { - return false, err + return nil, err } - auth1, err := auth.GetAuthorization() + any, err := codectypes.NewAnyWithValue(auth1) if err != nil { - return false, err + return nil, status.Errorf(codes.Internal, err.Error()) } - if accumulate { - any, err := codectypes.NewAnyWithValue(auth1) - if err != nil { - return false, status.Errorf(codes.Internal, err.Error()) - } - - grantee := firstAddressFromGrantStoreKey(key) - grants = append(grants, &authz.GrantAuthorization{ - Granter: granter.String(), - Grantee: grantee.String(), - Authorization: any, - Expiration: auth.Expiration, - }) - } - return true, nil + grantee := firstAddressFromGrantStoreKey(key) + return &authz.GrantAuthorization{ + Granter: granter.String(), + Grantee: grantee.String(), + Authorization: any, + Expiration: auth.Expiration, + }, nil + }) + if err != nil { return nil, err } @@ -154,36 +146,28 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe ctx := sdk.UnwrapSDKContext(c) store := prefix.NewStore(ctx.KVStore(k.storeKey), GrantKey) - var authorizations []*authz.GrantAuthorization - pageRes, err := query.FilteredPaginate(store, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { - auth, err := unmarshalAuthorization(k.cdc, value) + authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, store, req.Pagination, func(key []byte, auth authz.Grant) (*authz.GrantAuthorization, error) { + auth1, err := auth.GetAuthorization() if err != nil { - return false, err + return nil, err } granter, g, _ := parseGrantStoreKey(append(GrantKey, key...)) if !g.Equals(grantee) { - return false, nil + return nil, nil } - auth1, err := auth.GetAuthorization() + authorizationAny, err := codectypes.NewAnyWithValue(auth1) if err != nil { - return false, err - } - if accumulate { - any, err := codectypes.NewAnyWithValue(auth1) - if err != nil { - return false, status.Errorf(codes.Internal, err.Error()) - } - - authorizations = append(authorizations, &authz.GrantAuthorization{ - Authorization: any, - Expiration: auth.Expiration, - Granter: granter.String(), - Grantee: grantee.String(), - }) + return nil, status.Errorf(codes.Internal, err.Error()) } - return true, nil + + return &authz.GrantAuthorization{ + Authorization: authorizationAny, + Expiration: auth.Expiration, + Granter: granter.String(), + Grantee: grantee.String(), + }, nil }) if err != nil { return nil, err @@ -194,9 +178,3 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe Pagination: pageRes, }, nil } - -// unmarshal an authorization from a store value -func unmarshalAuthorization(cdc codec.BinaryCodec, value []byte) (v authz.Grant, err error) { - err = cdc.Unmarshal(value, &v) - return v, err -} diff --git a/x/authz/keeper/grpc_query_test.go b/x/authz/keeper/grpc_query_test.go index d2f0b09e8b19..9b717e9d40d7 100644 --- a/x/authz/keeper/grpc_query_test.go +++ b/x/authz/keeper/grpc_query_test.go @@ -198,7 +198,8 @@ func (suite *TestSuite) TestGRPCQueryGranterGrants() { }, { "valid case, pagination", - func() {}, + func() { + }, false, authz.QueryGranterGrantsRequest{ Granter: addrs[0].String(), From 4ebaaaa784e637ec9382b10ceb1025ddcd61173b Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Tue, 14 Jun 2022 19:42:53 -0300 Subject: [PATCH 03/12] fix when returned object is nil --- types/query/filtered_pagination.go | 8 ++++---- x/authz/keeper/grpc_query_test.go | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index ca4668c20060..53886b83e705 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -135,7 +135,7 @@ func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( results := []F{} if offset > 0 && key != nil { - return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") + return results, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") } if limit == 0 { @@ -173,13 +173,13 @@ func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( return nil, nil, err } - if any(val) != nil { + if val.Size() != 0 { results = append(results, val) numHits++ } } - return nil, &PageResponse{ + return results, &PageResponse{ NextKey: nextKey, }, nil } @@ -208,7 +208,7 @@ func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( return nil, nil, err } - if any(val) != nil { + if val.Size() != 0 { // Previously this was the "accumulate" flag if numHits >= offset && numHits < end { results = append(results, val) diff --git a/x/authz/keeper/grpc_query_test.go b/x/authz/keeper/grpc_query_test.go index 9b717e9d40d7..440859ee24b1 100644 --- a/x/authz/keeper/grpc_query_test.go +++ b/x/authz/keeper/grpc_query_test.go @@ -254,6 +254,15 @@ func (suite *TestSuite) TestGRPCQueryGranteeGrants() { }, 1, }, + { + "valid case, no authorization found", + func() {}, + false, + authz.QueryGranteeGrantsRequest{ + Grantee: addrs[2].String(), + }, + 0, + }, { "valid case, multiple authorization", func() { From 6db3de3d85fae502b7334f14869185564e7b1031 Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Wed, 15 Jun 2022 14:19:18 -0300 Subject: [PATCH 04/12] use reflect to accept codec.ProtoMarshaler --- types/query/filtered_pagination.go | 27 +++++++++++++++++++-------- x/authz/keeper/grpc_query.go | 6 +++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 53886b83e705..68f9f56a88a2 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -2,6 +2,7 @@ package query import ( "fmt" + "reflect" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/store/types" @@ -116,11 +117,11 @@ func FilteredPaginate( return res, nil } -func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( +func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( cdc codec.BinaryCodec, prefixStore types.KVStore, pageRequest *PageRequest, - onResult func(key []byte, value V) (F, error), + onResult func(key []byte, value T) (F, error), ) ([]F, *PageResponse, error) { // if the PageRequest is nil, use default PageRequest if pageRequest == nil { @@ -162,13 +163,18 @@ func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := any(new(V)) - err := cdc.Unmarshal(iterator.Value(), protoMsg.(codec.ProtoMarshaler)) + // initialize the proto message + va := reflect.ValueOf(new(T)).Elem() + v := reflect.New(va.Type().Elem()) + va.Set(v) + protoMsg := va.Interface().(codec.ProtoMarshaler) + + err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), *(protoMsg.(*V))) + val, err := onResult(iterator.Key(), protoMsg.(T)) if err != nil { return nil, nil, err } @@ -197,13 +203,18 @@ func GenericFilteredPaginate[V any, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := any(new(V)) - err := cdc.Unmarshal(iterator.Value(), protoMsg.(codec.ProtoMarshaler)) + // initialize the proto message + va := reflect.ValueOf(new(T)).Elem() + v := reflect.New(va.Type().Elem()) + va.Set(v) + protoMsg := va.Interface().(codec.ProtoMarshaler) + + err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), *(protoMsg.(*V))) + val, err := onResult(iterator.Key(), (protoMsg).(T)) if err != nil { return nil, nil, err } diff --git a/x/authz/keeper/grpc_query.go b/x/authz/keeper/grpc_query.go index e1a54d6dd619..88e057e44aa6 100644 --- a/x/authz/keeper/grpc_query.go +++ b/x/authz/keeper/grpc_query.go @@ -61,7 +61,7 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz key := grantStoreKey(grantee, granter, "") grantsStore := prefix.NewStore(store, key) - authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, grantsStore, req.Pagination, func(key []byte, auth authz.Grant) (*authz.Grant, error) { + authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, grantsStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.Grant, error) { auth1, err := auth.GetAuthorization() if err != nil { return nil, err @@ -101,7 +101,7 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe store := ctx.KVStore(k.storeKey) authzStore := prefix.NewStore(store, grantStoreKey(nil, granter, "")) - grants, pageRes, err := query.GenericFilteredPaginate(k.cdc, authzStore, req.Pagination, func(key []byte, auth authz.Grant) (*authz.GrantAuthorization, error) { + grants, pageRes, err := query.GenericFilteredPaginate(k.cdc, authzStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { auth1, err := auth.GetAuthorization() if err != nil { return nil, err @@ -146,7 +146,7 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe ctx := sdk.UnwrapSDKContext(c) store := prefix.NewStore(ctx.KVStore(k.storeKey), GrantKey) - authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, store, req.Pagination, func(key []byte, auth authz.Grant) (*authz.GrantAuthorization, error) { + authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, store, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { auth1, err := auth.GetAuthorization() if err != nil { return nil, err From 85fc29bc3e504467ab76231241b63bd90bb85ffc Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Wed, 15 Jun 2022 14:35:17 -0300 Subject: [PATCH 05/12] suggestion --- types/query/filtered_pagination.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 68f9f56a88a2..f71bd0988b65 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -47,8 +47,10 @@ func FilteredPaginate( iterator := getIterator(prefixStore, key, reverse) defer iterator.Close() - var numHits uint64 - var nextKey []byte + var ( + numHits uint64 + nextKey []byte + ) for ; iterator.Valid(); iterator.Next() { if numHits == limit { @@ -80,8 +82,10 @@ func FilteredPaginate( end := offset + limit - var numHits uint64 - var nextKey []byte + var ( + numHits uint64 + nextKey []byte + ) for ; iterator.Valid(); iterator.Next() { if iterator.Error() != nil { @@ -195,8 +199,10 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( end := offset + limit - var numHits uint64 - var nextKey []byte + var ( + numHits uint64 + nextKey []byte + ) for ; iterator.Valid(); iterator.Next() { if iterator.Error() != nil { From 6353df02fe5673eebbf37c7c075e59ae50bfb92f Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Wed, 15 Jun 2022 15:43:18 -0300 Subject: [PATCH 06/12] try out constructor --- types/query/filtered_pagination.go | 18 +++++------------- x/authz/keeper/grpc_query.go | 6 ++++++ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index f71bd0988b65..9d1614e34b6d 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -2,7 +2,6 @@ package query import ( "fmt" - "reflect" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/store/types" @@ -126,6 +125,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value T) (F, error), + constructor func() T, ) ([]F, *PageResponse, error) { // if the PageRequest is nil, use default PageRequest if pageRequest == nil { @@ -167,18 +167,14 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - // initialize the proto message - va := reflect.ValueOf(new(T)).Elem() - v := reflect.New(va.Type().Elem()) - va.Set(v) - protoMsg := va.Interface().(codec.ProtoMarshaler) + protoMsg := constructor() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), protoMsg.(T)) + val, err := onResult(iterator.Key(), protoMsg) if err != nil { return nil, nil, err } @@ -209,18 +205,14 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - // initialize the proto message - va := reflect.ValueOf(new(T)).Elem() - v := reflect.New(va.Type().Elem()) - va.Set(v) - protoMsg := va.Interface().(codec.ProtoMarshaler) + protoMsg := constructor() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { return nil, nil, err } - val, err := onResult(iterator.Key(), (protoMsg).(T)) + val, err := onResult(iterator.Key(), protoMsg) if err != nil { return nil, nil, err } diff --git a/x/authz/keeper/grpc_query.go b/x/authz/keeper/grpc_query.go index 88e057e44aa6..3273ee282533 100644 --- a/x/authz/keeper/grpc_query.go +++ b/x/authz/keeper/grpc_query.go @@ -75,6 +75,8 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz Authorization: authorizationAny, Expiration: auth.Expiration, }, nil + }, func() *authz.Grant { + return &authz.Grant{} }) if err != nil { return nil, err @@ -120,6 +122,8 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe Expiration: auth.Expiration, }, nil + }, func() *authz.Grant { + return &authz.Grant{} }) if err != nil { @@ -168,6 +172,8 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe Granter: granter.String(), Grantee: grantee.String(), }, nil + }, func() *authz.Grant { + return &authz.Grant{} }) if err != nil { return nil, err From 23b5fdc1162144e5708f50020da69ec88435f8af Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Wed, 15 Jun 2022 17:45:20 -0300 Subject: [PATCH 07/12] rename --- types/query/filtered_pagination.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 9d1614e34b6d..0fcf8c183cbc 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -125,7 +125,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value T) (F, error), - constructor func() T, + c func() T, ) ([]F, *PageResponse, error) { // if the PageRequest is nil, use default PageRequest if pageRequest == nil { @@ -167,7 +167,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := constructor() + protoMsg := c() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { @@ -205,7 +205,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := constructor() + protoMsg := c() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { From bf01daababb2d16d42d4e2af5f1f4cf87408cf82 Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Thu, 16 Jun 2022 10:36:10 -0300 Subject: [PATCH 08/12] docs --- types/query/filtered_pagination.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 0fcf8c183cbc..d6bd74913834 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -120,6 +120,13 @@ func FilteredPaginate( return res, nil } +// GenericFilteredPaginate does pagination of all the results in the PrefixStore based on the +// provided PageRequest. `onResult` should be used to filter or transform the results. +// `c` is a constructor function that needs to return a new instance of the type T. +// If key is provided, the pagination uses the optimized querying. +// If offset is used, the pagination uses lazy filtering i.e., searches through all the records. +// The resulting slice (of type F) can be of a different type than the one being iterated through +// (type T), so it's possible to do any necessary transformation inside the onResult function. func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( cdc codec.BinaryCodec, prefixStore types.KVStore, @@ -154,8 +161,10 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( iterator := getIterator(prefixStore, key, reverse) defer iterator.Close() - var numHits uint64 - var nextKey []byte + var ( + numHits uint64 + nextKey []byte + ) for ; iterator.Valid(); iterator.Next() { if numHits == limit { From 2c59802057c53226b4de30bb176d95d74b585e0e Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Thu, 16 Jun 2022 10:39:13 -0300 Subject: [PATCH 09/12] docs --- types/query/filtered_pagination.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index d6bd74913834..4f15f05553f6 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -122,7 +122,8 @@ func FilteredPaginate( // GenericFilteredPaginate does pagination of all the results in the PrefixStore based on the // provided PageRequest. `onResult` should be used to filter or transform the results. -// `c` is a constructor function that needs to return a new instance of the type T. +// `c` is a constructor function that needs to return a new instance of the type T (this is to +// workaround some generic pitfalls in which we can't instantiate a T struct inside the function). // If key is provided, the pagination uses the optimized querying. // If offset is used, the pagination uses lazy filtering i.e., searches through all the records. // The resulting slice (of type F) can be of a different type than the one being iterated through From 40cd7b62b1167f7963a982a8017487207d6fe696 Mon Sep 17 00:00:00 2001 From: Facundo Medica <14063057+facundomedica@users.noreply.github.com> Date: Mon, 27 Jun 2022 07:53:09 -0300 Subject: [PATCH 10/12] Apply suggestions from code review Co-authored-by: Aleksandr Bezobchuk --- types/query/filtered_pagination.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 4f15f05553f6..81fa265438d3 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -133,7 +133,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value T) (F, error), - c func() T, + constructor func() T, ) ([]F, *PageResponse, error) { // if the PageRequest is nil, use default PageRequest if pageRequest == nil { From baf7f7f91f3dd83782419fca84fb38967b438cdf Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Mon, 27 Jun 2022 08:03:52 -0300 Subject: [PATCH 11/12] cl++, small fix --- CHANGELOG.md | 1 + types/query/filtered_pagination.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd1265333ba..9bb091cb231b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features * (cli) [#12028](https://github.com/cosmos/cosmos-sdk/pull/12028) Add the `tendermint key-migrate` to perform Tendermint v0.35 DB key migration. +* (orm) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the ORM to improve UX. ### Improvements diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go index 81fa265438d3..50c85ac49a51 100644 --- a/types/query/filtered_pagination.go +++ b/types/query/filtered_pagination.go @@ -177,7 +177,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := c() + protoMsg := constructor() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { @@ -215,7 +215,7 @@ func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( return nil, nil, iterator.Error() } - protoMsg := c() + protoMsg := constructor() err := cdc.Unmarshal(iterator.Value(), protoMsg) if err != nil { From b0070c226b86d883d32797bf8b6f9eacba8c3fd1 Mon Sep 17 00:00:00 2001 From: Facundo Medica Date: Mon, 27 Jun 2022 08:20:30 -0300 Subject: [PATCH 12/12] cl++ --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb091cb231b..301c6e4deb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features * (cli) [#12028](https://github.com/cosmos/cosmos-sdk/pull/12028) Add the `tendermint key-migrate` to perform Tendermint v0.35 DB key migration. -* (orm) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the ORM to improve UX. +* (query) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the `query` package to improve UX. ### Improvements