Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup codec usage #2563

Merged
merged 11 commits into from
Dec 29, 2023
Merged

Cleanup codec usage #2563

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a322a36
Cleanup codec usage
StephenButtolph Dec 29, 2023
0d71dc9
reduce diff
StephenButtolph Dec 29, 2023
207610f
nits
StephenButtolph Dec 29, 2023
fb75170
nit
StephenButtolph Dec 29, 2023
196d1f2
nit
StephenButtolph Dec 29, 2023
b0cd447
nit
StephenButtolph Dec 29, 2023
d16862a
nit
StephenButtolph Dec 29, 2023
ed5f962
nit
StephenButtolph Dec 29, 2023
cb754fd
nit
StephenButtolph Dec 29, 2023
9f86b13
merged
StephenButtolph Dec 29, 2023
c212691
save
StephenButtolph Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions api/keystore/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@ import (
)

const (
CodecVersion = 0

maxPackerSize = 1 * units.GiB // max size, in bytes, of something being marshalled by Marshal()
maxSliceLength = linearcodec.DefaultMaxSliceLength

codecVersion = 0
)

var c codec.Manager
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Codec is much easier to read than c and this allows easier usage of the format.

var Codec codec.Manager

func init() {
lc := linearcodec.NewCustomMaxLength(maxSliceLength)
c = codec.NewManager(maxPackerSize)
if err := c.RegisterCodec(codecVersion, lc); err != nil {
Codec = codec.NewManager(maxPackerSize)
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
}
}
10 changes: 5 additions & 5 deletions api/keystore/keystore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ func (ks *keystore) CreateUser(username, pw string) error {
return err
}

passwordBytes, err := c.Marshal(codecVersion, passwordHash)
passwordBytes, err := Codec.Marshal(CodecVersion, passwordHash)
if err != nil {
return err
}
Expand Down Expand Up @@ -288,14 +288,14 @@ func (ks *keystore) ImportUser(username, pw string, userBytes []byte) error {
}

userData := user{}
if _, err := c.Unmarshal(userBytes, &userData); err != nil {
if _, err := Codec.Unmarshal(userBytes, &userData); err != nil {
return err
}
if !userData.Hash.Check(pw) {
return fmt.Errorf("%w: user %q", errIncorrectPassword, username)
}

usrBytes, err := c.Marshal(codecVersion, &userData.Hash)
usrBytes, err := Codec.Marshal(CodecVersion, &userData.Hash)
if err != nil {
return err
}
Expand Down Expand Up @@ -355,7 +355,7 @@ func (ks *keystore) ExportUser(username, pw string) ([]byte, error) {
}

// Return the byte representation of the user
return c.Marshal(codecVersion, &userData)
return Codec.Marshal(CodecVersion, &userData)
}

func (ks *keystore) getPassword(username string) (*password.Hash, error) {
Expand All @@ -377,6 +377,6 @@ func (ks *keystore) getPassword(username string) (*password.Hash, error) {
}

passwordHash = &password.Hash{}
_, err = c.Unmarshal(userBytes, passwordHash)
_, err = Codec.Unmarshal(userBytes, passwordHash)
return passwordHash, err
}
12 changes: 6 additions & 6 deletions chains/atomic/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ import (
"github.com/ava-labs/avalanchego/codec/linearcodec"
)

const codecVersion = 0
const CodecVersion = 0

// codecManager is used to marshal and unmarshal dbElements and chain IDs.
var codecManager codec.Manager
// Codec is used to marshal and unmarshal dbElements and chain IDs.
var Codec codec.Manager

func init() {
linearCodec := linearcodec.NewDefault()
codecManager = codec.NewDefaultManager()
if err := codecManager.RegisterCodec(codecVersion, linearCodec); err != nil {
lc := linearcodec.NewDefault()
Codec = codec.NewDefaultManager()
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
}
}
2 changes: 1 addition & 1 deletion chains/atomic/memory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func sharedID(id1, id2 ids.ID) ids.ID {
id1, id2 = id2, id1
}

combinedBytes, err := codecManager.Marshal(codecVersion, [2]ids.ID{id1, id2})
combinedBytes, err := Codec.Marshal(CodecVersion, [2]ids.ID{id1, id2})
if err != nil {
panic(err)
}
Expand Down
6 changes: 3 additions & 3 deletions chains/atomic/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func (s *state) SetValue(e *Element) error {
Traits: e.Traits,
}

valueBytes, err := codecManager.Marshal(codecVersion, &dbElem)
valueBytes, err := Codec.Marshal(CodecVersion, &dbElem)
if err != nil {
return err
}
Expand Down Expand Up @@ -156,7 +156,7 @@ func (s *state) RemoveValue(key []byte) error {

// The value doesn't exist, so we should optimistically delete it
dbElem := dbElement{Present: false}
valueBytes, err := codecManager.Marshal(codecVersion, &dbElem)
valueBytes, err := Codec.Marshal(CodecVersion, &dbElem)
if err != nil {
return err
}
Expand Down Expand Up @@ -189,7 +189,7 @@ func (s *state) loadValue(key []byte) (*dbElement, error) {

// The key was in the database
value := &dbElement{}
_, err = codecManager.Unmarshal(valueBytes, value)
_, err = Codec.Unmarshal(valueBytes, value)
return value, err
}

Expand Down
22 changes: 22 additions & 0 deletions database/encdb/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package encdb

import (
"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
)

const CodecVersion = 0

var Codec codec.Manager

func init() {
lc := linearcodec.NewDefault()
Codec = codec.NewDefaultManager()

if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
}
}
Comment on lines +11 to +22
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was no reason we created a new codec for each encdb instance.

19 changes: 3 additions & 16 deletions database/encdb/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,10 @@ import (

"golang.org/x/exp/slices"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
"github.com/ava-labs/avalanchego/database"
"github.com/ava-labs/avalanchego/utils/hashing"
)

const (
codecVersion = 0
)

var (
_ database.Database = (*Database)(nil)
_ database.Batch = (*batch)(nil)
Expand All @@ -32,7 +26,6 @@ var (
// Database encrypts all values that are provided
type Database struct {
lock sync.RWMutex
codec codec.Manager
cipher cipher.AEAD
db database.Database
closed bool
Expand All @@ -42,16 +35,10 @@ type Database struct {
func New(password []byte, db database.Database) (*Database, error) {
h := hashing.ComputeHash256(password)
aead, err := chacha20poly1305.NewX(h)
if err != nil {
return nil, err
}
c := linearcodec.NewDefault()
manager := codec.NewDefaultManager()
return &Database{
codec: manager,
cipher: aead,
db: db,
}, manager.RegisterCodec(codecVersion, c)
}, err
}

func (db *Database) Has(key []byte) (bool, error) {
Expand Down Expand Up @@ -297,15 +284,15 @@ func (db *Database) encrypt(plaintext []byte) ([]byte, error) {
return nil, err
}
ciphertext := db.cipher.Seal(nil, nonce, plaintext, nil)
return db.codec.Marshal(codecVersion, &encryptedValue{
return Codec.Marshal(CodecVersion, &encryptedValue{
Ciphertext: ciphertext,
Nonce: nonce,
})
}

func (db *Database) decrypt(ciphertext []byte) ([]byte, error) {
val := encryptedValue{}
if _, err := db.codec.Unmarshal(ciphertext, &val); err != nil {
if _, err := Codec.Unmarshal(ciphertext, &val); err != nil {
return nil, err
}
return db.cipher.Open(nil, val.Nonce, val.Ciphertext, nil)
Expand Down
13 changes: 4 additions & 9 deletions database/linkeddb/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,15 @@ import (
"github.com/ava-labs/avalanchego/codec/linearcodec"
)

const (
codecVersion = 0
)
const CodecVersion = 0

// c does serialization and deserialization
var (
c codec.Manager
)
var Codec codec.Manager

func init() {
lc := linearcodec.NewCustomMaxLength(math.MaxUint32)
c = codec.NewManager(math.MaxInt32)
Codec = codec.NewManager(math.MaxInt32)

if err := c.RegisterCodec(codecVersion, lc); err != nil {
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
}
}
4 changes: 2 additions & 2 deletions database/linkeddb/linkeddb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ func (ldb *linkedDB) getNode(key []byte) (node, error) {
return node{}, err
}
n := node{}
_, err = c.Unmarshal(nodeBytes, &n)
_, err = Codec.Unmarshal(nodeBytes, &n)
if err == nil {
ldb.nodeCache.Put(keyStr, &n)
}
Expand All @@ -325,7 +325,7 @@ func (ldb *linkedDB) getNode(key []byte) (node, error) {

func (ldb *linkedDB) putNode(key []byte, n node) error {
ldb.updatedNodes[string(key)] = &n
nodeBytes, err := c.Marshal(codecVersion, n)
nodeBytes, err := Codec.Marshal(CodecVersion, n)
if err != nil {
return err
}
Expand Down
24 changes: 24 additions & 0 deletions indexer/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package indexer

import (
"math"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
)

const CodecVersion = 0

var Codec codec.Manager

func init() {
lc := linearcodec.NewCustomMaxLength(math.MaxUint32)
Codec = codec.NewManager(math.MaxInt)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was no reason to have a complex calculated maximum length here.


if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
}
}
Comment on lines +13 to +24
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was no reason we created a new codec for each indexer.

8 changes: 2 additions & 6 deletions indexer/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (

"go.uber.org/zap"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/database"
"github.com/ava-labs/avalanchego/database/prefixdb"
"github.com/ava-labs/avalanchego/database/versiondb"
Expand Down Expand Up @@ -55,7 +54,6 @@ type Index interface {

// indexer indexes all accepted transactions by the order in which they were accepted
type index struct {
codec codec.Manager
clock mockable.Clock
lock sync.RWMutex
// The index of the next accepted transaction
Expand All @@ -76,7 +74,6 @@ type index struct {
func newIndex(
baseDB database.Database,
log logging.Logger,
codec codec.Manager,
clock mockable.Clock,
) (Index, error) {
vDB := versiondb.New(baseDB)
Expand All @@ -85,7 +82,6 @@ func newIndex(

i := &index{
clock: clock,
codec: codec,
baseDB: baseDB,
vDB: vDB,
indexToContainer: indexToContainer,
Expand Down Expand Up @@ -150,7 +146,7 @@ func (i *index) Accept(ctx *snow.ConsensusContext, containerID ids.ID, container
)
// Persist index --> Container
nextAcceptedIndexBytes := database.PackUInt64(i.nextAcceptedIndex)
bytes, err := i.codec.Marshal(codecVersion, Container{
bytes, err := Codec.Marshal(CodecVersion, Container{
ID: containerID,
Bytes: containerBytes,
Timestamp: i.clock.Time().UnixNano(),
Expand Down Expand Up @@ -209,7 +205,7 @@ func (i *index) getContainerByIndexBytes(indexBytes []byte) (Container, error) {
return Container{}, fmt.Errorf("couldn't read from database: %w", err)
}
var container Container
if _, err := i.codec.Unmarshal(containerBytes, &container); err != nil {
if _, err := Codec.Unmarshal(containerBytes, &container); err != nil {
return Container{}, fmt.Errorf("couldn't unmarshal container: %w", err)
}
return container, nil
Expand Down
16 changes: 4 additions & 12 deletions indexer/index_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (

"github.com/stretchr/testify/require"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
"github.com/ava-labs/avalanchego/database/memdb"
"github.com/ava-labs/avalanchego/database/versiondb"
"github.com/ava-labs/avalanchego/ids"
Expand All @@ -24,14 +22,12 @@ func TestIndex(t *testing.T) {
// Setup
pageSize := uint64(64)
require := require.New(t)
codec := codec.NewDefaultManager()
require.NoError(codec.RegisterCodec(codecVersion, linearcodec.NewDefault()))
baseDB := memdb.New()
db := versiondb.New(baseDB)
snowCtx := snowtest.Context(t, snowtest.CChainID)
ctx := snowtest.ConsensusContext(snowCtx)

indexIntf, err := newIndex(db, logging.NoLog{}, codec, mockable.Clock{})
indexIntf, err := newIndex(db, logging.NoLog{}, mockable.Clock{})
require.NoError(err)
idx := indexIntf.(*index)

Expand Down Expand Up @@ -84,7 +80,7 @@ func TestIndex(t *testing.T) {
require.NoError(db.Commit())
require.NoError(idx.Close())
db = versiondb.New(baseDB)
indexIntf, err = newIndex(db, logging.NoLog{}, codec, mockable.Clock{})
indexIntf, err = newIndex(db, logging.NoLog{}, mockable.Clock{})
require.NoError(err)
idx = indexIntf.(*index)

Expand Down Expand Up @@ -114,12 +110,10 @@ func TestIndex(t *testing.T) {
func TestIndexGetContainerByRangeMaxPageSize(t *testing.T) {
// Setup
require := require.New(t)
codec := codec.NewDefaultManager()
require.NoError(codec.RegisterCodec(codecVersion, linearcodec.NewDefault()))
db := memdb.New()
snowCtx := snowtest.Context(t, snowtest.CChainID)
ctx := snowtest.ConsensusContext(snowCtx)
indexIntf, err := newIndex(db, logging.NoLog{}, codec, mockable.Clock{})
indexIntf, err := newIndex(db, logging.NoLog{}, mockable.Clock{})
require.NoError(err)
idx := indexIntf.(*index)

Expand Down Expand Up @@ -155,12 +149,10 @@ func TestIndexGetContainerByRangeMaxPageSize(t *testing.T) {
func TestDontIndexSameContainerTwice(t *testing.T) {
// Setup
require := require.New(t)
codec := codec.NewDefaultManager()
require.NoError(codec.RegisterCodec(codecVersion, linearcodec.NewDefault()))
db := memdb.New()
snowCtx := snowtest.Context(t, snowtest.CChainID)
ctx := snowtest.ConsensusContext(snowCtx)
idx, err := newIndex(db, logging.NoLog{}, codec, mockable.Clock{})
idx, err := newIndex(db, logging.NoLog{}, mockable.Clock{})
require.NoError(err)

// Accept the same container twice
Expand Down
Loading
Loading