Catchup to python-herald schema. Plus lots of refactoring.

.gitignore

@@ -1 +1,3 @@
 .idea/
+.vscode/
+.DS_Store

db/db.go

@@ -850,8 +850,8 @@ func ReadPrefixN(db *grocksdb.DB, prefix []byte, n int) []*prefixes.PrefixRowKV
 		value := it.Value()
 
 		res[i] = &prefixes.PrefixRowKV{
-			Key:   key.Data(),
-			Value: value.Data(),
+			RawKey:   key.Data(),
+			RawValue: value.Data(),
 		}
 
 		key.Free()
@@ -908,8 +908,8 @@ func readWriteRawNCF(db *grocksdb.DB, options *IterOptions, out string, n int, f
 		if i >= n {
 			return
 		}
-		key := kv.Key.([]byte)
-		value := kv.Value.([]byte)
+		key := kv.RawKey
+		value := kv.RawValue
 		keyHex := hex.EncodeToString(key)
 		valueHex := hex.EncodeToString(value)
 		//log.Println(keyHex)
@@ -947,8 +947,8 @@ func ReadWriteRawN(db *grocksdb.DB, options *IterOptions, out string, n int) {
 		if i >= n {
 			return
 		}
-		key := kv.Key.([]byte)
-		value := kv.Value.([]byte)
+		key := kv.RawKey
+		value := kv.RawValue
 		keyHex := hex.EncodeToString(key)
 		valueHex := hex.EncodeToString(value)
 		log.Println(keyHex)

db/db_get.go

@@ -94,7 +94,7 @@ func (db *ReadOnlyDBColumnFamily) GetStreamsAndChannelRepostedByChannelHashes(re
 
 	for _, reposterChannelHash := range reposterChannelHashes {
 		key := prefixes.NewChannelToClaimKeyWHash(reposterChannelHash)
-		rawKeyPrefix := prefixes.ChannelToClaimKeyPackPartial(key, 1)
+		rawKeyPrefix := key.PartialPack(1)
 		options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 		options = options.WithIncludeKey(false).WithIncludeValue(true)
 		ch := IterCF(db.DB, options)
@@ -167,7 +167,7 @@ func (db *ReadOnlyDBColumnFamily) GetShortClaimIdUrl(name string, normalizedName
 		partialClaimId := claimId[:j]
 		partialKey := prefixes.NewClaimShortIDKey(normalizedName, partialClaimId)
 		log.Printf("partialKey: %#v\n", partialKey)
-		keyPrefix := prefixes.ClaimShortIDKeyPackPartial(partialKey, 2)
+		keyPrefix := partialKey.PartialPack(2)
 		// Prefix and handle
 		options := NewIterateOptions().WithPrefix(prefix).WithCfHandle(handle)
 		// Start and stop bounds
@@ -218,7 +218,7 @@ func (db *ReadOnlyDBColumnFamily) GetRepostedCount(claimHash []byte) (int, error
 	}
 
 	key := prefixes.NewRepostedKey(claimHash)
-	keyPrefix := prefixes.RepostedKeyPackPartial(key, 1)
+	keyPrefix := key.PartialPack(1)
 	// Prefix and handle
 	options := NewIterateOptions().WithPrefix(keyPrefix).WithCfHandle(handle)
 	// Start and stop bounds
@@ -267,8 +267,8 @@ func (db *ReadOnlyDBColumnFamily) GetActiveAmount(claimHash []byte, txoType uint
 	startKey := prefixes.NewActiveAmountKey(claimHash, txoType, 0)
 	endKey := prefixes.NewActiveAmountKey(claimHash, txoType, height)
 
-	startKeyRaw := prefixes.ActiveAmountKeyPackPartial(startKey, 3)
-	endKeyRaw := prefixes.ActiveAmountKeyPackPartial(endKey, 3)
+	startKeyRaw := startKey.PartialPack(3)
+	endKeyRaw := endKey.PartialPack(3)
 	// Prefix and handle
 	options := NewIterateOptions().WithPrefix([]byte{prefixes.ActiveAmount}).WithCfHandle(handle)
 	// Start and stop bounds
@@ -416,7 +416,7 @@ func (db *ReadOnlyDBColumnFamily) ControllingClaimIter() <-chan *prefixes.Prefix
 
 	key := prefixes.NewClaimTakeoverKey("")
 	var rawKeyPrefix []byte = nil
-	rawKeyPrefix = prefixes.ClaimTakeoverKeyPackPartial(key, 0)
+	rawKeyPrefix = key.PartialPack(0)
 	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
@@ -527,7 +527,7 @@ func (db *ReadOnlyDBColumnFamily) EffectiveAmountNameIter(normalizedName string)
 
 	key := prefixes.NewEffectiveAmountKey(normalizedName)
 	var rawKeyPrefix []byte = nil
-	rawKeyPrefix = prefixes.EffectiveAmountKeyPackPartial(key, 1)
+	rawKeyPrefix = key.PartialPack(1)
 	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)
@@ -542,9 +542,9 @@ func (db *ReadOnlyDBColumnFamily) ClaimShortIdIter(normalizedName string, claimI
 	key := prefixes.NewClaimShortIDKey(normalizedName, claimId)
 	var rawKeyPrefix []byte = nil
 	if claimId != "" {
-		rawKeyPrefix = prefixes.ClaimShortIDKeyPackPartial(key, 2)
+		rawKeyPrefix = key.PartialPack(2)
 	} else {
-		rawKeyPrefix = prefixes.ClaimShortIDKeyPackPartial(key, 1)
+		rawKeyPrefix = key.PartialPack(1)
 	}
 	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)

db/db_resolve.go

@@ -323,7 +323,7 @@ func (db *ReadOnlyDBColumnFamily) ResolveClaimInChannel(channelHash []byte, norm
 	}
 
 	key := prefixes.NewChannelToClaimKey(channelHash, normalizedName)
-	rawKeyPrefix := prefixes.ChannelToClaimKeyPackPartial(key, 2)
+	rawKeyPrefix := key.PartialPack(2)
 	options := NewIterateOptions().WithCfHandle(handle).WithPrefix(rawKeyPrefix)
 	options = options.WithIncludeValue(true) //.WithIncludeStop(true)
 	ch := IterCF(db.DB, options)

db/db_test.go

@@ -729,9 +729,9 @@ func TestIter(t *testing.T) {
 		// log.Println(kv.Key)
 		gotKey := kv.Key.(*prefixes.RepostedKey).PackKey()
 
-		keyPartial3 := prefixes.RepostedKeyPackPartial(kv.Key.(*prefixes.RepostedKey), 3)
-		keyPartial2 := prefixes.RepostedKeyPackPartial(kv.Key.(*prefixes.RepostedKey), 2)
-		keyPartial1 := prefixes.RepostedKeyPackPartial(kv.Key.(*prefixes.RepostedKey), 1)
+		keyPartial3 := kv.Key.(*prefixes.RepostedKey).PartialPack(3)
+		keyPartial2 := kv.Key.(*prefixes.RepostedKey).PartialPack(2)
+		keyPartial1 := kv.Key.(*prefixes.RepostedKey).PartialPack(1)
 
 		// Check pack partial for sanity
 		if !bytes.HasPrefix(gotKey, keyPartial3) {

db/iteroptions.go

@@ -24,6 +24,7 @@ type IterOptions struct {
 	RawValue     bool
 	CfHandle     *grocksdb.ColumnFamilyHandle
 	It           *grocksdb.Iterator
+	Serializer   *prefixes.SerializationAPI
 }
 
 // NewIterateOptions creates a defualt options structure for a db iterator.
@@ -41,6 +42,7 @@ func NewIterateOptions() *IterOptions {
 		RawValue:     false,
 		CfHandle:     nil,
 		It:           nil,
+		Serializer:   prefixes.ProductionAPI,
 	}
 }
 
@@ -99,6 +101,11 @@ func (o *IterOptions) WithRawValue(rawValue bool) *IterOptions {
 	return o
 }
 
+func (o *IterOptions) WithSerializer(serializer *prefixes.SerializationAPI) *IterOptions {
+	o.Serializer = serializer
+	return o
+}
+
 // ReadRow reads a row from the db, returns nil when no more rows are available.
 func (opts *IterOptions) ReadRow(prevKey *[]byte) *prefixes.PrefixRowKV {
 	it := opts.It
@@ -117,8 +124,10 @@ func (opts *IterOptions) ReadRow(prevKey *[]byte) *prefixes.PrefixRowKV {
 	valueData := value.Data()
 	valueLen := len(valueData)
 
-	var outKey interface{} = nil
-	var outValue interface{} = nil
+	var outKey prefixes.BaseKey = nil
+	var outValue prefixes.BaseValue = nil
+	var rawOutKey []byte = nil
+	var rawOutValue []byte = nil
 	var err error = nil
 
 	log.Trace("keyData:", keyData)
@@ -136,12 +145,12 @@ func (opts *IterOptions) ReadRow(prevKey *[]byte) *prefixes.PrefixRowKV {
 	newKeyData := make([]byte, keyLen)
 	copy(newKeyData, keyData)
 	if opts.IncludeKey && !opts.RawKey {
-		outKey, err = prefixes.UnpackGenericKey(newKeyData)
+		outKey, err = opts.Serializer.UnpackKey(newKeyData)
 		if err != nil {
 			log.Error(err)
 		}
 	} else if opts.IncludeKey {
-		outKey = newKeyData
+		rawOutKey = newKeyData
 	}
 
 	// Value could be quite large, so this setting could be important
@@ -150,18 +159,20 @@ func (opts *IterOptions) ReadRow(prevKey *[]byte) *prefixes.PrefixRowKV {
 		newValueData := make([]byte, valueLen)
 		copy(newValueData, valueData)
 		if !opts.RawValue {
-			outValue, err = prefixes.UnpackGenericValue(newKeyData, newValueData)
+			outValue, err = opts.Serializer.UnpackValue(newKeyData, newValueData)
 			if err != nil {
 				log.Error(err)
 			}
 		} else {
-			outValue = newValueData
+			rawOutValue = newValueData
 		}
 	}
 
 	kv := &prefixes.PrefixRowKV{
-		Key:   outKey,
-		Value: outValue,
+		Key:      outKey,
+		Value:    outValue,
+		RawKey:   rawOutKey,
+		RawValue: rawOutValue,
 	}
 	*prevKey = newKeyData