Merge remote-tracking branch 'origin/develop' into poet-issue-501

.github/workflows/ci.yml

@@ -282,7 +282,7 @@ jobs:
           GOTESTSUM_JUNITFILE: unit-tests.xml
         run: make test
       - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v4
+        uses: mikepenz/action-junit-report@v5
         # always run even if the previous step fails
         if: always()
         with:

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 See [RELEASE](./RELEASE.md) for workflow instructions.
 
+## Unreleased
+
+### Improvements
+
+* [#6431](https://github.com/spacemeshos/go-spacemesh/pull/6431) Fix db-allow-schema-drift handling
+* [#6408](https://github.com/spacemeshos/go-spacemesh/pull/6408) Prevent empty DB connection pool by freeing connections
+  upon errors during DB operations. This mostly fixes issues when a node is under heavy load from the API.
+
+* [#6417](https://github.com/spacemeshos/go-spacemesh/pull/6417) Fix initial post being deleted when the node is
+  restarted or times out before the first ATX is published.
+
+* [#6422](https://github.com/spacemeshos/go-spacemesh/pull/6422) Further improved performance of the proposal building
+  process to avoid late proposals.
+
 ## v1.7.6
 
 ### Upgrade information

Makefile-libs.Inc

@@ -46,7 +46,7 @@ else
 	endif
 endif
 
-POSTRS_SETUP_REV = 0.7.13
+POSTRS_SETUP_REV = 0.8.1
 POSTRS_SETUP_ZIP = libpost-$(platform)-v$(POSTRS_SETUP_REV).zip
 POSTRS_SETUP_URL_ZIP ?= https://github.com/spacemeshos/post-rs/releases/download/v$(POSTRS_SETUP_REV)/$(POSTRS_SETUP_ZIP)
 

activation/activation.go

@@ -874,7 +874,7 @@ func (b *Builder) createAtx(
 			PositioningATX: challenge.PositioningATX,
 			Coinbase:       b.Coinbase(),
 			VRFNonce:       (uint64)(nipostState.VRFNonce),
-			NiPosts: []wire.NiPostsV2{
+			NIPosts: []wire.NIPostV2{
 				{
 					Membership: wire.MerkleProofV2{
 						Nodes: nipostState.Membership.Nodes,

activation/certifier.go

@@ -344,7 +344,7 @@ func loadPost(ctx context.Context, db sql.Executor, id types.ATXID) (*types.Post
 		if err := codec.Decode(blob.Bytes, &atx); err != nil {
 			return nil, nil, fmt.Errorf("decoding ATX blob: %w", err)
 		}
-		return wire.PostFromWireV1(&atx.NiPosts[0].Posts[0].Post), atx.NiPosts[0].Challenge[:], nil
+		return wire.PostFromWireV1(&atx.NIPosts[0].Posts[0].Post), atx.NIPosts[0].Challenge[:], nil
 	}
 	panic("unsupported ATX version")
 }

activation/e2e/atx_merge_test.go

@@ -97,7 +97,7 @@ func createInitialAtx(
 			Post:          *wire.PostToWireV1(initial),
 		},
 		VRFNonce: uint64(nipost.VRFNonce),
-		NiPosts: []wire.NiPostsV2{
+		NIPosts: []wire.NIPostV2{
 			{
 				Membership: wire.MerkleProofV2{
 					Nodes: nipost.Membership.Nodes,
@@ -121,7 +121,7 @@ func createSoloAtx(publish types.EpochID, prev, pos types.ATXID, nipost *nipost.
 		PreviousATXs:   []types.ATXID{prev},
 		PositioningATX: pos,
 		VRFNonce:       uint64(nipost.VRFNonce),
-		NiPosts: []wire.NiPostsV2{
+		NIPosts: []wire.NIPostV2{
 			{
 				Membership: wire.MerkleProofV2{
 					Nodes: nipost.Membership.Nodes,
@@ -152,7 +152,7 @@ func createMerged(
 		PreviousATXs:   previous,
 		MarriageATX:    &marriage,
 		PositioningATX: positioning,
-		NiPosts: []wire.NiPostsV2{
+		NIPosts: []wire.NIPostV2{
 			{
 				Membership: membership,
 				Challenge:  types.Hash32(niposts[0].PostMetadata.Challenge),
@@ -163,7 +163,7 @@ func createMerged(
 	for i, nipost := range niposts {
 		idx := slices.IndexFunc(previous, func(a types.ATXID) bool { return a == nipost.previous })
 		require.NotEqual(tb, -1, idx)
-		atx.NiPosts[0].Posts = append(atx.NiPosts[0].Posts, wire.SubPostV2{
+		atx.NIPosts[0].Posts = append(atx.NIPosts[0].Posts, wire.SubPostV2{
 			MarriageIndex:       uint32(i),
 			PrevATXIndex:        uint32(idx),
 			MembershipLeafIndex: nipost.Membership.LeafIndex,

activation/handler_test.go

@@ -119,6 +119,7 @@ func toAtx(tb testing.TB, watx *wire.ActivationTxV1) *types.ActivationTx {
 }
 
 type handlerMocks struct {
+	ctrl        *gomock.Controller
 	goldenATXID types.ATXID
 
 	mclock      *MocklayerClock
@@ -184,6 +185,8 @@ func (h *handlerMocks) expectAtxV1(atx *wire.ActivationTxV1, nodeId types.NodeID
 func newTestHandlerMocks(tb testing.TB, golden types.ATXID) handlerMocks {
 	ctrl := gomock.NewController(tb)
 	return handlerMocks{
+		ctrl: ctrl,
+
 		goldenATXID: golden,
 		mclock:      NewMocklayerClock(ctrl),
 		mpub:        pubsubmocks.NewMockPublisher(ctrl),

activation/handler_v1.go

@@ -500,7 +500,7 @@ func (h *HandlerV1) storeAtx(
 		proof     *mwire.MalfeasanceProof
 		malicious bool
 	)
-	if err := h.cdb.WithTx(ctx, func(tx sql.Transaction) error {
+	if err := h.cdb.WithTxImmediate(ctx, func(tx sql.Transaction) error {
 		var err error
 		malicious, err = identities.IsMalicious(tx, atx.SmesherID)
 		if err != nil {

activation/handler_v2.go

@@ -182,13 +182,13 @@ func (h *HandlerV2) syntacticallyValidate(ctx context.Context, atx *wire.Activat
 	}
 
 	if atx.MarriageATX == nil {
-		if len(atx.NiPosts) != 1 {
+		if len(atx.NIPosts) != 1 {
 			return errors.New("solo atx must have one nipost")
 		}
-		if len(atx.NiPosts[0].Posts) != 1 {
+		if len(atx.NIPosts[0].Posts) != 1 {
 			return errors.New("solo atx must have one post")
 		}
-		if atx.NiPosts[0].Posts[0].PrevATXIndex != 0 {
+		if atx.NIPosts[0].Posts[0].PrevATXIndex != 0 {
 			return errors.New("solo atx post must have prevATXIndex 0")
 		}
 	}
@@ -204,7 +204,7 @@ func (h *HandlerV2) syntacticallyValidate(ctx context.Context, atx *wire.Activat
 			return errors.New("initial atx must not have previous atxs")
 		}
 
-		numUnits := atx.NiPosts[0].Posts[0].NumUnits
+		numUnits := atx.NIPosts[0].Posts[0].NumUnits
 		if err := h.nipostValidator.VRFNonceV2(
 			atx.SmesherID, atx.Initial.CommitmentATX, atx.VRFNonce, numUnits,
 		); err != nil {
@@ -309,7 +309,7 @@ func (h *HandlerV2) collectAtxDeps(atx *wire.ActivationTxV2) ([]types.Hash32, []
 	}
 
 	poetRefs := make(map[types.Hash32]struct{})
-	for _, nipost := range atx.NiPosts {
+	for _, nipost := range atx.NIPosts {
 		poetRefs[nipost.Challenge] = struct{}{}
 	}
 
@@ -495,7 +495,7 @@ func (n nipostSizes) sumUp() (units uint32, weight uint64, err error) {
 
 func (h *HandlerV2) verifyIncludedIDsUniqueness(atx *wire.ActivationTxV2) error {
 	seen := make(map[uint32]struct{})
-	for _, niposts := range atx.NiPosts {
+	for _, niposts := range atx.NIPosts {
 		for _, post := range niposts.Posts {
 			if _, ok := seen[post.MarriageIndex]; ok {
 				return fmt.Errorf("ID present twice (duplicated marriage index): %d", post.MarriageIndex)
@@ -540,8 +540,8 @@ func (h *HandlerV2) syntacticallyValidateDeps(
 	}
 
 	// validate previous ATXs
-	nipostSizes := make(nipostSizes, len(atx.NiPosts))
-	for i, niposts := range atx.NiPosts {
+	nipostSizes := make(nipostSizes, len(atx.NIPosts))
+	for i, niposts := range atx.NIPosts {
 		nipostSizes[i] = new(nipostSize)
 		for _, post := range niposts.Posts {
 			if post.MarriageIndex >= uint32(len(equivocationSet)) {
@@ -563,7 +563,7 @@ func (h *HandlerV2) syntacticallyValidateDeps(
 	}
 
 	// validate poet membership proofs
-	for i, niposts := range atx.NiPosts {
+	for i, niposts := range atx.NIPosts {
 		// verify PoET memberships in a single go
 		indexedChallenges := make(map[uint64][]byte)
 
@@ -608,7 +608,7 @@ func (h *HandlerV2) syntacticallyValidateDeps(
 
 	// validate all niposts
 	var smesherCommitment *types.ATXID
-	for _, niposts := range atx.NiPosts {
+	for idx, niposts := range atx.NIPosts {
 		for _, post := range niposts.Posts {
 			id := equivocationSet[post.MarriageIndex]
 			var commitment types.ATXID
@@ -632,24 +632,18 @@ func (h *HandlerV2) syntacticallyValidateDeps(
 				id,
 				commitment,
 				wire.PostFromWireV1(&post.Post),
-				niposts.Challenge[:],
+				niposts.Challenge.Bytes(),
 				post.NumUnits,
 				PostSubset([]byte(h.local)),
 			)
 			invalidIdx := &verifying.ErrInvalidIndex{}
-			if errors.As(err, invalidIdx) {
-				h.logger.Debug(
-					"ATX with invalid post index",
-					zap.Stringer("id", atx.ID()),
-					zap.Int("index", invalidIdx.Index),
-				)
-				// TODO(mafa): finish proof
-				var proof wire.Proof
-				if err := h.malPublisher.Publish(ctx, id, proof); err != nil {
-					return nil, fmt.Errorf("publishing malfeasance proof for invalid post: %w", err)
+			switch {
+			case errors.As(err, invalidIdx):
+				if err := h.publishInvalidPostProof(ctx, atx, id, idx, uint32(invalidIdx.Index)); err != nil {
+					return nil, fmt.Errorf("publishing invalid post proof: %w", err)
 				}
-			}
-			if err != nil {
+				return nil, fmt.Errorf("invalid post for ID %s: %w", id.ShortString(), err)
+			case err != nil:
 				return nil, fmt.Errorf("validating post for ID %s: %w", id.ShortString(), err)
 			}
 			result.ids[id] = idData{
@@ -674,6 +668,45 @@ func (h *HandlerV2) syntacticallyValidateDeps(
 	return &result, nil
 }
 
+func (h *HandlerV2) publishInvalidPostProof(
+	ctx context.Context,
+	atx *wire.ActivationTxV2,
+	nodeID types.NodeID,
+	nipostIndex int,
+	invalidPostIndex uint32,
+) error {
+	initialAtx := atx
+	if initialAtx.Initial == nil {
+		initialID, err := atxs.GetFirstIDByNodeID(h.cdb, nodeID)
+		if err != nil {
+			return fmt.Errorf("fetch initial ATX for ID %s: %w", nodeID.ShortString(), err)
+		}
+
+		// TODO(mafa): implement for v1 initial ATXs: https://github.com/spacemeshos/go-spacemesh/issues/6433
+		initialAtx, err = h.fetchWireAtx(ctx, h.cdb, initialID)
+		if err != nil {
+			return fmt.Errorf("fetch initial ATX blob for ID %s: %w", nodeID.ShortString(), err)
+		}
+	}
+
+	// TODO(mafa): checkpoints need to include all initial ATXs in full to be able to create this malfeasance proof:
+	//
+	// see https://github.com/spacemeshos/go-spacemesh/issues/6436
+	//
+	// TODO(mafa): checkpoints need to include all marriage ATXs in full to be able to create malfeasance proofs
+	// like this one (but also others)
+	//
+	// see https://github.com/spacemeshos/go-spacemesh/issues/6435
+	proof, err := wire.NewInvalidPostProof(h.cdb, atx, initialAtx, nodeID, nipostIndex, invalidPostIndex)
+	if err != nil {
+		return fmt.Errorf("creating invalid post proof: %w", err)
+	}
+	if err := h.malPublisher.Publish(ctx, nodeID, proof); err != nil {
+		return fmt.Errorf("publishing malfeasance proof for invalid post: %w", err)
+	}
+	return nil
+}
+
 func (h *HandlerV2) checkMalicious(ctx context.Context, tx sql.Transaction, atx *activationTx) (bool, error) {
 	malicious, err := malfeasance.IsMalicious(tx, atx.SmesherID)
 	if err != nil {
@@ -717,7 +750,7 @@ func (h *HandlerV2) checkMalicious(ctx context.Context, tx sql.Transaction, atx
 
 func (h *HandlerV2) fetchWireAtx(
 	ctx context.Context,
-	tx sql.Transaction,
+	tx sql.Executor,
 	id types.ATXID,
 ) (*wire.ActivationTxV2, error) {
 	var blob sql.Blob
@@ -808,6 +841,7 @@ func (h *HandlerV2) checkDoubleMerge(ctx context.Context, tx sql.Transaction, at
 		zap.Stringer("smesher_id", atx.SmesherID),
 	)
 
+	// TODO(mafa): finish proof
 	var proof wire.Proof
 	return true, h.malPublisher.Publish(ctx, atx.SmesherID, proof)
 }
@@ -851,7 +885,7 @@ func (h *HandlerV2) checkPrevAtx(ctx context.Context, tx sql.Transaction, atx *a
 
 // Store an ATX in the DB.
 func (h *HandlerV2) storeAtx(ctx context.Context, atx *types.ActivationTx, watx *activationTx) error {
-	if err := h.cdb.WithTx(ctx, func(tx sql.Transaction) error {
+	if err := h.cdb.WithTxImmediate(ctx, func(tx sql.Transaction) error {
 		if len(watx.marriages) != 0 {
 			newMarriageID, err := marriage.NewID(tx)
 			if err != nil {
@@ -927,7 +961,7 @@ func (h *HandlerV2) storeAtx(ctx context.Context, atx *types.ActivationTx, watx
 	atxs.AtxAdded(h.cdb, atx)
 
 	malicious := false
-	err := h.cdb.WithTx(ctx, func(tx sql.Transaction) error {
+	err := h.cdb.WithTxImmediate(ctx, func(tx sql.Transaction) error {
 		// malfeasance check happens after storing the ATX because storing updates the marriage set
 		// that is needed for the malfeasance proof
 		// TODO(mafa): don't store own ATX if it would mark the node as malicious