Fixing #191 : forks resolutionw. remembering semantic validity

examples/src/main/scala/examples/hybrid/HybridNodeViewHolder.scala

@@ -38,6 +38,7 @@ class HybridNodeViewHolder(settings: ScorexSettings, minerSettings: HybridMining
 
   override def preRestart(reason: Throwable, message: Option[Any]): Unit = {
     super.preRestart(reason, message)
+    log.error("HybridNodeViewHolder has been restarted, not a valid situation!")
     reason.printStackTrace()
     System.exit(100) // this actor shouldn't be restarted at all so kill the whole app if that happened
   }

examples/src/main/scala/examples/hybrid/history/HistoryStorage.scala

@@ -1,18 +1,17 @@
 package examples.hybrid.history
 
 import com.google.common.primitives.Longs
-import examples.commons.SimpleBoxTransaction
 import examples.hybrid.blocks._
 import examples.hybrid.mining.{HybridMiningSettings, PosForger}
 import io.iohk.iodb.{ByteArrayWrapper, LSMStore}
 import scorex.core.ModifierId
-import scorex.core.block.Block
-import scorex.core.transaction.box.proposition.PublicKey25519Proposition
+import scorex.core.consensus.{Absent, ModifierSemanticValidity, Unknown}
 import scorex.core.utils.ScorexLogging
 import scorex.crypto.hash.Sha256
 
 import scala.util.Failure
 
+//TODO: why to use IODB if there's no rollback?
 class HistoryStorage(storage: LSMStore,
                      settings: HybridMiningSettings) extends ScorexLogging {
 
@@ -39,8 +38,7 @@ class HistoryStorage(storage: LSMStore,
     modifierById(bestPosId).get.asInstanceOf[PosBlock]
   }
 
-  def modifierById(blockId: ModifierId): Option[HybridBlock with
-    Block[PublicKey25519Proposition, SimpleBoxTransaction]] = {
+  def modifierById(blockId: ModifierId): Option[HybridBlock] = {
     storage.get(ByteArrayWrapper(blockId)).flatMap { bw =>
       val bytes = bw.data
       val mtypeId = bytes.head
@@ -58,7 +56,28 @@ class HistoryStorage(storage: LSMStore,
     }
   }
 
-  def update(b: HybridBlock, diff: Option[(BigInt, Long)], isBest: Boolean) {
+
+  def semanticValidity(id: ModifierId): ModifierSemanticValidity = {
+    modifierById(id).map { b =>
+      storage
+        .get(validityKey(b))
+        .map(_.data.head)
+        .map(ModifierSemanticValidity.restoreFromCode)
+        .getOrElse(Unknown)
+    }.getOrElse(Absent)
+  }
+
+  def updateValidity(b: HybridBlock, status: ModifierSemanticValidity) {
+    val version = ByteArrayWrapper(Sha256(b.id :+ status.code))
+    storage.update(version, Seq(), Seq(validityKey(b) -> ByteArrayWrapper(Array(status. code))))
+  }
+
+  def updateBestChild(parentId: ModifierId, childId: ModifierId): Unit = {
+    val version = ByteArrayWrapper(Sha256(scala.util.Random.nextString(20).getBytes("UTF-8")))
+    storage.update(version, Seq(), Seq(bestChildKey(parentId) -> ByteArrayWrapper(childId)))
+  }
+
+  def update(b: HybridBlock, difficulty: Option[(BigInt, Long)], isBest: Boolean) {
     log.debug(s"Write new best=$isBest block ${b.encodedId}")
     val typeByte = b match {
       case _: PowBlock =>
@@ -70,7 +89,7 @@ class HistoryStorage(storage: LSMStore,
     val blockH: Iterable[(ByteArrayWrapper, ByteArrayWrapper)] =
       Seq(blockHeightKey(b.id) -> ByteArrayWrapper(Longs.toByteArray(parentHeight(b) + 1)))
 
-    val blockDiff: Iterable[(ByteArrayWrapper, ByteArrayWrapper)] = diff.map { d =>
+    val blockDiff: Iterable[(ByteArrayWrapper, ByteArrayWrapper)] = difficulty.map { d =>
       Seq(blockDiffKey(b.id, isPos = false) -> ByteArrayWrapper(d._1.toByteArray),
         blockDiffKey(b.id, isPos = true) -> ByteArrayWrapper(Longs.toByteArray(d._2)))
     }.getOrElse(Seq())
@@ -83,10 +102,16 @@ class HistoryStorage(storage: LSMStore,
       case _ => Seq()
     }
 
+    val childSeq = if(!isGenesis(b) && isBest) Seq(bestChildKey(b.parentId) -> ByteArrayWrapper(b.id)) else Seq()
+
     storage.update(
       ByteArrayWrapper(b.id),
       Seq(),
-      blockDiff ++ blockH ++ bestBlockSeq ++ Seq(ByteArrayWrapper(b.id) -> ByteArrayWrapper(typeByte +: b.bytes)))
+      blockDiff ++
+        blockH ++
+        bestBlockSeq ++
+        childSeq ++
+        Seq(ByteArrayWrapper(b.id) -> ByteArrayWrapper(typeByte +: b.bytes)))
   }
 
   def getPoWDifficulty(idOpt: Option[ModifierId]): BigInt = {
@@ -115,11 +140,21 @@ class HistoryStorage(storage: LSMStore,
     case posBlock: PosBlock => posBlock.parentId
   }
 
+  def bestChildId(block: HybridBlock): Option[ModifierId] =
+    storage.get(bestChildKey(block.id)).map(ModifierId @@ _.data)
+
+
+  private def bestChildKey(blockId: ModifierId): ByteArrayWrapper =
+    ByteArrayWrapper(Sha256("child".getBytes("UTF-8") ++ blockId))
+
+  private def validityKey(b: HybridBlock): ByteArrayWrapper =
+    ByteArrayWrapper(Sha256("validity".getBytes("UTF-8") ++ b.id))
+
   private def blockHeightKey(blockId: ModifierId): ByteArrayWrapper =
-    ByteArrayWrapper(Sha256("height".getBytes ++ blockId))
+    ByteArrayWrapper(Sha256("height".getBytes("UTF-8") ++ blockId))
 
   private def blockDiffKey(blockId: Array[Byte], isPos: Boolean): ByteArrayWrapper =
-    ByteArrayWrapper(Sha256(s"difficulties$isPos".getBytes ++ blockId))
+    ByteArrayWrapper(Sha256(s"difficulties$isPos".getBytes("UTF-8") ++ blockId))
 
   def heightOf(blockId: ModifierId): Option[Long] = storage.get(blockHeightKey(blockId))
     .map(b => Longs.fromByteArray(b.data))

examples/src/main/scala/examples/hybrid/history/HybridHistory.scala

@@ -9,8 +9,8 @@ import examples.hybrid.mining.HybridMiningSettings
 import examples.hybrid.validation.{DifficultyBlockValidator, ParentBlockValidator, SemanticBlockValidator}
 import io.iohk.iodb.LSMStore
 import scorex.core.block.{Block, BlockValidator}
-import scorex.core.consensus.History.{HistoryComparisonResult, ModifierIds, Nonsense, Equal, Younger, Older, ProgressInfo}
-import scorex.core.consensus.{History, ModifierSemanticValidity, Valid, Absent}
+import scorex.core.consensus.History._
+import scorex.core.consensus._
 import scorex.core.settings.ScorexSettings
 import scorex.core.transaction.box.proposition.PublicKey25519Proposition
 import scorex.core.utils.{NetworkTimeProvider, ScorexLogging}
@@ -102,6 +102,7 @@ class HybridHistory(val storage: HistoryStorage,
           val isBestBrother = (bestPosId sameElements powBlock.prevPosId) &&
             (bestPowBlock.brothersCount < powBlock.brothersCount)
 
+          //potentially the best block, if its not a block in a fork containing invalid block
           val isBest: Boolean = storage.height == storage.parentHeight(powBlock) || isBestBrother
 
           val mod: ProgressInfo[HybridBlock] = if (isBest) {
@@ -115,6 +116,7 @@ class HybridHistory(val storage: HistoryStorage,
               //new best brother
               ProgressInfo(Some(powBlock.prevPosId), Seq(bestPowBlock), Some(powBlock), Seq())
             } else {
+              //we're switching to a better chain, if it does not contain an invalid block
               bestForkChanges(powBlock)
             }
           } else {
@@ -194,17 +196,31 @@ class HybridHistory(val storage: HistoryStorage,
 
     val rollbackPoint = newSuffix.headOption
 
-    // TODO: fixme, What should we do if `oldSuffix` is empty?
-    @SuppressWarnings(Array("org.wartremover.warts.TraversableOps"))
-    val throwBlocks = oldSuffix.tail.map(id => modifierById(id).get)
-    // TODO: fixme, What should we do if `newSuffix` is empty?
-    @SuppressWarnings(Array("org.wartremover.warts.TraversableOps"))
-    val applyBlocks = newSuffix.tail.map(id => modifierById(id).get) ++ Seq(block)
-    require(applyBlocks.nonEmpty)
-    require(throwBlocks.nonEmpty)
+    val newSuffixValid = !newSuffix.drop(1).map(storage.semanticValidity).contains(Invalid)
+
+    if(newSuffixValid) {
+
+      //update best links
+      newSuffix.sliding(2, 1).foreach{p =>
+        storage.updateBestChild(p(0), p(1))
+      }
+
+      // TODO: fixme, What should we do if `oldSuffix` is empty?
+      @SuppressWarnings(Array("org.wartremover.warts.TraversableOps"))
+      val throwBlocks = oldSuffix.tail.map(id => modifierById(id).get)
+      // TODO: fixme, What should we do if `newSuffix` is empty?
+      @SuppressWarnings(Array("org.wartremover.warts.TraversableOps"))
+      val applyBlocks = newSuffix.tail.map(id => modifierById(id).get) ++ Seq(block)
+      require(applyBlocks.nonEmpty)
+      require(throwBlocks.nonEmpty)
+      require(storage.bestChildId(modifierById(rollbackPoint.get).get).get sameElements applyBlocks.headOption.get.id)
 
-    //TODO should be applyBlocks here
-    ProgressInfo[HybridBlock](rollbackPoint, throwBlocks, applyBlocks.headOption, Seq())
+      //TODO should be applyBlocks here
+      ProgressInfo[HybridBlock](rollbackPoint, throwBlocks, applyBlocks.headOption, Seq())
+    } else {
+      log.info(s"Orphaned block $block from invalid suffix")
+      ProgressInfo(None, Seq(), None, Seq())
+    }
   }
 
   private def calcDifficultiesForNewBlock(posBlock: PosBlock): (BigInt, Long) = {
@@ -421,7 +437,7 @@ class HybridHistory(val storage: HistoryStorage,
     val winnerChain = chainBack(forkBlock, in, limit).get.map(_._2)
     val i = loserChain.indexWhere { id =>
       winnerChain.headOption match {
-        case None                  => false
+        case None => false
         case Some(winnerChainHead) => id sameElements winnerChainHead
       }
     }
@@ -450,17 +466,24 @@ class HybridHistory(val storage: HistoryStorage,
     chainBack(storage.bestPosBlock, isGenesis).get.map(_._2).map(Base58.encode).mkString(",")
   }
 
-  //todo: real impl
   override def reportSemanticValidity(modifier: HybridBlock,
                                       valid: Boolean,
                                       lastApplied: ModifierId): (HybridHistory, ProgressInfo[HybridBlock]) = {
-    this -> ProgressInfo(None, Seq(), None, Seq())
-  }
+    val v = if (valid) Valid else Invalid
+    storage.updateValidity(modifier, v)
 
-  //todo: real impl
-  override def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = {
-    modifierById(modifierId).map(_ => Valid).getOrElse(Absent)
+    val h = new HybridHistory(storage, settings, validators, statsLogger, timeProvider)
+
+    if (valid) {
+      val p = ProgressInfo(None, Seq(), storage.bestChildId(modifier).flatMap(storage.modifierById), Seq())
+      h -> p
+    } else {
+      h -> ProgressInfo(None, Seq(), None, Seq())
+    }
   }
+
+  override def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity =
+    storage.semanticValidity(modifierId)
 }
 
 

examples/src/main/scala/examples/hybrid/state/HBoxStoredState.scala

@@ -62,7 +62,10 @@ case class HBoxStoredState(store: LSMStore, override val version: VersionTag) ex
         closedBox(psb.generatorBox.id).get
         psb.transactions.foreach(tx => validate(tx).get)
     }
-  }.recoverWith{case t => log.warn(s"Not valid modifier ${mod.encodedId}", t); Failure(t)}
+  }.recoverWith{case t =>
+    log.warn(s"Not valid modifier ${mod.encodedId}", t)
+    Failure(t)
+  }
 
   override def applyChanges(changes: BoxStateChanges[PublicKey25519Proposition, PublicKey25519NoncedBox],
                             newVersion: VersionTag): Try[HBoxStoredState] = Try {

examples/src/test/scala/hybrid/history/HybridHistorySpecification.scala

@@ -113,6 +113,4 @@ class HybridHistorySpecification extends PropSpec
 
     compareAndCheck(history, betterForkSyncInfo) shouldBe Older
   }
-
-
 }

src/main/scala/scorex/core/NodeViewHolder.scala

@@ -197,7 +197,7 @@ trait NodeViewHolder[P <: Proposition, TX <: Transaction[P], PMOD <: PersistentN
 
   /*
 
-    Assume history knew following blocktree:
+    Assume that history knows the following blocktree:
 
            G
           / \
@@ -232,12 +232,7 @@ trait NodeViewHolder[P <: Proposition, TX <: Transaction[P], PMOD <: PersistentN
     val stateToApplyTry: Try[MS] = if (progressInfo.chainSwitchingNeeded) {
       Try {
         val branchingPoint = VersionTag @@ progressInfo.branchPoint.get
-
-        if (!state.version.sameElements(branchingPoint)) {
-          state.rollbackTo(branchingPoint).map { rs =>
-            rs
-          }
-        } else Success(state)
+        if (!state.version.sameElements(branchingPoint)) state.rollbackTo(branchingPoint) else Success(state)
       }.flatten
     } else Success(state)
 

src/main/scala/scorex/core/consensus/ModifierSemanticValidity.scala

@@ -1,7 +1,30 @@
 package scorex.core.consensus
 
-sealed trait ModifierSemanticValidity
-case object Absent extends ModifierSemanticValidity
-case object Unknown extends ModifierSemanticValidity
-case object Valid extends ModifierSemanticValidity
-case object Invalid extends ModifierSemanticValidity
+sealed trait ModifierSemanticValidity {
+  val code: Byte
+}
+
+object ModifierSemanticValidity {
+  def restoreFromCode(code: Byte): ModifierSemanticValidity = code match {
+    case b: Byte if b == Absent.code => Absent
+    case b: Byte if b == Unknown.code => Unknown
+    case b: Byte if b == Valid.code => Valid
+    case b: Byte if b == Invalid.code => Invalid
+  }
+}
+
+case object Absent extends ModifierSemanticValidity {
+  override val code = 0
+}
+
+case object Unknown extends ModifierSemanticValidity {
+  override val code = 1
+}
+
+case object Valid extends ModifierSemanticValidity {
+  override val code = 2
+}
+
+case object Invalid extends ModifierSemanticValidity {
+  override val code = 3
+}