Merge branch '1.5.x' into 1.5-release

build.sbt

@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
+import com.typesafe.tools.mima.core._
+
 enablePlugins(SiteScaladocPlugin)
 
 lazy val commonSettings = Seq(
@@ -61,10 +63,21 @@ lazy val firrtlSettings = Seq(
 )
 
 lazy val mimaSettings = Seq(
-  mimaPreviousArtifacts := Set("edu.berkeley.cs" %% "firrtl" % "1.5.0")
+  mimaPreviousArtifacts := Set("edu.berkeley.cs" %% "firrtl" % "1.5.4"),
+  mimaBinaryIssueFilters ++= Seq(
+    // These are real issues but users should not be extending things in the firrtl.antlr package
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLListener.enterVersion"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLListener.exitVersion"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLListener.enterSemver"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLListener.exitSemver"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLVisitor.visitVersion"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("firrtl.antlr.FIRRTLVisitor.visitSemver"),
+  )
 )
 
 lazy val protobufSettings = Seq(
+  // The parentheses around the version help avoid version ambiguity in release scripts
+  ProtobufConfig / version := ("3.18.2"), // CVE-2021-22569
   ProtobufConfig / sourceDirectory := baseDirectory.value / "src" / "main" / "proto",
   ProtobufConfig / protobufRunProtoc := (args => com.github.os72.protocjar.Protoc.runProtoc("-v351" +: args.toArray))
 )

src/main/antlr4/FIRRTL.g4

@@ -29,7 +29,16 @@ import firrtl.LexerHelper;
 
 // Does there have to be at least one module?
 circuit
-  : 'circuit' id ':' info? INDENT module* DEDENT EOF
+  : version? NEWLINE* 'circuit' id ':' info? INDENT module* DEDENT EOF
+  ;
+
+version
+  : 'FIRRTL' 'version' semver NEWLINE
+  ;
+
+// Due to lexer problems, something like 1.1.0 is lexed as DoubleLit '.' UnsignedInt
+semver
+  : DoubleLit '.' UnsignedInt
   ;
 
 module
@@ -272,6 +281,7 @@ keywordAsId
   | 'assert'
   | 'assume'
   | 'cover'
+  | 'version'
   ;
 
 // Parentheses are added as part of name because semantics require no space between primop and open parentheses

src/main/scala/firrtl/Parser.scala

@@ -10,13 +10,16 @@ import firrtl.parser.Listener
 import firrtl.Utils.time
 import firrtl.antlr.{FIRRTLParser, _}
 
+import scala.util.control.NonFatal
+
 class ParserException(message: String) extends FirrtlUserException(message)
 
 case class ParameterNotSpecifiedException(message: String) extends ParserException(message)
 case class ParameterRedefinedException(message: String) extends ParserException(message)
 case class InvalidStringLitException(message: String) extends ParserException(message)
 case class InvalidEscapeCharException(message: String) extends ParserException(message)
 case class SyntaxErrorsException(message: String) extends ParserException(message)
+case class UnsupportedVersionException(message: String) extends ParserException(message)
 
 object Parser extends LazyLogging {
 
@@ -41,12 +44,25 @@ object Parser extends LazyLogging {
       parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
       parser.addParseListener(listener)
 
-      // Concrete Syntax Tree
-      parser.circuit
+      // Syntax errors may violate assumptions in the Listener and Visitor.
+      // We need to handle these errors gracefully.
+      val throwable =
+        try {
+          parser.circuit
+          None
+        } catch {
+          case e: ParserException => throw e
+          case NonFatal(e) => Some(e)
+        }
 
       val numSyntaxErrors = parser.getNumberOfSyntaxErrors
       if (numSyntaxErrors > 0) throw new SyntaxErrorsException(s"$numSyntaxErrors syntax error(s) detected")
 
+      // Note that this should never happen because any throwables caught should be due to syntax
+      // errors that are reported above. This is just to ensure that we don't accidentally mask any
+      // bugs in the Parser, Listener, or Visitor.
+      if (throwable.nonEmpty) Utils.throwInternalError(exception = throwable)
+
       listener.getCircuit
     }
 

src/main/scala/firrtl/Utils.scala

@@ -165,7 +165,7 @@ object Utils extends LazyLogging {
     * @param message - possible string to emit,
     * @param exception - possible exception triggering the error.
     */
-  def throwInternalError(message: String = "", exception: Option[Exception] = None) = {
+  def throwInternalError(message: String = "", exception: Option[Throwable] = None) = {
     // We'll get the first exception in the chain, keeping it intact.
     val first = true
     val throwable = getThrowable(exception, true)

src/main/scala/firrtl/ir/Serializer.scala

@@ -6,10 +6,19 @@ import firrtl.Utils
 import firrtl.backends.experimental.smt.random.DefRandom
 import firrtl.constraint.Constraint
 
+case class Version(major: Int, minor: Int, patch: Int) {
+  def serialize: String = s"$major.$minor.$patch"
+  def incompatible(that: Version): Boolean =
+    this.major > that.major || (this.major == that.major && this.minor > that.minor)
+}
+
 object Serializer {
   val NewLine = '\n'
   val Indent = "  "
 
+  // The version supported by the serializer.
+  val version = Version(1, 1, 0)
+
   /** Converts a `FirrtlNode` into its string representation with
     * default indentation.
     */
@@ -24,21 +33,44 @@ object Serializer {
       case n: Info        => s(n)(builder, indent)
       case n: StringLit   => s(n)(builder, indent)
       case n: Expression  => s(n)(builder, indent)
-      case n: Statement   => s(n)(builder, indent)
+      case n: Statement   => builder ++= lazily(n, indent).mkString
       case n: Width       => s(n)(builder, indent)
       case n: Orientation => s(n)(builder, indent)
       case n: Field       => s(n)(builder, indent)
       case n: Type        => s(n)(builder, indent)
       case n: Direction   => s(n)(builder, indent)
       case n: Port        => s(n)(builder, indent)
       case n: Param       => s(n)(builder, indent)
-      case n: DefModule   => s(n)(builder, indent)
-      case n: Circuit     => s(n)(builder, indent)
+      case n: DefModule   => builder ++= lazily(n, indent).mkString
+      case n: Circuit     => builder ++= lazily(n, indent).mkString
       case other => builder ++= other.serialize // Handle user-defined nodes
     }
     builder.toString()
   }
 
+  /** Converts a `FirrtlNode` to an Iterable of Strings
+    *
+    * The Strings in the Iterable can be concatenated to give the String representation of the
+    * `FirrtlNode`. This is useful for buffered emission, especially for large Circuits that
+    * encroach on the JVM limit on String size (2 GiB).
+    */
+  def lazily(node: FirrtlNode): Iterable[String] = lazily(node, 0)
+
+  /** Converts a `FirrtlNode` to an Iterable of Strings
+    *
+    * The Strings in the Iterable can be concatenated to give the String representation of the
+    * `FirrtlNode`. This is useful for buffered emission, especially for large Circuits that
+    * encroach on the JVM limit on String size (2 GiB).
+    */
+  def lazily(node: FirrtlNode, indent: Int): Iterable[String] = new Iterable[String] {
+    def iterator = node match {
+      case n: Statement => sIt(n)(indent)
+      case n: DefModule => sIt(n)(indent)
+      case n: Circuit   => sIt(n)(indent)
+      case other => Iterator(serialize(other, indent))
+    }
+  }
+
   /** Converts a `Constraint` into its string representation. */
   def serialize(con: Constraint): String = {
     val builder = new StringBuilder()
@@ -92,24 +124,134 @@ object Serializer {
     case other                  => b ++= other.serialize // Handle user-defined nodes
   }
 
+  // Helper for some not-real Statements that only exist for Serialization
+  private abstract class PseudoStatement extends Statement {
+    def foreachExpr(f:   Expression => Unit):       Unit = ???
+    def foreachInfo(f:   Info => Unit):             Unit = ???
+    def foreachStmt(f:   Statement => Unit):        Unit = ???
+    def foreachString(f: String => Unit):           Unit = ???
+    def foreachType(f:   Type => Unit):             Unit = ???
+    def mapExpr(f:       Expression => Expression): Statement = ???
+    def mapInfo(f:       Info => Info):             Statement = ???
+    def mapStmt(f:       Statement => Statement):   Statement = ???
+    def mapString(f:     String => String):         Statement = ???
+    def mapType(f:       Type => Type):             Statement = ???
+    def serialize: String = ???
+  }
+
+  // To treat Statments as Iterable, we need to flatten out when scoping
+  private case class WhenBegin(info: Info, pred: Expression) extends PseudoStatement
+  private case object AltBegin extends PseudoStatement
+  private case object WhenEnd extends PseudoStatement
+
+  // This does not extend Iterator[Statement] because
+  //  1. It is extended by StmtsSerializer which extends Iterator[String]
+  //  2. Flattening out whens introduces fake Statements needed for [un]indenting
+  private abstract class FlatStmtsIterator(stmts: Iterable[Statement]) {
+    private var underlying: Iterator[Statement] = stmts.iterator
+
+    protected def hasNextStmt = underlying.hasNext
+
+    protected def nextStmt(): Statement = {
+      var next: Statement = null
+      while (next == null && hasNextStmt) {
+        val head = underlying
+        head.next() match {
+          case b: Block if b.stmts.isEmpty =>
+            next = EmptyStmt
+          case b: Block =>
+            val first = b.stmts.iterator
+            val last = underlying
+            underlying = first ++ last
+          case Conditionally(info, pred, conseq, alt) =>
+            val begin = WhenBegin(info, pred)
+            val stmts = if (alt == EmptyStmt) {
+              Iterator(begin, conseq, WhenEnd)
+            } else {
+              Iterator(begin, conseq, AltBegin, alt, WhenEnd)
+            }
+            val last = underlying
+            underlying = stmts ++ last
+          case other =>
+            next = other
+        }
+      }
+      next
+    }
+  }
+
+  // Extend FlatStmtsIterator directly (rather than wrapping a FlatStmtsIterator object) to reduce
+  // the boxing overhead
+  private class StmtsSerializer(stmts: Iterable[Statement], initialIndent: Int)
+      extends FlatStmtsIterator(stmts)
+      with Iterator[String] {
+
+    private def bufferSize = 2048
+
+    // We could initialze the StringBuilder size, but this is bad for small modules which may not
+    // even reach the bufferSize.
+    private implicit val b = new StringBuilder
+
+    // The flattening of Whens into WhenBegin and friends requires us to keep track of the
+    // indention level
+    private implicit var indent: Int = initialIndent
+
+    def hasNext: Boolean = this.hasNextStmt
+
+    def next(): String = {
+      def consumeStmt(stmt: Statement): Unit = {
+        stmt match {
+          case wb: WhenBegin =>
+            doIndent()
+            b ++= "when "; s(wb.pred); b ++= " :"; s(wb.info)
+            indent += 1
+          case AltBegin =>
+            indent -= 1
+            doIndent()
+            b ++= "else :"
+            indent += 1
+          case WhenEnd =>
+            indent -= 1
+          case other =>
+            doIndent()
+            s(other)
+        }
+        if (this.hasNext && stmt != WhenEnd) {
+          newLineNoIndent()
+        }
+      }
+      b.clear()
+      // There must always be at least 1 Statement because we're nonEmpty
+      var stmt: Statement = nextStmt()
+      while (stmt != null && b.size < bufferSize) {
+        consumeStmt(stmt)
+        stmt = nextStmt()
+      }
+      if (stmt != null) {
+        consumeStmt(stmt)
+      }
+      b.toString
+    }
+  }
+
+  private def sIt(node: Statement)(implicit indent: Int): Iterator[String] = node match {
+    case b: Block =>
+      if (b.stmts.isEmpty) sIt(EmptyStmt)
+      else new StmtsSerializer(b.stmts, indent)
+    case cond: Conditionally => new StmtsSerializer(Seq(cond), indent)
+    case other =>
+      implicit val b = new StringBuilder
+      doIndent()
+      s(other)
+      Iterator(b.toString)
+  }
+
   private def s(node: Statement)(implicit b: StringBuilder, indent: Int): Unit = node match {
     case DefNode(info, name, value) => b ++= "node "; b ++= name; b ++= " = "; s(value); s(info)
     case Connect(info, loc, expr)   => s(loc); b ++= " <= "; s(expr); s(info)
-    case Conditionally(info, pred, conseq, alt) =>
-      b ++= "when "; s(pred); b ++= " :"; s(info)
-      newLineAndIndent(1); s(conseq)(b, indent + 1)
-      if (alt != EmptyStmt) {
-        newLineAndIndent(); b ++= "else :"
-        newLineAndIndent(1); s(alt)(b, indent + 1)
-      }
-    case EmptyStmt    => b ++= "skip"
-    case Block(Seq()) => b ++= "skip"
-    case Block(stmts) =>
-      val it = stmts.iterator
-      while (it.hasNext) {
-        s(it.next())
-        if (it.hasNext) newLineAndIndent()
-      }
+    case c: Conditionally => b ++= sIt(c).mkString
+    case EmptyStmt => b ++= "skip"
+    case bb: Block => b ++= sIt(bb).mkString
     case stop @ Stop(info, ret, clk, en) =>
       b ++= "stop("; s(clk); b ++= ", "; s(en); b ++= ", "; b ++= ret.toString; b += ')'
       sStmtName(stop.name); s(info)
@@ -238,28 +380,43 @@ object Serializer {
     case other => b ++= other.serialize // Handle user-defined nodes
   }
 
-  private def s(node: DefModule)(implicit b: StringBuilder, indent: Int): Unit = node match {
+  private def sIt(node: DefModule)(implicit indent: Int): Iterator[String] = node match {
     case Module(info, name, ports, body) =>
-      doIndent(0); b ++= "module "; b ++= name; b ++= " :"; s(info)
-      ports.foreach { p => newLineAndIndent(1); s(p) }
-      newLineNoIndent() // add a new line between port declaration and body
-      newLineAndIndent(1); s(body)(b, indent + 1)
+      val start = {
+        implicit val b = new StringBuilder
+        doIndent(0); b ++= "module "; b ++= name; b ++= " :"; s(info)
+        ports.foreach { p => newLineAndIndent(1); s(p) }
+        newLineNoIndent() // add a blank line between port declaration and body
+        newLineNoIndent() // newline for body, sIt will indent
+        b.toString
+      }
+      Iterator(start) ++ sIt(body)(indent + 1)
     case ExtModule(info, name, ports, defname, params) =>
+      implicit val b = new StringBuilder
       doIndent(0); b ++= "extmodule "; b ++= name; b ++= " :"; s(info)
       ports.foreach { p => newLineAndIndent(1); s(p) }
       newLineAndIndent(1); b ++= "defname = "; b ++= defname
       params.foreach { p => newLineAndIndent(1); s(p) }
-    case other => doIndent(0); b ++= other.serialize // Handle user-defined nodes
+      Iterator(b.toString)
+    case other =>
+      Iterator(Indent * indent, other.serialize) // Handle user-defined nodes
   }
 
-  private def s(node: Circuit)(implicit b: StringBuilder, indent: Int): Unit = node match {
+  private def sIt(node: Circuit)(implicit indent: Int): Iterator[String] = node match {
     case Circuit(info, modules, main) =>
-      b ++= "circuit "; b ++= main; b ++= " :"; s(info)
-      if (modules.nonEmpty) {
-        newLineNoIndent(); s(modules.head)(b, indent + 1)
-        modules.drop(1).foreach { m => newLineNoIndent(); newLineNoIndent(); s(m)(b, indent + 1) }
+      val prelude = {
+        implicit val b = new StringBuilder // Scope this so we don't accidentally pass it anywhere
+        b ++= s"FIRRTL version ${version.serialize}\n"
+        b ++= "circuit "; b ++= main; b ++= " :"; s(info)
+        b.toString
       }
-      newLineNoIndent()
+      Iterator(prelude) ++
+        modules.iterator.zipWithIndex.flatMap {
+          case (m, i) =>
+            val newline = Iterator(if (i == 0) s"$NewLine" else s"${NewLine}${NewLine}")
+            newline ++ sIt(m)(indent + 1)
+        } ++
+        Iterator(s"$NewLine")
   }
 
   // serialize constraints
@@ -281,7 +438,7 @@ object Serializer {
   private def newLineNoIndent()(implicit b: StringBuilder): Unit = b += NewLine
 
   /** create indent, inc allows for a temporary increment */
-  private def doIndent(inc: Int)(implicit b: StringBuilder, indent: Int): Unit = {
+  private def doIndent(inc: Int = 0)(implicit b: StringBuilder, indent: Int): Unit = {
     (0 until (indent + inc)).foreach { _ => b ++= Indent }
   }