Merge pull request #14506 from ethereum/extracted-natspec-json-tests

Replace Boost-based Natspec test case with one derived from `SyntaxTest`

scripts/error_codes.py

@@ -171,6 +171,7 @@ def print_ids_per_file(ids, id_to_file_names, top_dir):
 
 def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False):
     test_sub_dirs = [
+        path.join("test", "libsolidity", "natspecJSON"),
         path.join("test", "libsolidity", "smtCheckerTests"),
         path.join("test", "libsolidity", "syntaxTests"),
         path.join("test", "libyul", "yulSyntaxTests")

test/CMakeLists.txt

@@ -84,6 +84,8 @@ set(libsolidity_sources
     libsolidity/Metadata.cpp
     libsolidity/MemoryGuardTest.cpp
     libsolidity/MemoryGuardTest.h
+    libsolidity/NatspecJSONTest.cpp
+    libsolidity/NatspecJSONTest.h
     libsolidity/SemanticTest.cpp
     libsolidity/SemanticTest.h
     libsolidity/SemVerMatcher.cpp
@@ -95,7 +97,6 @@ set(libsolidity_sources
     libsolidity/SolidityExecutionFramework.h
     libsolidity/SolidityExpressionCompiler.cpp
     libsolidity/SolidityNameAndTypeResolution.cpp
-    libsolidity/SolidityNatspecJSON.cpp
     libsolidity/SolidityOptimizer.cpp
     libsolidity/SolidityParser.cpp
     libsolidity/SolidityTypes.cpp

test/InteractiveTests.h

@@ -24,6 +24,7 @@
 #include <test/libsolidity/ASTPropertyTest.h>
 #include <test/libsolidity/GasTest.h>
 #include <test/libsolidity/MemoryGuardTest.h>
+#include <test/libsolidity/NatspecJSONTest.h>
 #include <test/libsolidity/SyntaxTest.h>
 #include <test/libsolidity/SemanticTest.h>
 #include <test/libsolidity/SMTCheckerTest.h>
@@ -74,6 +75,7 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Semantic",               "libsolidity", "semanticTests",         false, true,  &SemanticTest::create},
 	{"JSON AST",               "libsolidity", "ASTJSON",               false, false, &ASTJSONTest::create},
 	{"JSON ABI",               "libsolidity", "ABIJson",               false, false, &ABIJsonTest::create},
+	{"JSON Natspec",           "libsolidity", "natspecJSON",           false, false, &NatspecJSONTest::create},
 	{"SMT Checker",            "libsolidity", "smtCheckerTests",       true,  false, &SMTCheckerTest::create},
 	{"Gas Estimates",          "libsolidity", "gasTests",              false, false, &GasTest::create},
 	{"Memory Guard",           "libsolidity", "memoryGuardTests",      false, false, &MemoryGuardTest::create},

test/libsolidity/NatspecJSONTest.cpp

@@ -0,0 +1,210 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/**
+ * Unit tests for the solidity compiler ABI JSON Interface output.
+ */
+
+#include <test/libsolidity/NatspecJSONTest.h>
+
+#include <libsolutil/CommonIO.h>
+#include <libsolutil/StringUtils.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <fmt/format.h>
+
+#include <vector>
+
+using namespace std;
+using namespace solidity::frontend::test;
+using namespace solidity::util;
+
+ostream& solidity::frontend::test::operator<<(ostream& _output, NatspecJSONKind _kind)
+{
+	switch (_kind) {
+	case NatspecJSONKind::Devdoc: _output << "devdoc"; break;
+	case NatspecJSONKind::Userdoc: _output << "userdoc"; break;
+	}
+	return _output;
+}
+
+unique_ptr<TestCase> NatspecJSONTest::create(Config const& _config)
+{
+	return make_unique<NatspecJSONTest>(_config.filename, _config.evmVersion);
+}
+
+void NatspecJSONTest::parseCustomExpectations(istream& _stream)
+{
+	soltestAssert(m_expectedNatspecJSON.empty());
+
+	// We expect a series of expectations in the following format:
+	//
+	//     // <qualified contract name> <devdoc|userdoc>
+	//     // <json>
+
+	string line;
+	while (getline(_stream, line))
+	{
+		string_view strippedLine = expectLinePrefix(line);
+		if (strippedLine.empty())
+			continue;
+
+		auto [contractName, kind] = parseExpectationHeader(strippedLine);
+
+		string rawJSON = extractExpectationJSON(_stream);
+		string jsonErrors;
+		Json::Value parsedJSON;
+		bool jsonParsingSuccessful = jsonParseStrict(rawJSON, parsedJSON, &jsonErrors);
+		if (!jsonParsingSuccessful)
+			BOOST_THROW_EXCEPTION(runtime_error(fmt::format(
+				"Malformed JSON in {} expectation for contract {}.\n"
+				"Note that JSON expectations must be pretty-printed to be split correctly. "
+				"The object is assumed to and at the first unindented closing brace.\n"
+				"{}",
+				toString(kind),
+				contractName,
+				rawJSON
+			)));
+
+		m_expectedNatspecJSON[string(contractName)][kind] = parsedJSON;
+	}
+}
+
+bool NatspecJSONTest::expectationsMatch()
+{
+	// NOTE: Comparing pretty printed Json::Values to avoid using its operator==, which fails to
+	// compare equal numbers as equal. For example, for 'version' field the value is sometimes int,
+	// sometimes uint and they compare as different even when both are 1.
+	return
+		SyntaxTest::expectationsMatch() &&
+		prettyPrinted(obtainedNatspec()) == prettyPrinted(m_expectedNatspecJSON);
+}
+
+void NatspecJSONTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const
+{
+	SyntaxTest::printExpectedResult(_stream, _linePrefix, _formatted);
+	if (!m_expectedNatspecJSON.empty())
+	{
+		_stream << _linePrefix << "----" << endl;
+		printIndented(_stream, formatNatspecExpectations(m_expectedNatspecJSON), _linePrefix);
+	}
+}
+
+void NatspecJSONTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const
+{
+	SyntaxTest::printObtainedResult(_stream, _linePrefix, _formatted);
+
+	NatspecMap natspecJSON = obtainedNatspec();
+	if (!natspecJSON.empty())
+	{
+		_stream << _linePrefix << "----" << endl;
+		// TODO: Diff both versions and highlight differences.
+		// We should have a helper for doing that in newly defined test cases without much effort.
+		printIndented(_stream, formatNatspecExpectations(natspecJSON), _linePrefix);
+	}
+}
+
+tuple<string_view, NatspecJSONKind> NatspecJSONTest::parseExpectationHeader(string_view _line)
+{
+	for (NatspecJSONKind kind: {NatspecJSONKind::Devdoc, NatspecJSONKind::Userdoc})
+	{
+		string kindSuffix = " " + toString(kind);
+		if (boost::algorithm::ends_with(_line, kindSuffix))
+			return {_line.substr(0, _line.size() - kindSuffix.size()), kind};
+	}
+
+	BOOST_THROW_EXCEPTION(runtime_error(
+		"Natspec kind (devdoc/userdoc) not present in the expectation: "s.append(_line)
+	));
+}
+
+string NatspecJSONTest::extractExpectationJSON(istream& _stream)
+{
+	string rawJSON;
+	string line;
+	while (getline(_stream, line))
+	{
+		string_view strippedLine = expectLinePrefix(line);
+		rawJSON += strippedLine;
+		rawJSON += "\n";
+
+		if (boost::algorithm::starts_with(strippedLine, "}"))
+			break;
+	}
+
+	return rawJSON;
+}
+
+string_view NatspecJSONTest::expectLinePrefix(string_view _line)
+{
+	size_t startPosition = 0;
+	if (!boost::algorithm::starts_with(_line, "//"))
+		BOOST_THROW_EXCEPTION(runtime_error(
+			"Expectation line is not a comment: "s.append(_line)
+		));
+
+	startPosition += 2;
+	if (startPosition < _line.size() && _line[startPosition] == ' ')
+		++startPosition;
+
+	return _line.substr(startPosition, _line.size() - startPosition);
+}
+
+string NatspecJSONTest::formatNatspecExpectations(NatspecMap const& _expectations) const
+{
+	string output;
+	bool first = true;
+	// NOTE: Not sorting explicitly because CompilerStack seems to put contracts roughly in the
+	// order in which they appear in the source, which is much better than alphabetical order.
+	for (auto const& [contractName, expectationsForAllKinds]: _expectations)
+		for (auto const& [jsonKind, natspecJSON]: expectationsForAllKinds)
+		{
+			if (!first)
+				output += "\n\n";
+			first = false;
+
+			output += contractName + " " + toString(jsonKind) + "\n";
+			output += jsonPrint(natspecJSON, {JsonFormat::Pretty, 4});
+		}
+
+	return output;
+}
+
+NatspecMap NatspecJSONTest::obtainedNatspec() const
+{
+	if (compiler().state() < CompilerStack::AnalysisSuccessful)
+		return {};
+
+	NatspecMap result;
+	for (string contractName: compiler().contractNames())
+	{
+		result[contractName][NatspecJSONKind::Devdoc]  = compiler().natspecDev(contractName);
+		result[contractName][NatspecJSONKind::Userdoc] = compiler().natspecUser(contractName);
+	}
+
+	return result;
+}
+
+SerializedNatspecMap NatspecJSONTest::prettyPrinted(NatspecMap const& _expectations) const
+{
+	SerializedNatspecMap result;
+	for (auto const& [contractName, expectationsForAllKinds]: _expectations)
+		for (auto const& [jsonKind, natspecJSON]: expectationsForAllKinds)
+			result[contractName][jsonKind] = jsonPrint(natspecJSON, {JsonFormat::Pretty, 4});
+
+	return result;
+}

test/libsolidity/NatspecJSONTest.h

@@ -0,0 +1,82 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+/**
+ * Unit tests for the Natspec userdoc and devdoc JSON output.
+ */
+
+#pragma once
+
+#include <test/libsolidity/SyntaxTest.h>
+
+#include <libsolutil/JSON.h>
+
+#include <istream>
+#include <map>
+#include <memory>
+#include <ostream>
+#include <string>
+#include <string_view>
+#include <tuple>
+
+namespace solidity::frontend::test
+{
+
+enum class NatspecJSONKind
+{
+	Devdoc,
+	Userdoc,
+};
+
+std::ostream& operator<<(std::ostream& _output, NatspecJSONKind _kind);
+
+using NatspecMap = std::map<std::string, std::map<NatspecJSONKind, Json::Value>>;
+using SerializedNatspecMap = std::map<std::string, std::map<NatspecJSONKind, std::string>>;
+
+class NatspecJSONTest: public SyntaxTest
+{
+public:
+
+	static std::unique_ptr<TestCase> create(Config const& _config);
+
+	NatspecJSONTest(std::string const& _filename, langutil::EVMVersion _evmVersion):
+		SyntaxTest(
+			_filename,
+			_evmVersion,
+			langutil::Error::Severity::Error // _minSeverity
+		)
+	{}
+
+protected:
+	void parseCustomExpectations(std::istream& _stream) override;
+	bool expectationsMatch() override;
+	void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const override;
+	void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const override;
+
+	NatspecMap m_expectedNatspecJSON;
+
+private:
+	static std::tuple<std::string_view, NatspecJSONKind> parseExpectationHeader(std::string_view _line);
+	static std::string extractExpectationJSON(std::istream& _stream);
+	static std::string_view expectLinePrefix(std::string_view _line);
+
+	std::string formatNatspecExpectations(NatspecMap const& _expectations) const;
+	SerializedNatspecMap prettyPrinted(NatspecMap const& _expectations) const;
+	NatspecMap obtainedNatspec() const;
+};
+
+}