From db86acf036e7ec4a2cfa58a0aff232c72e769929 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 16 Jan 2025 07:52:25 -0800 Subject: [PATCH] Migrate all upb plugins over to the C++ protoc plugin framework We need the minitable plugin to use the C++ plugin framework because it's soon going to live in the same binary as the Rust code generator. The other plugins we might as well move just for consistency, and this allows us to delete the `upb::Plugin` class as well. PiperOrigin-RevId: 716234662 --- benchmarks/BUILD | 2 +- cmake/upb_generators.cmake | 1 + editions/BUILD | 3 +- src/google/protobuf/compiler/BUILD.bazel | 9 ++ upb/message/BUILD | 6 +- upb/mini_descriptor/BUILD | 2 +- upb/test/BUILD | 2 +- upb/util/BUILD | 4 +- upb_generator/BUILD | 5 + upb_generator/c/BUILD | 5 +- upb_generator/c/generator.cc | 93 ++++++++++----- upb_generator/common/BUILD | 1 - upb_generator/file_layout.h | 4 + upb_generator/minitable/BUILD | 11 +- upb_generator/minitable/generator.cc | 27 +++-- upb_generator/minitable/generator.h | 4 +- upb_generator/minitable/main.cc | 98 ++++++++++------ upb_generator/plugin.cc | 58 ++++++++++ upb_generator/plugin.h | 139 +---------------------- upb_generator/reflection/BUILD | 7 ++ upb_generator/reflection/generator.cc | 81 ++++++++++--- 21 files changed, 328 insertions(+), 234 deletions(-) create mode 100644 upb_generator/plugin.cc diff --git a/benchmarks/BUILD b/benchmarks/BUILD index c92d844f73805..ce253957adb4e 100644 --- a/benchmarks/BUILD +++ b/benchmarks/BUILD @@ -69,7 +69,7 @@ cc_test( ":benchmark_descriptor_sv_cc_proto", ":benchmark_descriptor_upb_proto", ":benchmark_descriptor_upb_proto_reflection", - "//:protobuf", + "//src/google/protobuf", "//src/google/protobuf/json", "//upb:base", "//upb:json", diff --git a/cmake/upb_generators.cmake b/cmake/upb_generators.cmake index 807236bf59eef..81a7a452a3d11 100644 --- a/cmake/upb_generators.cmake +++ b/cmake/upb_generators.cmake @@ -16,6 +16,7 @@ foreach(generator upb upbdefs upb_minitable) ) target_include_directories(protoc-gen-${generator} PRIVATE ${bootstrap_cmake_dir}) target_link_libraries(protoc-gen-${generator} + utf8_validity ${protobuf_LIB_UPB} ${protobuf_ABSL_USED_TARGETS} ) diff --git a/editions/BUILD b/editions/BUILD index 54e53a373caea..4c5fb78982ea7 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -104,7 +104,6 @@ cc_test( ], deps = [ ":defaults_test_embedded", - "//:protobuf", "//java/core:java_features_cc_proto", "//src/google/protobuf", "//src/google/protobuf:cpp_features_cc_proto", @@ -310,7 +309,7 @@ cc_test( ":test_editions_default_features_cc_proto", ":test_messages_proto2_editions_cc_proto", ":test_messages_proto3_editions_cc_proto", - "//:protobuf", + "//src/google/protobuf", "//src/google/protobuf:test_textproto", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index 504a1a46e3e78..2088f89efe685 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -112,6 +112,15 @@ cc_library( ], ) +cc_library( + name = "plugin", + hdrs = ["plugin.h"], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//visibility:public"], + deps = [":code_generator"], +) + cc_library( name = "code_generator_lite", srcs = ["code_generator_lite.cc"], diff --git a/upb/message/BUILD b/upb/message/BUILD index de904bd61e4d1..0d5f468ebf3f0 100644 --- a/upb/message/BUILD +++ b/upb/message/BUILD @@ -254,7 +254,7 @@ cc_test( srcs = ["accessors_test.cc"], deps = [ ":message", - "//:protobuf", + "//src/google/protobuf", "//upb:base", "//upb:mem", "//upb:mini_descriptor", @@ -312,7 +312,7 @@ cc_test( ":copy", ":internal", ":message", - "//:protobuf", + "//src/google/protobuf", "//upb:base", "//upb:mem", "//upb:mini_table", @@ -347,7 +347,7 @@ cc_test( ":internal", ":message", ":promote", - "//:protobuf", + "//src/google/protobuf", "//upb:base", "//upb:mem", "//upb:mini_descriptor", diff --git a/upb/mini_descriptor/BUILD b/upb/mini_descriptor/BUILD index a5dbd7ad0f871..a41c1017fc8ea 100644 --- a/upb/mini_descriptor/BUILD +++ b/upb/mini_descriptor/BUILD @@ -62,7 +62,7 @@ cc_test( deps = [ ":internal", ":mini_descriptor", - "//:protobuf", + "//src/google/protobuf", "//upb:base", "//upb:mem", "//upb:message", diff --git a/upb/test/BUILD b/upb/test/BUILD index e5a46e573269c..63794cc25b8be 100644 --- a/upb/test/BUILD +++ b/upb/test/BUILD @@ -28,7 +28,7 @@ cc_library( hdrs = ["parse_text_proto.h"], visibility = ["//upb:__subpackages__"], deps = [ - "//:protobuf", + "//src/google/protobuf", "@com_google_googletest//:gtest", ], ) diff --git a/upb/util/BUILD b/upb/util/BUILD index a8e0ed3a46aec..768859ed87e79 100644 --- a/upb/util/BUILD +++ b/upb/util/BUILD @@ -50,7 +50,7 @@ cc_library( hdrs = ["def_to_proto_test.h"], deps = [ ":def_to_proto", - "//:protobuf", + "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", "//src/google/protobuf/util:differencer", "//upb:base", @@ -70,7 +70,7 @@ cc_test( ":def_to_proto_test_lib", ":def_to_proto_test_upb_proto", ":def_to_proto_test_upb_proto_reflection", - "//:protobuf", + "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", "//src/google/protobuf:descriptor_upb_reflection_proto", "//src/google/protobuf/util:differencer", diff --git a/upb_generator/BUILD b/upb_generator/BUILD index 80888ab1e6e64..f27369977ff90 100644 --- a/upb_generator/BUILD +++ b/upb_generator/BUILD @@ -86,6 +86,9 @@ bootstrap_cc_library( bootstrap_cc_library( name = "plugin", + srcs = [ + "plugin.cc", + ], hdrs = [ "plugin.h", ], @@ -93,10 +96,12 @@ bootstrap_cc_library( ":plugin_upb_proto", "//upb/reflection:descriptor_upb_proto", "//upb/reflection:reflection", + "//upb_generator:file_layout", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//upb:friend_generators"], deps = [ + "//src/google/protobuf", "//src/google/protobuf/compiler:code_generator_lite", "//upb:base", "//upb:mem", diff --git a/upb_generator/c/BUILD b/upb_generator/c/BUILD index c55c9c1435999..2c169e560d75a 100644 --- a/upb_generator/c/BUILD +++ b/upb_generator/c/BUILD @@ -62,7 +62,6 @@ bootstrap_cc_library( "//upb_generator:common", "//upb_generator:file_layout", "//upb_generator:plugin", - "//upb_generator:plugin_upb_proto", "//upb/reflection:descriptor_upb_proto", "//upb/reflection:reflection", ], @@ -71,6 +70,8 @@ bootstrap_cc_library( deps = [ ":names", ":names_internal", + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:plugin", "//upb:base", "//upb:mem", "//upb:mini_table", @@ -84,7 +85,9 @@ bootstrap_cc_library( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", ], ) diff --git a/upb_generator/c/generator.cc b/upb_generator/c/generator.cc index 58f71957751aa..bc0459142198f 100644 --- a/upb_generator/c/generator.cc +++ b/upb_generator/c/generator.cc @@ -19,8 +19,11 @@ #include #include "absl/base/macros.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/memory/memory.h" +#include "absl/strings/cord.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" @@ -28,9 +31,12 @@ #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/compiler/plugin.h" #include "upb/base/descriptor_constants.h" #include "upb/base/status.hpp" #include "upb/base/string_view.h" +#include "upb/mem/arena.hpp" #include "upb/mini_table/field.h" #include "upb/reflection/def.hpp" #include "upb_generator/c/names.h" @@ -1148,35 +1154,42 @@ void WriteMiniDescriptorSource(const DefPoolPair& pools, upb::FileDefPtr file, } void GenerateFile(const DefPoolPair& pools, upb::FileDefPtr file, - const Options& options, Plugin* plugin) { + const Options& options, + google::protobuf::compiler::GeneratorContext* context) { Output h_output; WriteHeader(pools, file, options, h_output); - plugin->AddOutputFile(CApiHeaderFilename(file.name(), false), - h_output.output()); + { + auto stream = + absl::WrapUnique(context->Open(CApiHeaderFilename(file.name(), false))); + ABSL_CHECK(stream->WriteCord(absl::Cord(h_output.output()))); + } if (options.bootstrap_stage == 0) { Output c_output; WriteMiniDescriptorSource(pools, file, options, c_output); - plugin->AddOutputFile(SourceFilename(file), c_output.output()); + auto stream = absl::WrapUnique(context->Open(SourceFilename(file))); + ABSL_CHECK(stream->WriteCord(absl::Cord(c_output.output()))); } else { // TODO: remove once we can figure out how to make both Blaze // and Bazel happy with header-only libraries. - plugin->AddOutputFile(SourceFilename(file), "\n"); + auto stream = absl::WrapUnique(context->Open(SourceFilename(file))); + ABSL_CHECK(stream->WriteCord(absl::Cord("\n"))); } } -bool ParseOptions(Plugin* plugin, Options* options) { - for (const auto& pair : ParseGeneratorParameter(plugin->parameter())) { +bool ParseOptions(absl::string_view parameter, Options* options, + std::string* error) { + for (const auto& pair : ParseGeneratorParameter(parameter)) { if (pair.first == "bootstrap_stage") { if (!absl::SimpleAtoi(pair.second, &options->bootstrap_stage)) { - plugin->SetError(absl::Substitute("Bad stage: $0", pair.second)); + *error = absl::Substitute("Bad stage: $0", pair.second); return false; } } else if (pair.first == "experimental_strip_nonfunctional_codegen") { options->strip_nonfunctional_codegen = true; } else { - plugin->SetError(absl::Substitute("Unknown parameter: $0", pair.first)); + *error = absl::Substitute("Unknown parameter: $0", pair.first); return false; } } @@ -1184,9 +1197,46 @@ bool ParseOptions(Plugin* plugin, Options* options) { return true; } -absl::string_view ToStringView(upb_StringView str) { - return absl::string_view(str.data, str.size); -} +class CGenerator : public google::protobuf::compiler::CodeGenerator { + bool Generate(const google::protobuf::FileDescriptor* file, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + std::vector files{file}; + return GenerateAll(files, parameter, generator_context, error); + } + + bool GenerateAll(const std::vector& files, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + Options options; + if (!ParseOptions(parameter, &options, error)) { + return false; + } + + upb::Arena arena; + DefPoolPair pools; + absl::flat_hash_set files_seen; + for (const auto* file : files) { + PopulateDefPool(file, &arena, &pools, &files_seen); + upb::FileDefPtr upb_file = pools.GetFile(file->name()); + GenerateFile(pools, upb_file, options, generator_context); + } + + return true; + } + + uint64_t GetSupportedFeatures() const override { + return FEATURE_PROTO3_OPTIONAL | FEATURE_SUPPORTS_EDITIONS; + } + google::protobuf::Edition GetMinimumEdition() const override { + return google::protobuf::Edition::EDITION_PROTO2; + } + google::protobuf::Edition GetMaximumEdition() const override { + return google::protobuf::Edition::EDITION_2023; + } +}; } // namespace @@ -1194,21 +1244,6 @@ absl::string_view ToStringView(upb_StringView str) { } // namespace upb int main(int argc, char** argv) { - upb::generator::DefPoolPair pools; - upb::generator::Plugin plugin; - upb::generator::Options options; - if (!ParseOptions(&plugin, &options)) return 0; - plugin.GenerateFilesRaw( - [&](const UPB_DESC(FileDescriptorProto) * file_proto, bool generate) { - upb::Status status; - upb::FileDefPtr file = pools.AddFile(file_proto, &status); - if (!file) { - absl::string_view name = upb::generator::ToStringView( - UPB_DESC(FileDescriptorProto_name)(file_proto)); - ABSL_LOG(FATAL) << "Couldn't add file " << name - << " to DefPool: " << status.error_message(); - } - if (generate) GenerateFile(pools, file, options, &plugin); - }); - return 0; + upb::generator::CGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); } diff --git a/upb_generator/common/BUILD b/upb_generator/common/BUILD index ae8c297e86848..8980b24e15863 100644 --- a/upb_generator/common/BUILD +++ b/upb_generator/common/BUILD @@ -30,7 +30,6 @@ cc_library( "//third_party/kotlin/protobuf/generator/native:__subpackages__", ], deps = [ - "//:protobuf", "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", "//upb:base", diff --git a/upb_generator/file_layout.h b/upb_generator/file_layout.h index 4bf832e12467d..1c73d03cbf4f5 100644 --- a/upb_generator/file_layout.h +++ b/upb_generator/file_layout.h @@ -64,6 +64,10 @@ class DefPoolPair { return file64; } + upb::FileDefPtr GetFile(absl::string_view name) const { + return pool64_.FindFileByName(name.data()); + } + const upb_MiniTable* GetMiniTable32(upb::MessageDefPtr m) const { return pool32_.FindMessageByName(m.full_name()).mini_table(); } diff --git a/upb_generator/minitable/BUILD b/upb_generator/minitable/BUILD index 83e75b0d7d7c2..9988948f9e586 100644 --- a/upb_generator/minitable/BUILD +++ b/upb_generator/minitable/BUILD @@ -28,8 +28,6 @@ bootstrap_cc_library( bootstrap_deps = [ "//upb_generator:common", "//upb_generator:file_layout", - "//upb_generator:plugin", - "//upb_generator:plugin_upb_proto", "//upb/reflection:descriptor_upb_proto", "//upb/reflection:reflection", ], @@ -37,6 +35,7 @@ bootstrap_cc_library( deps = [ ":names", ":names_internal", + "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:code_generator_lite", "//upb:base", "//upb:mem", @@ -49,7 +48,9 @@ bootstrap_cc_library( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", ], ) @@ -115,13 +116,19 @@ bootstrap_cc_library( deps = [ ":names", ":names_internal", + "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:code_generator_lite", + "//src/google/protobuf/compiler:plugin", "//upb:base", + "//upb:mem", "//upb:port", "//upb_generator/common:names", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", ], ) diff --git a/upb_generator/minitable/generator.cc b/upb_generator/minitable/generator.cc index 99c63da067cc8..089b3d6d9d9b6 100644 --- a/upb_generator/minitable/generator.cc +++ b/upb_generator/minitable/generator.cc @@ -15,9 +15,12 @@ #include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" +#include "absl/memory/memory.h" +#include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" #include "upb/mini_table/enum.h" #include "upb/mini_table/field.h" #include "upb/mini_table/internal/field.h" @@ -29,7 +32,6 @@ #include "upb_generator/minitable/fasttable.h" #include "upb_generator/minitable/names.h" #include "upb_generator/minitable/names_internal.h" -#include "upb_generator/plugin.h" // Must be last. #include "upb/port/def.inc" @@ -431,10 +433,10 @@ std::string MultipleSourceFilename(upb::FileDefPtr file, *i, ".upb.c"); } -void WriteMiniTableMultipleSources(const DefPoolPair& pools, - upb::FileDefPtr file, - const MiniTableOptions& options, - Plugin* plugin) { +void WriteMiniTableMultipleSources( + const DefPoolPair& pools, upb::FileDefPtr file, + const MiniTableOptions& options, + google::protobuf::compiler::GeneratorContext* context) { std::vector messages = SortedMessages(file); std::vector extensions = SortedExtensions(file); std::vector enums = SortedEnums(file, kClosedEnums); @@ -444,22 +446,25 @@ void WriteMiniTableMultipleSources(const DefPoolPair& pools, Output output; WriteMiniTableSourceIncludes(file, options, output); WriteMessage(message, pools, options, output); - plugin->AddOutputFile(MultipleSourceFilename(file, message.full_name(), &i), - output.output()); + auto stream = absl::WrapUnique( + context->Open(MultipleSourceFilename(file, message.full_name(), &i))); + ABSL_CHECK(stream->WriteCord(absl::Cord(output.output()))); } for (const auto e : enums) { Output output; WriteMiniTableSourceIncludes(file, options, output); WriteEnum(e, output); - plugin->AddOutputFile(MultipleSourceFilename(file, e.full_name(), &i), - output.output()); + auto stream = absl::WrapUnique( + context->Open(MultipleSourceFilename(file, e.full_name(), &i))); + ABSL_CHECK(stream->WriteCord(absl::Cord(output.output()))); } for (const auto ext : extensions) { Output output; WriteMiniTableSourceIncludes(file, options, output); WriteExtension(pools, ext, output); - plugin->AddOutputFile(MultipleSourceFilename(file, ext.full_name(), &i), - output.output()); + auto stream = absl::WrapUnique( + context->Open(MultipleSourceFilename(file, ext.full_name(), &i))); + ABSL_CHECK(stream->WriteCord(absl::Cord(output.output()))); } } diff --git a/upb_generator/minitable/generator.h b/upb_generator/minitable/generator.h index 1422974457670..065e8fc358075 100644 --- a/upb_generator/minitable/generator.h +++ b/upb_generator/minitable/generator.h @@ -5,10 +5,10 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include "google/protobuf/compiler/code_generator.h" #include "upb/reflection/def.hpp" #include "upb_generator/common.h" #include "upb_generator/file_layout.h" -#include "upb_generator/plugin.h" namespace upb { namespace generator { @@ -24,7 +24,7 @@ void WriteMiniTableSource(const DefPoolPair& pools, upb::FileDefPtr file, void WriteMiniTableMultipleSources(const DefPoolPair& pools, upb::FileDefPtr file, const MiniTableOptions& options, - Plugin* plugin); + google::protobuf::compiler::GeneratorContext* context); void WriteMiniTableHeader(const DefPoolPair& pools, upb::FileDefPtr file, const MiniTableOptions& options, Output& output); diff --git a/upb_generator/minitable/main.cc b/upb_generator/minitable/main.cc index eff0186dd3aaa..9ea5a1ea42aea 100644 --- a/upb_generator/minitable/main.cc +++ b/upb_generator/minitable/main.cc @@ -5,16 +5,24 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include #include #include #include +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/memory/memory.h" +#include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/code_generator_lite.h" +#include "google/protobuf/compiler/plugin.h" #include "upb/base/status.hpp" #include "upb/base/string_view.h" +#include "upb/mem/arena.hpp" #include "upb/reflection/def.hpp" #include "upb_generator/common.h" #include "upb_generator/common/names.h" @@ -33,28 +41,32 @@ std::string SourceFilename(upb::FileDefPtr file) { return StripExtension(file.name()) + ".upb_minitable.c"; } -absl::string_view ToStringView(upb_StringView str) { - return absl::string_view(str.data, str.size); -} - void GenerateFile(const DefPoolPair& pools, upb::FileDefPtr file, - const MiniTableOptions& options, Plugin* plugin) { + const MiniTableOptions& options, + google::protobuf::compiler::GeneratorContext* context) { Output h_output; WriteMiniTableHeader(pools, file, options, h_output); - plugin->AddOutputFile(MiniTableHeaderFilename(file.name(), false), - h_output.output()); + { + auto stream = absl::WrapUnique( + context->Open(MiniTableHeaderFilename(file.name(), false))); + ABSL_CHECK(stream->WriteCord(absl::Cord(h_output.output()))); + } Output c_output; WriteMiniTableSource(pools, file, options, c_output); - plugin->AddOutputFile(SourceFilename(file), c_output.output()); + { + auto stream = absl::WrapUnique(context->Open(SourceFilename(file))); + ABSL_CHECK(stream->WriteCord(absl::Cord(c_output.output()))); + } if (options.one_output_per_message) { - WriteMiniTableMultipleSources(pools, file, options, plugin); + WriteMiniTableMultipleSources(pools, file, options, context); } } -bool ParseOptions(MiniTableOptions* options, Plugin* plugin) { - for (const auto& pair : ParseGeneratorParameter(plugin->parameter())) { +bool ParseOptions(MiniTableOptions* options, absl::string_view parameter, + std::string* error) { + for (const auto& pair : ParseGeneratorParameter(parameter)) { if (pair.first == "bootstrap_stage") { options->bootstrap = true; } else if (pair.first == "experimental_strip_nonfunctional_codegen") { @@ -62,7 +74,7 @@ bool ParseOptions(MiniTableOptions* options, Plugin* plugin) { } else if (pair.first == "one_output_per_message") { options->one_output_per_message = true; } else { - plugin->SetError(absl::Substitute("Unknown parameter: $0", pair.first)); + *error = absl::Substitute("Unknown parameter: $0", pair.first); return false; } } @@ -70,29 +82,51 @@ bool ParseOptions(MiniTableOptions* options, Plugin* plugin) { return true; } -int PluginMain(int argc, char** argv) { - DefPoolPair pools; - MiniTableOptions options; - Plugin plugin; - if (!ParseOptions(&options, &plugin)) return 0; - plugin.GenerateFilesRaw( - [&](const UPB_DESC(FileDescriptorProto) * file_proto, bool generate) { - upb::Status status; - upb::FileDefPtr file = pools.AddFile(file_proto, &status); - if (!file) { - absl::string_view name = - ToStringView(UPB_DESC(FileDescriptorProto_name)(file_proto)); - ABSL_LOG(FATAL) << "Couldn't add file " << name - << " to DefPool: " << status.error_message(); - } - if (generate) GenerateFile(pools, file, options, &plugin); - }); - return 0; -} +class MiniTableGenerator : public google::protobuf::compiler::CodeGenerator { + bool Generate(const google::protobuf::FileDescriptor* file, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + std::vector files{file}; + return GenerateAll(files, parameter, generator_context, error); + } + + bool GenerateAll(const std::vector& files, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + MiniTableOptions options; + if (!ParseOptions(&options, parameter, error)) { + return false; + } + + upb::Arena arena; + DefPoolPair pools; + absl::flat_hash_set files_seen; + for (const auto* file : files) { + PopulateDefPool(file, &arena, &pools, &files_seen); + upb::FileDefPtr upb_file = pools.GetFile(file->name()); + GenerateFile(pools, upb_file, options, generator_context); + } + + return true; + } + + uint64_t GetSupportedFeatures() const override { + return FEATURE_PROTO3_OPTIONAL | FEATURE_SUPPORTS_EDITIONS; + } + google::protobuf::Edition GetMinimumEdition() const override { + return google::protobuf::Edition::EDITION_PROTO2; + } + google::protobuf::Edition GetMaximumEdition() const override { + return google::protobuf::Edition::EDITION_2023; + } +}; } // namespace generator } // namespace upb int main(int argc, char** argv) { - return upb::generator::PluginMain(argc, argv); + upb::generator::MiniTableGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); } diff --git a/upb_generator/plugin.cc b/upb_generator/plugin.cc new file mode 100644 index 0000000000000..dab330ccab74e --- /dev/null +++ b/upb_generator/plugin.cc @@ -0,0 +1,58 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2025 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "upb_generator/plugin.h" + +#include + +#include "google/protobuf/descriptor.pb.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_log.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/descriptor.h" +#include "upb/base/status.hpp" +#include "upb/base/string_view.h" +#include "upb/mem/arena.hpp" +#include "upb/reflection/def.hpp" +#include "upb_generator/file_layout.h" + +// Must be last. +#include "upb/port/def.inc" + +namespace upb { +namespace generator { + +absl::string_view ToStringView(upb_StringView str) { + return absl::string_view(str.data, str.size); +} + +void PopulateDefPool(const google::protobuf::FileDescriptor* file, upb::Arena* arena, + DefPoolPair* pools, + absl::flat_hash_set* files_seen) { + bool new_file = files_seen->insert(std::string(file->name())).second; + if (new_file) { + for (int i = 0; i < file->dependency_count(); ++i) { + PopulateDefPool(file->dependency(i), arena, pools, files_seen); + } + google::protobuf::FileDescriptorProto raw_proto; + file->CopyTo(&raw_proto); + std::string serialized = raw_proto.SerializeAsString(); + auto* file_proto = UPB_DESC(FileDescriptorProto_parse)( + serialized.data(), serialized.size(), arena->ptr()); + upb::Status status; + upb::FileDefPtr upb_file = pools->AddFile(file_proto, &status); + if (!upb_file) { + absl::string_view name = + ToStringView(UPB_DESC(FileDescriptorProto_name)(file_proto)); + ABSL_LOG(FATAL) << "Couldn't add file " << name + << " to DefPool: " << status.error_message(); + } + } +} + +} // namespace generator +} // namespace upb diff --git a/upb_generator/plugin.h b/upb_generator/plugin.h index c75bd033e26a9..b3b36676b5c60 100644 --- a/upb_generator/plugin.h +++ b/upb_generator/plugin.h @@ -21,16 +21,13 @@ #endif #include "absl/container/flat_hash_set.h" -#include "absl/log/absl_log.h" #include "absl/strings/string_view.h" #include "google/protobuf/compiler/code_generator_lite.h" -#include "upb/base/status.hpp" -#include "upb/base/string_view.h" +#include "google/protobuf/descriptor.h" #include "upb/mem/arena.h" #include "upb/mem/arena.hpp" -#include "upb/reflection/def.hpp" #include "upb/reflection/descriptor_bootstrap.h" -#include "upb_generator/plugin_bootstrap.h" +#include "upb_generator/file_layout.h" // Must be last. #include "upb/port/def.inc" @@ -45,134 +42,10 @@ inline std::vector> ParseGeneratorParameter( return ret; } -class Plugin { - public: - Plugin() { ReadRequest(); } - ~Plugin() { WriteResponse(); } - - absl::string_view parameter() const { - return ToStringView( - UPB_DESC(compiler_CodeGeneratorRequest_parameter)(request_)); - } - - template - void GenerateFilesRaw(T&& func) { - absl::flat_hash_set files_to_generate; - size_t size; - const upb_StringView* file_to_generate = UPB_DESC( - compiler_CodeGeneratorRequest_file_to_generate)(request_, &size); - for (size_t i = 0; i < size; i++) { - files_to_generate.insert( - {file_to_generate[i].data, file_to_generate[i].size}); - } - - const UPB_DESC(FileDescriptorProto)* const* files = - UPB_DESC(compiler_CodeGeneratorRequest_proto_file)(request_, &size); - for (size_t i = 0; i < size; i++) { - upb::Status status; - absl::string_view name = - ToStringView(UPB_DESC(FileDescriptorProto_name)(files[i])); - func(files[i], files_to_generate.contains(name)); - } - } - - template - void GenerateFiles(T&& func) { - GenerateFilesRaw( - [this, &func](const UPB_DESC(FileDescriptorProto) * file_proto, - bool generate) { - upb::Status status; - upb::FileDefPtr file = pool_.AddFile(file_proto, &status); - if (!file) { - absl::string_view name = - ToStringView(UPB_DESC(FileDescriptorProto_name)(file_proto)); - ABSL_LOG(FATAL) << "Couldn't add file " << name - << " to DefPool: " << status.error_message(); - } - if (generate) func(file); - }); - } - - void SetError(absl::string_view error) { - char* data = - static_cast(upb_Arena_Malloc(arena_.ptr(), error.size())); - memcpy(data, error.data(), error.size()); - UPB_DESC(compiler_CodeGeneratorResponse_set_error) - (response_, upb_StringView_FromDataAndSize(data, error.size())); - } - - void AddOutputFile(absl::string_view filename, absl::string_view content) { - UPB_DESC(compiler_CodeGeneratorResponse_File)* file = UPB_DESC( - compiler_CodeGeneratorResponse_add_file)(response_, arena_.ptr()); - UPB_DESC(compiler_CodeGeneratorResponse_File_set_name) - (file, StringDup(filename)); - UPB_DESC(compiler_CodeGeneratorResponse_File_set_content) - (file, StringDup(content)); - } - - private: - upb::Arena arena_; - upb::DefPool pool_; - UPB_DESC(compiler_CodeGeneratorRequest) * request_; - UPB_DESC(compiler_CodeGeneratorResponse) * response_; - - static absl::string_view ToStringView(upb_StringView sv) { - return absl::string_view(sv.data, sv.size); - } - - upb_StringView StringDup(absl::string_view s) { - char* data = - reinterpret_cast(upb_Arena_Malloc(arena_.ptr(), s.size())); - memcpy(data, s.data(), s.size()); - return upb_StringView_FromDataAndSize(data, s.size()); - } - - std::string ReadAllStdinBinary() { - std::string data; -#ifdef _WIN32 - _setmode(_fileno(stdin), _O_BINARY); - _setmode(_fileno(stdout), _O_BINARY); -#endif - char buf[4096]; - while (size_t len = fread(buf, 1, sizeof(buf), stdin)) { - data.append(buf, len); - } - return data; - } - - void ReadRequest() { - std::string data = ReadAllStdinBinary(); - request_ = UPB_DESC(compiler_CodeGeneratorRequest_parse)( - data.data(), data.size(), arena_.ptr()); - if (!request_) { - ABSL_LOG(FATAL) << "Failed to parse CodeGeneratorRequest"; - } - response_ = UPB_DESC(compiler_CodeGeneratorResponse_new)(arena_.ptr()); - - int features = - UPB_DESC(compiler_CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL) | - UPB_DESC(compiler_CodeGeneratorResponse_FEATURE_SUPPORTS_EDITIONS); - UPB_DESC(compiler_CodeGeneratorResponse_set_supported_features) - (response_, features); - UPB_DESC(compiler_CodeGeneratorResponse_set_minimum_edition) - (response_, UPB_DESC(EDITION_PROTO2)); - UPB_DESC(compiler_CodeGeneratorResponse_set_maximum_edition) - (response_, UPB_DESC(EDITION_2023)); - } - - void WriteResponse() { - size_t size; - char* serialized = UPB_DESC(compiler_CodeGeneratorResponse_serialize)( - response_, arena_.ptr(), &size); - if (!serialized) { - ABSL_LOG(FATAL) << "Failed to serialize CodeGeneratorResponse"; - } - - if (fwrite(serialized, 1, size, stdout) != size) { - ABSL_LOG(FATAL) << "Failed to write response to stdout"; - } - } -}; +// Recursively populates the DefPoolPair with the given FileDescriptor. +void PopulateDefPool(const google::protobuf::FileDescriptor* file, upb::Arena* arena, + DefPoolPair* pools, + absl::flat_hash_set* files_seen); } // namespace generator } // namespace upb diff --git a/upb_generator/reflection/BUILD b/upb_generator/reflection/BUILD index db8bf892d6b99..f41db6c94020c 100644 --- a/upb_generator/reflection/BUILD +++ b/upb_generator/reflection/BUILD @@ -44,7 +44,10 @@ cc_library( deps = [ ":names", "//src/google/protobuf:descriptor_upb_c_proto", + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:plugin", "//upb:mem", + "//upb:port", "//upb:reflection", "//upb/util:def_to_proto", "//upb_generator:common", @@ -52,7 +55,11 @@ cc_library( "//upb_generator:plugin", "//upb_generator/common:names", "//upb_generator/minitable:names", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/upb_generator/reflection/generator.cc b/upb_generator/reflection/generator.cc index 3c748f9650584..a98eb0418ff53 100644 --- a/upb_generator/reflection/generator.cc +++ b/upb_generator/reflection/generator.cc @@ -6,12 +6,20 @@ // https://developers.google.com/open-source/licenses/bsd #include +#include #include +#include #include "google/protobuf/descriptor.upb.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/memory/memory.h" +#include "absl/strings/cord.h" #include "absl/strings/escaping.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/compiler/plugin.h" #include "upb/mem/arena.hpp" #include "upb/reflection/def.hpp" #include "upb/util/def_to_proto.h" @@ -22,6 +30,9 @@ #include "upb_generator/plugin.h" #include "upb_generator/reflection/names.h" +// Must be last. +#include "upb/port/def.inc" + namespace upb { namespace generator { namespace { @@ -141,22 +152,29 @@ void WriteDefSource(upb::FileDefPtr file, const Options& options, } void GenerateFile(upb::FileDefPtr file, const Options& options, - Plugin* plugin) { + google::protobuf::compiler::GeneratorContext* context) { Output h_def_output; WriteDefHeader(file, options, h_def_output); - plugin->AddOutputFile(DefHeaderFilename(file), h_def_output.output()); + { + auto stream = absl::WrapUnique(context->Open(DefHeaderFilename(file))); + ABSL_CHECK(stream->WriteCord(absl::Cord(h_def_output.output()))); + } Output c_def_output; WriteDefSource(file, options, c_def_output); - plugin->AddOutputFile(DefSourceFilename(file), c_def_output.output()); + { + auto stream = absl::WrapUnique(context->Open(DefSourceFilename(file))); + ABSL_CHECK(stream->WriteCord(absl::Cord(c_def_output.output()))); + } } -bool ParseOptions(Plugin* plugin, Options* options) { - for (const auto& pair : ParseGeneratorParameter(plugin->parameter())) { +bool ParseOptions(absl::string_view parameter, Options* options, + std::string* error) { + for (const auto& pair : ParseGeneratorParameter(parameter)) { if (pair.first == "dllexport_decl") { options->dllexport_decl = pair.second; } else { - plugin->SetError(absl::Substitute("Unknown parameter: $0", pair.first)); + *error = absl::Substitute("Unknown parameter: $0", pair.first); return false; } } @@ -165,15 +183,52 @@ bool ParseOptions(Plugin* plugin, Options* options) { } } // namespace + +class ReflectionGenerator : public google::protobuf::compiler::CodeGenerator { + bool Generate(const google::protobuf::FileDescriptor* file, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + std::vector files{file}; + return GenerateAll(files, parameter, generator_context, error); + } + + bool GenerateAll(const std::vector& files, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* generator_context, + std::string* error) const override { + Options options; + if (!ParseOptions(parameter, &options, error)) { + return false; + } + + upb::Arena arena; + DefPoolPair pools; + absl::flat_hash_set files_seen; + for (const auto* file : files) { + PopulateDefPool(file, &arena, &pools, &files_seen); + upb::FileDefPtr upb_file = pools.GetFile(file->name()); + GenerateFile(upb_file, options, generator_context); + } + + return true; + } + + uint64_t GetSupportedFeatures() const override { + return FEATURE_PROTO3_OPTIONAL | FEATURE_SUPPORTS_EDITIONS; + } + google::protobuf::Edition GetMinimumEdition() const override { + return google::protobuf::Edition::EDITION_PROTO2; + } + google::protobuf::Edition GetMaximumEdition() const override { + return google::protobuf::Edition::EDITION_2023; + } +}; + } // namespace generator } // namespace upb int main(int argc, char** argv) { - upb::generator::Plugin plugin; - upb::generator::Options options; - if (!ParseOptions(&plugin, &options)) return 0; - plugin.GenerateFiles([&](upb::FileDefPtr file) { - upb::generator::GenerateFile(file, options, &plugin); - }); - return 0; + upb::generator::ReflectionGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); }