Skip to content

Commit dce6b0f

Browse files
acozzettecopybara-github
acozzette
authored and
copybara-github
committed
Move upb minitable code generator into protoc
The main goal of this is to incorporate the minitable generator into our released protoc binaries so that Rust users don't have to build it from source. PiperOrigin-RevId: 728785507
1 parent cd46d34 commit dce6b0f

File tree

12 files changed

+121
-121
lines changed

12 files changed

+121
-121
lines changed

‎.github/workflows/test_cpp.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ jobs:
135135
-c "set -ex;
136136
sccache -z;
137137
cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
138-
-Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=17
138+
-Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=17
139139
-Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
140140
cmake --build . --parallel 20;
141141
ctest --parallel 20;

‎.github/workflows/test_upb.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
bazel-cache: "upb-bazel-gcc"
7575
bazel: >-
7676
test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
77-
--copt="-Wno-error=maybe-uninitialized" --copt="-Wno-error=attributes"
77+
--copt="-Wno-error=maybe-uninitialized"
7878
//bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
7979
8080
windows:

‎cmake/libprotoc.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ if(protobuf_HAVE_LD_VERSION_SCRIPT)
1717
LINK_DEPENDS ${protobuf_SOURCE_DIR}/src/libprotoc.map)
1818
endif()
1919
target_link_libraries(libprotoc PRIVATE libprotobuf)
20-
target_link_libraries(libprotoc PUBLIC ${protobuf_ABSL_USED_TARGETS})
20+
target_link_libraries(libprotoc PUBLIC libupb ${protobuf_ABSL_USED_TARGETS})
2121
protobuf_configure_target(libprotoc)
2222
if(protobuf_BUILD_SHARED_LIBS)
2323
target_compile_definitions(libprotoc

‎pkg/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ cc_dist_library(
194194
dist_deps = [
195195
":protobuf",
196196
":protobuf_lite",
197+
":upb",
197198
],
198199
tags = ["manual"],
199200
deps = [
@@ -207,6 +208,7 @@ cc_dist_library(
207208
"//src/google/protobuf/compiler/python",
208209
"//src/google/protobuf/compiler/ruby",
209210
"//src/google/protobuf/compiler/rust",
211+
"//upb_generator/minitable:generator",
210212
],
211213
)
212214

‎src/google/protobuf/compiler/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ cc_library(
238238
"//src/google/protobuf/compiler/python",
239239
"//src/google/protobuf/compiler/ruby",
240240
"//src/google/protobuf/compiler/rust",
241+
"//upb_generator/minitable:generator_stage0",
241242
"@abseil-cpp//absl/log:initialize",
242243
],
243244
)

‎src/google/protobuf/compiler/main.cc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "google/protobuf/compiler/python/pyi_generator.h"
1818
#include "google/protobuf/compiler/ruby/ruby_generator.h"
1919
#include "google/protobuf/compiler/rust/generator.h"
20+
#include "upb_generator/minitable/generator.h"
2021

2122
#ifdef DISABLE_PROTOC_CONFIG
2223
#include "google/protobuf/compiler/allowlists/allowlist.h"
@@ -106,6 +107,11 @@ int ProtobufMain(int argc, char* argv[]) {
106107
rust::RustGenerator rust_generator;
107108
cli.RegisterGenerator("--rust_out", "--rust_opt", &rust_generator,
108109
"Generate Rust sources.");
110+
111+
// upb minitables
112+
upb::generator::MiniTableGenerator minitable_generator;
113+
cli.RegisterGenerator("--upb_minitable_out", "--upb_minitable_opt",
114+
&minitable_generator, "Generate upb minitables");
109115
#ifdef DISABLE_PROTOC_CONFIG
110116
auto cleanup = internal::DisableAllowlistInternalOnly();
111117
#endif // DISABLE_PROTOC_CONFIG

‎upb/bazel/build_defs.bzl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ _DEFAULT_CPPOPTS = []
1111
_DEFAULT_COPTS = [
1212
# this is a compile error in C++ clang and GNU C, but not clang C by default
1313
"-Werror=incompatible-pointer-types",
14+
# GCC does not seem to suppport the no_sanitize attribute in some places
15+
# where we use it.
16+
"-Wno-error=attributes",
1417
]
1518

1619
_DEFAULT_CPPOPTS.extend([

‎upb_generator/bootstrap_compiler.bzl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ _extra_proto_path = "-I$$(dirname $(location @com_google_protobuf//:descriptor_p
2020

2121
# This visibility is used automatically for anything used by the bootstrapping process.
2222
_bootstrap_visibility = [
23+
"//src/google/protobuf/compiler:__pkg__",
2324
"//third_party/upb/github:__pkg__",
2425
"//upb_generator:__subpackages__",
2526
"//upb/reflection:__pkg__",

‎upb_generator/minitable/BUILD

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,17 @@ bootstrap_cc_library(
3030
"//upb_generator:file_layout",
3131
"//upb/reflection:descriptor_upb_proto",
3232
"//upb/reflection:reflection",
33+
"//upb_generator:plugin",
3334
],
3435
copts = UPB_DEFAULT_CPPOPTS,
36+
visibility = ["//pkg:__pkg__"],
3537
deps = [
3638
":names",
3739
":names_internal",
40+
"//src/google/protobuf:port",
3841
"//src/google/protobuf/compiler:code_generator",
3942
"//src/google/protobuf/compiler:code_generator_lite",
43+
"//src/google/protobuf/compiler:plugin",
4044
"//upb:base",
4145
"//upb:mem",
4246
"//upb:mini_table",

‎upb_generator/minitable/generator.cc

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include "absl/strings/string_view.h"
2222
#include "absl/strings/substitute.h"
2323
#include "google/protobuf/compiler/code_generator.h"
24+
#include "google/protobuf/compiler/code_generator_lite.h"
25+
#include "upb/mem/arena.hpp"
2426
#include "upb/mini_table/enum.h"
2527
#include "upb/mini_table/field.h"
2628
#include "upb/mini_table/internal/field.h"
@@ -32,6 +34,7 @@
3234
#include "upb_generator/minitable/fasttable.h"
3335
#include "upb_generator/minitable/names.h"
3436
#include "upb_generator/minitable/names_internal.h"
37+
#include "upb_generator/plugin.h"
3538

3639
// Must be last.
3740
#include "upb/port/def.inc"
@@ -468,5 +471,72 @@ void WriteMiniTableMultipleSources(
468471
}
469472
}
470473

474+
std::string SourceFilename(upb::FileDefPtr file) {
475+
return StripExtension(file.name()) + ".upb_minitable.c";
476+
}
477+
478+
void GenerateFile(const DefPoolPair& pools, upb::FileDefPtr file,
479+
const MiniTableOptions& options,
480+
google::protobuf::compiler::GeneratorContext* context) {
481+
Output h_output;
482+
WriteMiniTableHeader(pools, file, options, h_output);
483+
{
484+
auto stream = absl::WrapUnique(
485+
context->Open(MiniTableHeaderFilename(file.name(), false)));
486+
ABSL_CHECK(stream->WriteCord(absl::Cord(h_output.output())));
487+
}
488+
489+
Output c_output;
490+
WriteMiniTableSource(pools, file, options, c_output);
491+
{
492+
auto stream = absl::WrapUnique(context->Open(SourceFilename(file)));
493+
ABSL_CHECK(stream->WriteCord(absl::Cord(c_output.output())));
494+
}
495+
496+
if (options.one_output_per_message) {
497+
WriteMiniTableMultipleSources(pools, file, options, context);
498+
}
499+
}
500+
501+
bool ParseOptions(MiniTableOptions* options, absl::string_view parameter,
502+
std::string* error) {
503+
for (const auto& pair : ParseGeneratorParameter(parameter)) {
504+
if (pair.first == "bootstrap_stage") {
505+
options->bootstrap = true;
506+
} else if (pair.first == "experimental_strip_nonfunctional_codegen") {
507+
options->strip_nonfunctional_codegen = true;
508+
} else if (pair.first == "one_output_per_message") {
509+
options->one_output_per_message = true;
510+
} else {
511+
*error = absl::Substitute("Unknown parameter: $0", pair.first);
512+
return false;
513+
}
514+
}
515+
516+
return true;
517+
}
518+
519+
bool MiniTableGenerator::GenerateAll(
520+
const std::vector<const google::protobuf::FileDescriptor*>& files,
521+
const std::string& parameter,
522+
google::protobuf::compiler::GeneratorContext* generator_context,
523+
std::string* error) const {
524+
MiniTableOptions options;
525+
if (!ParseOptions(&options, parameter, error)) {
526+
return false;
527+
}
528+
529+
upb::Arena arena;
530+
DefPoolPair pools;
531+
absl::flat_hash_set<std::string> files_seen;
532+
for (const auto* file : files) {
533+
PopulateDefPool(file, &arena, &pools, &files_seen);
534+
upb::FileDefPtr upb_file = pools.GetFile(file->name());
535+
GenerateFile(pools, upb_file, options, generator_context);
536+
}
537+
538+
return true;
539+
}
540+
471541
} // namespace generator
472542
} // namespace upb

‎upb_generator/minitable/generator.h

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
#include "upb_generator/common.h"
1111
#include "upb_generator/file_layout.h"
1212

13+
// Must be included last.
14+
#include "google/protobuf/port_def.inc"
15+
1316
namespace upb {
1417
namespace generator {
1518

@@ -28,5 +31,33 @@ void WriteMiniTableMultipleSources(const DefPoolPair& pools,
2831
void WriteMiniTableHeader(const DefPoolPair& pools, upb::FileDefPtr file,
2932
const MiniTableOptions& options, Output& output);
3033

34+
class PROTOC_EXPORT MiniTableGenerator
35+
: public google::protobuf::compiler::CodeGenerator {
36+
bool Generate(const google::protobuf::FileDescriptor* file,
37+
const std::string& parameter,
38+
google::protobuf::compiler::GeneratorContext* generator_context,
39+
std::string* error) const override {
40+
std::vector<const google::protobuf::FileDescriptor*> files{file};
41+
return GenerateAll(files, parameter, generator_context, error);
42+
}
43+
44+
bool GenerateAll(const std::vector<const google::protobuf::FileDescriptor*>& files,
45+
const std::string& parameter,
46+
google::protobuf::compiler::GeneratorContext* generator_context,
47+
std::string* error) const override;
48+
49+
uint64_t GetSupportedFeatures() const override {
50+
return FEATURE_PROTO3_OPTIONAL | FEATURE_SUPPORTS_EDITIONS;
51+
}
52+
google::protobuf::Edition GetMinimumEdition() const override {
53+
return google::protobuf::Edition::EDITION_PROTO2;
54+
}
55+
google::protobuf::Edition GetMaximumEdition() const override {
56+
return google::protobuf::Edition::EDITION_2023;
57+
}
58+
};
59+
3160
} // namespace generator
3261
} // namespace upb
62+
63+
#include "google/protobuf/port_undef.inc"

‎upb_generator/minitable/main.cc

Lines changed: 0 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -5,126 +5,8 @@
55
// license that can be found in the LICENSE file or at
66
// https://developers.google.com/open-source/licenses/bsd
77

8-
#include <cstdint>
9-
#include <string>
10-
#include <utility>
11-
#include <vector>
12-
13-
#include "absl/container/flat_hash_set.h"
14-
#include "absl/log/absl_check.h"
15-
#include "absl/log/absl_log.h"
16-
#include "absl/memory/memory.h"
17-
#include "absl/strings/cord.h"
18-
#include "absl/strings/string_view.h"
19-
#include "absl/strings/substitute.h"
20-
#include "google/protobuf/compiler/code_generator.h"
21-
#include "google/protobuf/compiler/code_generator_lite.h"
228
#include "google/protobuf/compiler/plugin.h"
23-
#include "upb/base/status.hpp"
24-
#include "upb/base/string_view.h"
25-
#include "upb/mem/arena.hpp"
26-
#include "upb/reflection/def.hpp"
27-
#include "upb_generator/common.h"
28-
#include "upb_generator/common/names.h"
29-
#include "upb_generator/file_layout.h"
309
#include "upb_generator/minitable/generator.h"
31-
#include "upb_generator/minitable/names_internal.h"
32-
#include "upb_generator/plugin.h"
33-
34-
// Must be last.
35-
#include "upb/port/def.inc"
36-
37-
namespace upb {
38-
namespace generator {
39-
40-
std::string SourceFilename(upb::FileDefPtr file) {
41-
return StripExtension(file.name()) + ".upb_minitable.c";
42-
}
43-
44-
void GenerateFile(const DefPoolPair& pools, upb::FileDefPtr file,
45-
const MiniTableOptions& options,
46-
google::protobuf::compiler::GeneratorContext* context) {
47-
Output h_output;
48-
WriteMiniTableHeader(pools, file, options, h_output);
49-
{
50-
auto stream = absl::WrapUnique(
51-
context->Open(MiniTableHeaderFilename(file.name(), false)));
52-
ABSL_CHECK(stream->WriteCord(absl::Cord(h_output.output())));
53-
}
54-
55-
Output c_output;
56-
WriteMiniTableSource(pools, file, options, c_output);
57-
{
58-
auto stream = absl::WrapUnique(context->Open(SourceFilename(file)));
59-
ABSL_CHECK(stream->WriteCord(absl::Cord(c_output.output())));
60-
}
61-
62-
if (options.one_output_per_message) {
63-
WriteMiniTableMultipleSources(pools, file, options, context);
64-
}
65-
}
66-
67-
bool ParseOptions(MiniTableOptions* options, absl::string_view parameter,
68-
std::string* error) {
69-
for (const auto& pair : ParseGeneratorParameter(parameter)) {
70-
if (pair.first == "bootstrap_stage") {
71-
options->bootstrap = true;
72-
} else if (pair.first == "experimental_strip_nonfunctional_codegen") {
73-
options->strip_nonfunctional_codegen = true;
74-
} else if (pair.first == "one_output_per_message") {
75-
options->one_output_per_message = true;
76-
} else {
77-
*error = absl::Substitute("Unknown parameter: $0", pair.first);
78-
return false;
79-
}
80-
}
81-
82-
return true;
83-
}
84-
85-
class MiniTableGenerator : public google::protobuf::compiler::CodeGenerator {
86-
bool Generate(const google::protobuf::FileDescriptor* file,
87-
const std::string& parameter,
88-
google::protobuf::compiler::GeneratorContext* generator_context,
89-
std::string* error) const override {
90-
std::vector<const google::protobuf::FileDescriptor*> files{file};
91-
return GenerateAll(files, parameter, generator_context, error);
92-
}
93-
94-
bool GenerateAll(const std::vector<const google::protobuf::FileDescriptor*>& files,
95-
const std::string& parameter,
96-
google::protobuf::compiler::GeneratorContext* generator_context,
97-
std::string* error) const override {
98-
MiniTableOptions options;
99-
if (!ParseOptions(&options, parameter, error)) {
100-
return false;
101-
}
102-
103-
upb::Arena arena;
104-
DefPoolPair pools;
105-
absl::flat_hash_set<std::string> files_seen;
106-
for (const auto* file : files) {
107-
PopulateDefPool(file, &arena, &pools, &files_seen);
108-
upb::FileDefPtr upb_file = pools.GetFile(file->name());
109-
GenerateFile(pools, upb_file, options, generator_context);
110-
}
111-
112-
return true;
113-
}
114-
115-
uint64_t GetSupportedFeatures() const override {
116-
return FEATURE_PROTO3_OPTIONAL | FEATURE_SUPPORTS_EDITIONS;
117-
}
118-
google::protobuf::Edition GetMinimumEdition() const override {
119-
return google::protobuf::Edition::EDITION_PROTO2;
120-
}
121-
google::protobuf::Edition GetMaximumEdition() const override {
122-
return google::protobuf::Edition::EDITION_2023;
123-
}
124-
};
125-
126-
} // namespace generator
127-
} // namespace upb
12810

12911
int main(int argc, char** argv) {
13012
upb::generator::MiniTableGenerator generator;

0 commit comments

Comments
 (0)