pw_protobuf: Add support for imports
This updates the protobuf codegen to support importing external .proto
files, and the GN proto build to set include directories for proto
dependencies.
Change-Id: I28f4884935a266b1b40de1935ab67af9cad44e11
diff --git a/pw_protobuf/BUILD.gn b/pw_protobuf/BUILD.gn
index 66765f8..9e37211 100644
--- a/pw_protobuf/BUILD.gn
+++ b/pw_protobuf/BUILD.gn
@@ -90,16 +90,15 @@
}
pw_test("codegen_test") {
- deps = [
- ":codegen_test_protos_pwpb",
- ":pw_protobuf",
- ]
+ deps = [ ":codegen_test_protos_pwpb" ]
sources = [ "codegen_test.cc" ]
}
pw_proto_library("codegen_test_protos") {
sources = [
"pw_protobuf_protos/test_protos/full_test.proto",
+ "pw_protobuf_protos/test_protos/imported.proto",
+ "pw_protobuf_protos/test_protos/importer.proto",
"pw_protobuf_protos/test_protos/proto2.proto",
"pw_protobuf_protos/test_protos/repeated.proto",
]
diff --git a/pw_protobuf/codegen_test.cc b/pw_protobuf/codegen_test.cc
index e9e1efa..cf8969b 100644
--- a/pw_protobuf/codegen_test.cc
+++ b/pw_protobuf/codegen_test.cc
@@ -1,4 +1,4 @@
-// Copyright 2019 The Pigweed Authors
+// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
@@ -23,6 +23,7 @@
// generated C++ interface is valid rather than the correctness of the
// low-level encoder.
#include "pw_protobuf_protos/test_protos/full_test.pwpb.h"
+#include "pw_protobuf_protos/test_protos/importer.pwpb.h"
#include "pw_protobuf_protos/test_protos/proto2.pwpb.h"
#include "pw_protobuf_protos/test_protos/repeated.pwpb.h"
@@ -273,5 +274,26 @@
0);
}
+TEST(Codegen, Import) {
+ std::byte encode_buffer[64];
+ NestedEncoder<1, 3> encoder(encode_buffer);
+
+ Period::Encoder period(&encoder);
+ {
+ imported::Timestamp::Encoder start = period.GetStartEncoder();
+ start.WriteSeconds(1589501793);
+ start.WriteNanoseconds(511613110);
+ }
+
+ {
+ imported::Timestamp::Encoder end = period.GetEndEncoder();
+ end.WriteSeconds(1589501841);
+ end.WriteNanoseconds(490367432);
+ }
+
+ span<const std::byte> proto;
+ EXPECT_EQ(encoder.Encode(&proto), Status::OK);
+}
+
} // namespace
} // namespace pw::protobuf
diff --git a/pw_protobuf/pw_protobuf_protos/test_protos/imported.proto b/pw_protobuf/pw_protobuf_protos/test_protos/imported.proto
new file mode 100644
index 0000000..c375380
--- /dev/null
+++ b/pw_protobuf/pw_protobuf_protos/test_protos/imported.proto
@@ -0,0 +1,21 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+syntax = "proto3";
+
+package pw.protobuf.test.imported;
+
+message Timestamp {
+ uint64 seconds = 1;
+ uint32 nanoseconds = 2;
+}
diff --git a/pw_protobuf/pw_protobuf_protos/test_protos/importer.proto b/pw_protobuf/pw_protobuf_protos/test_protos/importer.proto
new file mode 100644
index 0000000..298a88c
--- /dev/null
+++ b/pw_protobuf/pw_protobuf_protos/test_protos/importer.proto
@@ -0,0 +1,23 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+syntax = "proto3";
+
+import 'pw_protobuf_protos/test_protos/imported.proto';
+
+package pw.protobuf.test;
+
+message Period {
+ imported.Timestamp start = 1;
+ imported.Timestamp end = 2;
+}
diff --git a/pw_protobuf/py/pw_protobuf/codegen.py b/pw_protobuf/py/pw_protobuf/codegen.py
index c09a0a2..a9329b5 100755
--- a/pw_protobuf/py/pw_protobuf/codegen.py
+++ b/pw_protobuf/py/pw_protobuf/codegen.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -21,19 +21,22 @@
import os
import sys
-from typing import List
+from typing import Iterable, List
import google.protobuf.compiler.plugin_pb2 as plugin_pb2
import google.protobuf.descriptor_pb2 as descriptor_pb2
from pw_protobuf.methods import PROTO_FIELD_METHODS
-from pw_protobuf.proto_structures import ProtoEnum, ProtoMessage
+from pw_protobuf.proto_structures import ProtoEnum, ProtoExternal, ProtoMessage
from pw_protobuf.proto_structures import ProtoMessageField, ProtoNode
from pw_protobuf.proto_structures import ProtoPackage
PLUGIN_NAME = 'pw_protobuf'
PLUGIN_VERSION = '0.1.0'
+PROTO_H_EXTENSION = '.pwpb.h'
+PROTO_CC_EXTENSION = '.pwpb.cc'
+
PROTOBUF_NAMESPACE = 'pw::protobuf'
BASE_PROTO_CLASS = 'ProtoMessageEncoder'
@@ -209,9 +212,13 @@
output.write_line(f'}} // namespace {namespace}')
-# TODO(frolv): Right now, this plugin assumes that package is synonymous with
-# .proto file. This will need to be updated to handle compiling multiple files.
-def generate_code_for_package(package: ProtoNode, output: OutputFile) -> None:
+def _proto_filename_to_generated_header(proto_file: str) -> str:
+ """Returns the generated C++ header name for a .proto file."""
+ return os.path.splitext(proto_file)[0] + PROTO_H_EXTENSION
+
+
+def generate_code_for_package(file_descriptor_proto, package: ProtoNode,
+ output: OutputFile) -> None:
"""Generates code for a single .pb.h file corresponding to a .proto file."""
assert package.type() == ProtoNode.Type.PACKAGE
@@ -223,8 +230,16 @@
output.write_line('#include <cstdint>\n')
output.write_line('#include "pw_protobuf/codegen.h"')
+ for imported_file in file_descriptor_proto.dependency:
+ generated_header = _proto_filename_to_generated_header(imported_file)
+ output.write_line(f'#include "{generated_header}"')
+
if package.cpp_namespace():
- output.write_line(f'\nnamespace {package.cpp_namespace()} {{')
+ file_namespace = package.cpp_namespace()
+ if file_namespace.startswith('::'):
+ file_namespace = file_namespace[2:]
+
+ output.write_line(f'\nnamespace {file_namespace} {{')
for node in package:
forward_declare(node, package, output)
@@ -258,8 +273,22 @@
enum.add_value(value.name, value.number)
-def add_message_fields(root: ProtoNode, message: ProtoNode,
- proto_message) -> None:
+def create_external_nodes(root: ProtoNode, path: str) -> ProtoNode:
+ """Creates external nodes for a path starting from the given root."""
+
+ node = root
+ for part in path.split('.'):
+ child = node.find(part)
+ if not child:
+ child = ProtoExternal(part)
+ node.add_child(child)
+ node = child
+
+ return node
+
+
+def add_message_fields(global_root: ProtoNode, package_root: ProtoNode,
+ message: ProtoNode, proto_message) -> None:
"""Adds fields from a protobuf message descriptor to a message node."""
assert message.type() == ProtoNode.Type.MESSAGE
@@ -267,24 +296,24 @@
if field.type_name:
# The "type_name" member contains the global .proto path of the
# field's type object, for example ".pw.protobuf.test.KeyValuePair".
- # Since only a single proto file is currently supported, the root
- # node has the value of the file's package ("pw.protobuf.test").
- # This must be stripped from the path to find the desired node
- # within the tree.
- #
- # TODO(frolv): Once multiple files are supported, the root node
- # should refer to the global namespace, and this should no longer
- # be needed.
- path = field.type_name
- if path[0] == '.':
- path = path[1:]
+ # Try to find the node for this object within the current context.
- if path.startswith(root.name()):
- relative_path = path[len(root.name()):].lstrip('.')
+ if field.type_name[0] == '.':
+ # Fully qualified path.
+ root_relative_path = field.type_name[1:]
+ search_root = global_root
else:
- relative_path = path
+ root_relative_path = field.type_name
+ search_root = package_root
- type_node = root.find(relative_path)
+ type_node = search_root.find(root_relative_path)
+
+ if type_node is None:
+ # Create nodes for field types that don't exist within this
+ # compilation context, such as those imported from other .proto
+ # files.
+ type_node = create_external_nodes(search_root,
+ root_relative_path)
else:
type_node = None
@@ -300,11 +329,12 @@
))
-def populate_fields(proto_file, root: ProtoNode) -> None:
+def populate_fields(proto_file, global_root: ProtoNode,
+ package_root: ProtoNode) -> None:
"""Traverses a proto file, adding all message and enum fields to a tree."""
def populate_message(node, message):
"""Recursively populates nested messages and enums."""
- add_message_fields(root, node, message)
+ add_message_fields(global_root, package_root, node, message)
for enum in message.enum_type:
add_enum_fields(node.find(enum.name), enum)
@@ -313,15 +343,21 @@
# Iterate through the proto file, populating top-level enums and messages.
for enum in proto_file.enum_type:
- add_enum_fields(root.find(enum.name), enum)
+ add_enum_fields(package_root.find(enum.name), enum)
for message in proto_file.message_type:
- populate_message(root.find(message.name), message)
+ populate_message(package_root.find(message.name), message)
def build_hierarchy(proto_file):
"""Creates a ProtoNode hierarchy from a proto file descriptor."""
- root = ProtoPackage(proto_file.package)
+ root = ProtoPackage('')
+ package_root = root
+
+ for part in proto_file.package.split('.'):
+ package = ProtoPackage(part)
+ package_root.add_child(package)
+ package_root = package
def build_message_subtree(proto_message):
node = ProtoMessage(proto_message.name)
@@ -333,29 +369,29 @@
return node
for enum in proto_file.enum_type:
- root.add_child(ProtoEnum(enum.name))
+ package_root.add_child(ProtoEnum(enum.name))
for message in proto_file.message_type:
- root.add_child(build_message_subtree(message))
+ package_root.add_child(build_message_subtree(message))
- return root
+ return root, package_root
-def process_proto_file(proto_file):
+def process_proto_file(proto_file) -> Iterable[OutputFile]:
"""Generates code for a single .proto file."""
# Two passes are made through the file. The first builds the tree of all
# message/enum nodes, then the second creates the fields in each. This is
# done as non-primitive fields need pointers to their types, which requires
# the entire tree to have been parsed into memory.
- root = build_hierarchy(proto_file)
- populate_fields(proto_file, root)
+ global_root, package_root = build_hierarchy(proto_file)
+ populate_fields(proto_file, global_root, package_root)
- output_filename = os.path.splitext(proto_file.name)[0] + '.pwpb.h'
+ output_filename = _proto_filename_to_generated_header(proto_file.name)
output_file = OutputFile(output_filename)
- generate_code_for_package(root, output_file)
+ generate_code_for_package(proto_file, package_root, output_file)
- return output_file
+ return [output_file]
def process_proto_request(req: plugin_pb2.CodeGeneratorRequest,
@@ -372,10 +408,11 @@
for proto_file in req.proto_file:
# TODO(frolv): Proto files are currently processed individually. Support
# for multiple files with cross-dependencies should be added.
- output_file = process_proto_file(proto_file)
- fd = res.file.add()
- fd.name = output_file.name()
- fd.content = output_file.content()
+ output_files = process_proto_file(proto_file)
+ for output_file in output_files:
+ fd = res.file.add()
+ fd.name = output_file.name()
+ fd.content = output_file.content()
def main() -> int:
diff --git a/pw_protobuf/py/pw_protobuf/methods.py b/pw_protobuf/py/pw_protobuf/methods.py
index 293448b..023437d 100644
--- a/pw_protobuf/py/pw_protobuf/methods.py
+++ b/pw_protobuf/py/pw_protobuf/methods.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
diff --git a/pw_protobuf/py/pw_protobuf/proto_structures.py b/pw_protobuf/py/pw_protobuf/proto_structures.py
index d70ea61..cd65060 100644
--- a/pw_protobuf/py/pw_protobuf/proto_structures.py
+++ b/pw_protobuf/py/pw_protobuf/proto_structures.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -34,10 +34,12 @@
PACKAGE maps to a C++ namespace.
MESSAGE maps to a C++ "Encoder" class within its own namespace.
ENUM maps to a C++ enum within its parent's namespace.
+ EXTERNAL represents a node defined within a different compilation unit.
"""
PACKAGE = 1
MESSAGE = 2
ENUM = 3
+ EXTERNAL = 4
def __init__(self, name: str):
self._name: str = name
@@ -130,9 +132,10 @@
# pylint: disable=protected-access
for section in path.split('.'):
- node = node._children[section]
- if node is None:
+ child = node._children.get(section)
+ if child is None:
return None
+ node = child
# pylint: enable=protected-access
return node
@@ -226,6 +229,22 @@
or child.type() == self.Type.MESSAGE)
+class ProtoExternal(ProtoNode):
+ """A node from a different compilation unit.
+
+ An external node is one that isn't defined within the current compilation
+ unit, most likely as it comes from an imported proto file. Its type is not
+ known, so it does not have any members or additional data. Its purpose
+ within the node graph is to provide namespace resolution between compile
+ units.
+ """
+ def type(self) -> ProtoNode.Type:
+ return ProtoNode.Type.EXTERNAL
+
+ def _supports_child(self, child: ProtoNode) -> bool:
+ return True
+
+
# This class is not a node and does not appear in the proto tree.
# Fields belong to proto messages and are processed separately.
class ProtoMessageField:
diff --git a/pw_protobuf_compiler/proto.gni b/pw_protobuf_compiler/proto.gni
index 97ac763..ee2e2dd 100644
--- a/pw_protobuf_compiler/proto.gni
+++ b/pw_protobuf_compiler/proto.gni
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -26,9 +26,14 @@
# protos: List of input .proto files.
template("_pw_pwpb_proto_library") {
_proto_gen_dir = "$root_gen_dir/protos"
- _outputs = process_file_template(
- invoker.protos,
- "$_proto_gen_dir/{{source_root_relative_dir}}/{{source_name_part}}.pwpb.h")
+ _module_path = get_path_info(".", "abspath")
+ _relative_proto_paths = rebase_path(invoker.protos, _module_path)
+
+ _outputs = []
+ foreach(_proto, _relative_proto_paths) {
+ _output = string_replace(_proto, ".proto", ".pwpb.h")
+ _outputs += [ "$_proto_gen_dir/$_output" ]
+ }
_gen_target = "${target_name}_gen"
pw_python_script(_gen_target) {
@@ -37,34 +42,33 @@
"--language",
"cc",
"--module-path",
- "//",
+ _module_path,
+ "--include-file",
+ invoker.include_file,
"--out-dir",
_proto_gen_dir,
] + get_path_info(invoker.protos, "abspath")
inputs = invoker.protos
outputs = _outputs
-
+ deps = invoker.deps
if (defined(invoker.protoc_deps)) {
- deps = invoker.protoc_deps
+ deps += invoker.protoc_deps
}
}
# For C++ proto files, the generated proto directory is added as an include
# path for the code. This requires using "all_dependent_configs" to force the
# include on any code that transitively depends on the generated protos.
- _include_root = rebase_path(get_path_info(".", "abspath"), "//")
_include_config_target = "${target_name}_includes"
config(_include_config_target) {
- include_dirs = [ "$_proto_gen_dir/$_include_root" ]
+ include_dirs = [ "$_proto_gen_dir" ]
}
# Create a library with the generated source files.
- # TODO(frolv): This currently only supports pw_protobuf, which is header-only.
- # Figure out how to support .cc files.
source_set(target_name) {
all_dependent_configs = [ ":$_include_config_target" ]
- deps = [ ":$_gen_target" ] + invoker.deps
- public_deps = [ dir_pw_protobuf ]
+ deps = [ ":$_gen_target" ]
+ public_deps = [ dir_pw_protobuf ] + invoker.gen_deps
sources = get_target_outputs(":$_gen_target")
public = filter_include(sources, [ "*.pwpb.h" ])
}
@@ -81,12 +85,18 @@
"\$dir_third_party_nanopb must be set to compile nanopb protobufs")
_proto_gen_dir = "$root_gen_dir/protos"
- _outputs = process_file_template(
- invoker.protos,
- [
- "$_proto_gen_dir/{{source_root_relative_dir}}/{{source_name_part}}.pb.h",
- "$_proto_gen_dir/{{source_root_relative_dir}}/{{source_name_part}}.pb.c",
- ])
+ _module_path = get_path_info(".", "abspath")
+ _relative_proto_paths = rebase_path(invoker.protos, _module_path)
+
+ _outputs = []
+ foreach(_proto, _relative_proto_paths) {
+ _output_h = string_replace(_proto, ".proto", ".pb.h")
+ _output_c = string_replace(_proto, ".proto", ".pb.c")
+ _outputs += [
+ "$_proto_gen_dir/$_output_h",
+ "$_proto_gen_dir/$_output_c",
+ ]
+ }
_nanopb_plugin = "$dir_third_party_nanopb/generator/protoc-gen-nanopb"
if (host_os == "win") {
@@ -102,9 +112,11 @@
"--language",
"nanopb",
"--module-path",
- "//",
+ _module_path,
"--include-paths",
"$dir_third_party_nanopb/generator/proto",
+ "--include-file",
+ invoker.include_file,
"--out-dir",
_proto_gen_dir,
"--custom-plugin",
@@ -113,8 +125,9 @@
inputs = invoker.protos
outputs = _outputs
+ deps = invoker.deps
if (defined(invoker.protoc_deps)) {
- deps = invoker.protoc_deps
+ deps += invoker.protoc_deps
}
}
@@ -133,11 +146,8 @@
# Create a library with the generated source files.
source_set(target_name) {
all_dependent_configs = [ ":$_include_config_target" ]
- deps = invoker.deps
- public_deps = [
- ":$_gen_target",
- dir_third_party_nanopb,
- ]
+ deps = [ ":$_gen_target" ]
+ public_deps = [ dir_third_party_nanopb ] + invoker.gen_deps
sources = get_target_outputs(":$_gen_target")
public = filter_include(sources, [ "*.pb.h" ])
}
@@ -167,11 +177,13 @@
"go",
"--module-path",
"//",
+ "--include-file",
+ invoker.include_file,
"--out-dir",
_proto_gen_dir,
] + get_path_info(invoker.protos, "abspath")
inputs = invoker.protos
- deps = invoker.deps
+ deps = invoker.deps + invoker.gen_deps
stamp = true
}
}
@@ -203,18 +215,40 @@
assert(defined(invoker.sources) && invoker.sources != [],
"pw_proto_codegen requires .proto source files")
- foreach(_gen, pw_protobuf_generators) {
+ # For each proto target, create a file which collects the base directories of
+ # all of its dependencies to list as include paths to protoc.
+ _include_metadata_target = "${target_name}_include_paths"
+ _include_metadata_file = "${target_gen_dir}/${target_name}_includes.txt"
+ generated_file(_include_metadata_target) {
if (defined(invoker.deps)) {
- _gen_deps = process_file_template(invoker.deps, "{{source}}_${_gen}")
+ # Collect metadata from the include path files of each dependency.
+ deps = process_file_template(invoker.deps, "{{source}}_include_paths")
} else {
- _gen_deps = []
+ deps = []
}
+ data_keys = [ "protoc_includes" ]
+ outputs = [ _include_metadata_file ]
+
+ # Indicate this library's base directory for its dependents.
+ metadata = {
+ protoc_includes = [ rebase_path(".", root_out_dir) ]
+ }
+ }
+
+ foreach(_gen, pw_protobuf_generators) {
_lang_target = "${target_name}_${_gen}"
if (_gen == "pwpb") {
+ _gen_deps = []
+ if (defined(invoker.deps)) {
+ _gen_deps = process_file_template(invoker.deps, "{{source}}_${_gen}")
+ }
+
_pw_pwpb_proto_library(_lang_target) {
protos = invoker.sources
- deps = _gen_deps
+ deps = [ ":$_include_metadata_target" ]
+ include_file = _include_metadata_file
+ gen_deps = _gen_deps
# List the pw_protobuf plugin's files as a dependency to recompile
# generated code if they are modified.
@@ -223,12 +257,16 @@
} else if (_gen == "nanopb") {
_pw_nanopb_proto_library(_lang_target) {
protos = invoker.sources
- deps = _gen_deps
+ deps = [ ":$_include_metadata_target" ]
+ include_file = _include_metadata_file
+ gen_deps = _gen_deps
}
} else if (_gen == "go") {
_pw_go_proto_library(_lang_target) {
protos = invoker.sources
- deps = _gen_deps
+ deps = [ ":$_include_metadata_target" ]
+ include_file = _include_metadata_file
+ gen_deps = _gen_deps
}
} else {
assert(false,
diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
index c42692c..b73a599 100644
--- a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
+++ b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
@@ -44,6 +44,9 @@
default=[],
type=lambda arg: arg.split(';'),
help='protoc include paths')
+ parser.add_argument('--include-file',
+ type=argparse.FileType('r'),
+ help='File containing additional protoc include paths')
parser.add_argument('--out-dir',
required=True,
help='Output directory for generated code')
@@ -96,12 +99,15 @@
return 1
include_paths = [f'-I{path}' for path in args.include_paths]
+ include_paths += [f'-I{line.strip()}' for line in args.include_file]
return pw_cli.process.run(
'protoc',
- *include_paths,
'-I',
args.module_path,
+ '-I',
+ args.out_dir,
+ *include_paths,
*lang_args,
*args.protos,
).returncode