Merge pull request #1229 from keveman/unlimited_binary_proto
Added ALLOW_UNLIMITED_BINARY_PROTO macro and setting it when
diff --git a/BUILD b/BUILD
index 3cac4a8..c4de1c4 100644
--- a/BUILD
+++ b/BUILD
@@ -462,11 +462,11 @@
name = "gen_well_known_protos_java",
srcs = WELL_KNOWN_PROTOS,
outs = [
- "wellknown.srcjar"
+ "wellknown.srcjar",
],
cmd = "$(location :protoc) --java_out=$(@D)/wellknown.jar" +
- " -Isrc $(SRCS) " +
- " && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
+ " -Isrc $(SRCS) " +
+ " && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
tools = [":protoc"],
)
@@ -539,7 +539,10 @@
]),
copts = COPTS + [
"-DGOOGLE_PROTOBUF_HAS_ONEOF=1",
- ],
+ ] + select({
+ "//conditions:default": [],
+ ":allow_oversize_protos": ["-DPROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS=1"],
+ }),
includes = [
"python/",
"src/",
@@ -561,6 +564,13 @@
},
)
+config_setting(
+ name = "allow_oversize_protos",
+ values = {
+ "define": "allow_oversize_protos=true",
+ },
+)
+
py_proto_library(
name = "protobuf_python",
srcs = WELL_KNOWN_PROTOS,
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 863cde0..60ec9c1 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -1921,6 +1921,15 @@
AssureWritable(self);
io::CodedInputStream input(
reinterpret_cast<const uint8*>(data), data_length);
+#if PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
+ // Protobuf has a 64MB limit built in, this code will override this. Please do
+ // not enable this unless you fully understand the implications: protobufs
+ // must all be kept in memory at the same time, so if they grow too big you
+ // may get OOM errors. The protobuf APIs do not provide any tools for
+ // processing protobufs in chunks. If you have protos this big you should
+ // break them up if it is at all convenient to do so.
+ input.SetTotalBytesLimit(INT_MAX, INT_MAX);
+#endif // PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
PyDescriptorPool* pool = GetDescriptorPoolForMessage(self);
input.SetExtensionRegistry(pool->pool, pool->message_factory);
bool success = self->message->MergePartialFromCodedStream(&input);