Merge pull request #1369 from jskeet/tools-nuspec
Introduce a new nuget package, Google.Protobuf.Tools, basically to contain protoc on multiple platforms.
diff --git a/.travis.yml b/.travis.yml
index 60c0427..bcf3851 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@
# The Objective C build needs Xcode 7.0 or later.
osx_image: xcode7.2
script:
- - ./travis.sh $CONFIG
+ - ./tests.sh $CONFIG
env:
- CONFIG=cpp
- CONFIG=cpp_distcheck
diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs
index 6083f17..de6e571 100644
--- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs
+++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs
@@ -30,9 +30,8 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
-using System;
-using System.Linq;
using Google.Protobuf.Compatibility;
+using System;
namespace Google.Protobuf.Reflection
{
@@ -41,20 +40,35 @@
/// </summary>
public sealed class FieldDescriptor : DescriptorBase, IComparable<FieldDescriptor>
{
- private readonly FieldDescriptorProto proto;
private EnumDescriptor enumType;
private MessageDescriptor messageType;
- private readonly MessageDescriptor containingType;
- private readonly OneofDescriptor containingOneof;
private FieldType fieldType;
private readonly string propertyName; // Annoyingly, needed in Crosslink.
private IFieldAccessor accessor;
+ /// <summary>
+ /// Get the field's containing message type.
+ /// </summary>
+ public MessageDescriptor ContainingType { get; }
+
+ /// <summary>
+ /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
+ /// </summary>
+ public OneofDescriptor ContainingOneof { get; }
+
+ /// <summary>
+ /// The effective JSON name for this field. This is usually the lower-camel-cased form of the field name,
+ /// but can be overridden using the <c>json_name</c> option in the .proto file.
+ /// </summary>
+ public string JsonName { get; }
+
+ internal FieldDescriptorProto Proto { get; }
+
internal FieldDescriptor(FieldDescriptorProto proto, FileDescriptor file,
MessageDescriptor parent, int index, string propertyName)
: base(file, file.ComputeFullName(parent, proto.Name), index)
{
- this.proto = proto;
+ Proto = proto;
if (proto.Type != 0)
{
fieldType = GetFieldTypeFromProtoType(proto.Type);
@@ -64,7 +78,7 @@
{
throw new DescriptorValidationException(this, "Field numbers must be positive integers.");
}
- containingType = parent;
+ ContainingType = parent;
// OneofIndex "defaults" to -1 due to a hack in FieldDescriptor.OnConstruction.
if (proto.OneofIndex != -1)
{
@@ -73,7 +87,7 @@
throw new DescriptorValidationException(this,
$"FieldDescriptorProto.oneof_index is out of range for type {parent.Name}");
}
- containingOneof = parent.Oneofs[proto.OneofIndex];
+ ContainingOneof = parent.Oneofs[proto.OneofIndex];
}
file.DescriptorPool.AddSymbol(this);
@@ -83,20 +97,14 @@
// We could trust the generated code and check whether the type of the property is
// a MapField, but that feels a tad nasty.
this.propertyName = propertyName;
+ JsonName = Proto.JsonName == "" ? JsonFormatter.ToCamelCase(Proto.Name) : Proto.JsonName;
}
+
/// <summary>
/// The brief name of the descriptor's target.
/// </summary>
- public override string Name { get { return proto.Name; } }
-
-
- /// <summary>
- /// The json_name option of the descriptor's target.
- /// </summary>
- public string JsonName { get { return proto.JsonName == "" ? JsonFormatter.ToCamelCase(proto.Name) : proto.JsonName; } }
-
- internal FieldDescriptorProto Proto { get { return proto; } }
+ public override string Name => Proto.Name;
/// <summary>
/// Returns the accessor for this field.
@@ -116,7 +124,7 @@
/// and this property will return null.
/// </para>
/// </remarks>
- public IFieldAccessor Accessor { get { return accessor; } }
+ public IFieldAccessor Accessor => accessor;
/// <summary>
/// Maps a field type as included in the .proto file to a FieldType.
@@ -169,62 +177,32 @@
/// <summary>
/// Returns <c>true</c> if this field is a repeated field; <c>false</c> otherwise.
/// </summary>
- public bool IsRepeated
- {
- get { return Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED; }
- }
+ public bool IsRepeated => Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED;
/// <summary>
/// Returns <c>true</c> if this field is a map field; <c>false</c> otherwise.
/// </summary>
- public bool IsMap
- {
- get { return fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; }
- }
+ public bool IsMap => fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry;
/// <summary>
/// Returns <c>true</c> if this field is a packed, repeated field; <c>false</c> otherwise.
/// </summary>
- public bool IsPacked
- {
+ public bool IsPacked =>
// Note the || rather than && here - we're effectively defaulting to packed, because that *is*
// the default in proto3, which is all we support. We may give the wrong result for the protos
// within descriptor.proto, but that's okay, as they're never exposed and we don't use IsPacked
// within the runtime.
- get { return Proto.Options == null || Proto.Options.Packed; }
- }
-
- /// <summary>
- /// Get the field's containing message type.
- /// </summary>
- public MessageDescriptor ContainingType
- {
- get { return containingType; }
- }
-
- /// <summary>
- /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
- /// </summary>
- public OneofDescriptor ContainingOneof
- {
- get { return containingOneof; }
- }
-
+ Proto.Options == null || Proto.Options.Packed;
+
/// <summary>
/// Returns the type of the field.
/// </summary>
- public FieldType FieldType
- {
- get { return fieldType; }
- }
+ public FieldType FieldType => fieldType;
/// <summary>
/// Returns the field number declared in the proto file.
/// </summary>
- public int FieldNumber
- {
- get { return Proto.Number; }
- }
+ public int FieldNumber => Proto.Number;
/// <summary>
/// Compares this descriptor with another one, ordering in "canonical" order
@@ -234,7 +212,7 @@
/// </summary>
public int CompareTo(FieldDescriptor other)
{
- if (other.containingType != containingType)
+ if (other.ContainingType != ContainingType)
{
throw new ArgumentException("FieldDescriptors can only be compared to other FieldDescriptors " +
"for fields of the same message type.");
@@ -337,14 +315,14 @@
File.DescriptorPool.AddFieldByNumber(this);
- if (containingType != null && containingType.Proto.Options != null && containingType.Proto.Options.MessageSetWireFormat)
+ if (ContainingType != null && ContainingType.Proto.Options != null && ContainingType.Proto.Options.MessageSetWireFormat)
{
throw new DescriptorValidationException(this, "MessageSet format is not supported.");
}
- accessor = CreateAccessor(propertyName);
+ accessor = CreateAccessor();
}
- private IFieldAccessor CreateAccessor(string propertyName)
+ private IFieldAccessor CreateAccessor()
{
// If we're given no property name, that's because we really don't want an accessor.
// (At the moment, that means it's a map entry message...)
@@ -352,10 +330,10 @@
{
return null;
}
- var property = containingType.ClrType.GetProperty(propertyName);
+ var property = ContainingType.ClrType.GetProperty(propertyName);
if (property == null)
{
- throw new DescriptorValidationException(this, $"Property {propertyName} not found in {containingType.ClrType}");
+ throw new DescriptorValidationException(this, $"Property {propertyName} not found in {ContainingType.ClrType}");
}
return IsMap ? new MapFieldAccessor(property, this)
: IsRepeated ? new RepeatedFieldAccessor(property, this)
diff --git a/jenkins/README.md b/jenkins/README.md
new file mode 100644
index 0000000..29f664f
--- /dev/null
+++ b/jenkins/README.md
@@ -0,0 +1,6 @@
+
+Jenkins Infrastructure
+----------------------
+
+The scripts in this directory serve as plumbing for running the protobuf
+tests under Jenkins.
diff --git a/jenkins/build_and_run_docker.sh b/jenkins/build_and_run_docker.sh
new file mode 100755
index 0000000..abc6f05
--- /dev/null
+++ b/jenkins/build_and_run_docker.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+set -ex
+
+cd $(dirname $0)/..
+git_root=$(pwd)
+cd -
+
+# Use image name based on Dockerfile location checksum
+DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+
+# Make sure docker image has been built. Should be instantaneous if so.
+docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+# Run command inside docker
+docker run \
+ "$@" \
+ -e CCACHE_DIR=$CCACHE_DIR \
+ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
+ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+ -v "$git_root:/var/local/jenkins/protobuf:ro" \
+ -v $CCACHE_DIR:$CCACHE_DIR \
+ -w /var/local/git/protobuf \
+ --name=$CONTAINER_NAME \
+ $DOCKER_IMAGE_NAME \
+ bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# Copy output artifacts
+if [ "$OUTPUT_DIR" != "" ]
+then
+ docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
+fi
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+if [ "$FAILED" != "" ]
+then
+ exit 1
+fi
diff --git a/jenkins/buildcmds/README.md b/jenkins/buildcmds/README.md
new file mode 100644
index 0000000..7a48f2d
--- /dev/null
+++ b/jenkins/buildcmds/README.md
@@ -0,0 +1,6 @@
+
+Jenkins Build Commands
+----------------------
+
+The scripts in this directory are designed to be top-level entry points for
+Jenkins projects.
diff --git a/jenkins/buildcmds/pull_request.sh b/jenkins/buildcmds/pull_request.sh
new file mode 100755
index 0000000..01fda79
--- /dev/null
+++ b/jenkins/buildcmds/pull_request.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Jenkins as the entry point for
+# running the "pull request" project:
+#
+# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+export DOCKERFILE_DIR=jenkins/docker
+export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+./jenkins/build_and_run_docker.sh
diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile
new file mode 100644
index 0000000..8467aef
--- /dev/null
+++ b/jenkins/docker/Dockerfile
@@ -0,0 +1,130 @@
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
+#
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached. They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
+
+# Base Dockerfile for gRPC dev images
+FROM debian:latest
+
+# Apt source for old Python versions.
+RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C
+
+# Apt source for Oracle Java.
+run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \
+ echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
+
+# Apt source for Mono
+run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \
+ echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+
+# Install dependencies. We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get update && apt-get install -y \
+ autoconf \
+ autotools-dev \
+ build-essential \
+ bzip2 \
+ ccache \
+ curl \
+ gcc \
+ git \
+ libc6 \
+ libc6-dbg \
+ libc6-dev \
+ libgtest-dev \
+ libtool \
+ make \
+ parallel \
+ time \
+ wget \
+ # -- For csharp --
+ mono-devel \
+ referenceassemblies-pcl \
+ nunit \
+ # -- For all Java builds -- \
+ maven \
+ # -- For java_jdk6 -- \
+ # oops! not in jessie. too old? openjdk-6-jdk \
+ # -- For java_jdk7 -- \
+ openjdk-7-jdk \
+ # -- For java_oracle7 -- \
+ oracle-java7-installer \
+ # -- For python / python_cpp -- \
+ python-setuptools \
+ python-pip \
+ python-dev \
+ python2.6-dev \
+ python3.3-dev \
+ python3.4-dev \
+ # -- For Ruby --
+ ruby \
+ && apt-get clean
+
+##################
+# C# dependencies
+
+RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe
+
+##################
+# Python dependencies
+
+# These packages exist in apt-get, but their versions are too old, so we have
+# to get updates from pip.
+
+RUN pip install pip --upgrade
+RUN pip install virtualenv tox yattag
+
+
+##################
+# Ruby dependencies
+
+# Install rvm
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN \curl -sSL https://get.rvm.io | bash -s stable
+
+# Install Ruby 2.1
+RUN /bin/bash -l -c "rvm install ruby-2.1"
+RUN /bin/bash -l -c "rvm use --default ruby-2.1"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
+RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+##################
+# Java dependencies
+
+# This step requires compiling protoc. :(
+
+ENV MAVEN_REPO /var/maven_local_repository
+ENV MVN mvn --batch-mode
+
+RUN cd /tmp && \
+ git clone https://github.com/google/protobuf.git && \
+ cd protobuf && \
+ ./autogen.sh && \
+ ./configure && \
+ make -j6 && \
+ cd java && \
+ $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \
+ $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \
+ cd ../javanano && \
+ $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO
+
+##################
+# Prepare ccache
+
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+# Define the default command.
+CMD ["bash"]
diff --git a/jenkins/make_test_output.py b/jenkins/make_test_output.py
new file mode 100644
index 0000000..b1f2e2c
--- /dev/null
+++ b/jenkins/make_test_output.py
@@ -0,0 +1,91 @@
+"""Gathers output from test runs and create an XML file in JUnit format.
+
+The output files from the individual tests have been written in a directory
+structure like:
+
+ $DIR/joblog (output from "parallel --joblog joblog")
+ $DIR/logs/1/cpp/stdout
+ $DIR/logs/1/cpp/stderr
+ $DIR/logs/1/csharp/stdout
+ $DIR/logs/1/csharp/stderr
+ $DIR/logs/1/java_jdk7/stdout
+ $DIR/logs/1/java_jdk7/stderr
+ etc.
+
+This script bundles them into a single output XML file so Jenkins can show
+detailed test results. It runs as the last step before the Jenkins build
+finishes.
+"""
+
+import os;
+import sys;
+from yattag import Doc
+from collections import defaultdict
+
+def readtests(basedir):
+ tests = defaultdict(dict)
+
+ # Sample input (note: separators are tabs).
+ #
+ # Seq Host Starttime Runtime Send Receive Exitval Signal Command
+ # 1 : 1456263838.313 0.005 0 0 0 0 echo A
+ with open(basedir + "/joblog") as jobs:
+ firstline = next(jobs)
+ for line in jobs:
+ values = line.split("\t")
+
+ name = values[8].split()[-1]
+ test = tests[name]
+ test["name"] = name
+ test["time"] = values[3]
+
+ exitval = values[6]
+ if int(exitval):
+ # We don't have a more specific message. User should look at stderr.
+ test["failure"] = "TEST FAILURE"
+ else:
+ test["failure"] = False
+
+ for testname in os.listdir(basedir + "/logs/1"):
+ test = tests[testname]
+
+ with open(basedir + "/logs/1/" + testname + "/stdout") as f:
+ test["stdout"] = f.read()
+
+ with open(basedir + "/logs/1/" + testname + "/stderr") as f:
+ test["stderr"] = f.read()
+
+ # The cpp test is special since it doesn't run under parallel so doesn't show
+ # up in the job log.
+ tests["cpp"]["name"] = "cpp"
+
+ with open(basedir + '/logs/1/cpp/build_time', 'r') as f:
+ tests["cpp"]["time"] = f.read().strip()
+ tests["cpp"]["failure"] = False
+
+ ret = tests.values()
+ ret.sort(key=lambda x: x["name"])
+
+ return ret
+
+def genxml(tests):
+ doc, tag, text = Doc().tagtext()
+
+ with tag("testsuites"):
+ with tag("testsuite", name="Protobuf Tests"):
+ for test in tests:
+ with tag("testcase", name=test["name"], classname=test["name"],
+ time=test["time"]):
+ with tag("system-out"):
+ text(test["stdout"])
+ with tag("system-err"):
+ text(test["stderr"])
+ if test["failure"]:
+ with tag("failure"):
+ text(test["failure"])
+
+ return doc.getvalue()
+
+sys.stderr.write("make_test_output.py: writing XML from directory: " +
+ sys.argv[1] + "\n");
+print genxml(readtests(sys.argv[1]))
diff --git a/jenkins/pull_request_in_docker.sh b/jenkins/pull_request_in_docker.sh
new file mode 100755
index 0000000..887f97c
--- /dev/null
+++ b/jenkins/pull_request_in_docker.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# This is the script that runs inside Docker, once the image has been built,
+# to execute all tests for the "pull request" project.
+
+WORKSPACE_BASE=`pwd`
+MY_DIR="$(dirname "$0")"
+TEST_SCRIPT=$MY_DIR/../tests.sh
+BUILD_DIR=/tmp/protobuf
+
+set -e # exit immediately on error
+set -x # display all commands
+
+# The protobuf repository is mounted into our Docker image, but read-only.
+# We clone into a directory inside Docker (this is faster than cp).
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git clone /var/local/jenkins/protobuf
+cd protobuf
+
+# Set up the directory where our test output is going to go.
+OUTPUT_DIR=`mktemp -d`
+LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
+mkdir -p $LOG_OUTPUT_DIR/1/cpp
+
+################################################################################
+# cpp build needs to run first, non-parallelized, so that protoc is available
+# for other builds.
+
+# Output filenames to follow the overall scheme used by parallel, ie:
+# $DIR/logs/1/cpp/stdout
+# $DIR/logs/1/cpp/stderr
+# $DIR/logs/1/csharp/stdout
+# $DIR/logs/1/csharp/stderr
+# $DIR/logs/1/java_jdk7/stdout
+# $DIR/logs/1/java_jdk7/stderr
+CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
+CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
+
+# Time the C++ build, so we can put this info in the test output.
+# It's important that we get /usr/bin/time (which supports -f and -o) and not
+# the bash builtin "time" which doesn't.
+TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
+
+$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
+
+# Other tests are run in parallel.
+
+parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
+ csharp \
+ java_jdk7 \
+ javanano_jdk7 \
+ java_oracle7 \
+ javanano_oracle7 \
+ python \
+ python_cpp \
+ ruby21 \
+ || true # Process test results even if tests fail.
+
+cat $OUTPUT_DIR/joblog
+
+# The directory that is copied from Docker back into the Jenkins workspace.
+COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
+mkdir -p $COPY_FROM_DOCKER
+TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
+
+# Process all the output files from "parallel" and package them into a single
+# .xml file with detailed, broken-down test output.
+python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+
+ls -l $TESTOUTPUT_XML_FILE
diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py
index d03f2d2..ee6b944 100755
--- a/python/google/protobuf/internal/message_test.py
+++ b/python/google/protobuf/internal/message_test.py
@@ -57,7 +57,11 @@
except ImportError:
import unittest
from google.protobuf.internal import _parameterized
+from google.protobuf import descriptor_pb2
+from google.protobuf import descriptor_pool
from google.protobuf import map_unittest_pb2
+from google.protobuf import message_factory
+from google.protobuf import text_format
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import any_test_pb2
@@ -1776,5 +1780,60 @@
b'\x70\x01')
self.assertEqual(golden_data, message.SerializeToString())
+
+@unittest.skipIf(api_implementation.Type() != 'cpp',
+ 'explicit tests of the C++ implementation')
+class OversizeProtosTest(unittest.TestCase):
+
+ def setUp(self):
+ self.file_desc = """
+ name: "f/f.msg2"
+ package: "f"
+ message_type {
+ name: "msg1"
+ field {
+ name: "payload"
+ number: 1
+ label: LABEL_OPTIONAL
+ type: TYPE_STRING
+ }
+ }
+ message_type {
+ name: "msg2"
+ field {
+ name: "field"
+ number: 1
+ label: LABEL_OPTIONAL
+ type: TYPE_MESSAGE
+ type_name: "msg1"
+ }
+ }
+ """
+ pool = descriptor_pool.DescriptorPool()
+ desc = descriptor_pb2.FileDescriptorProto()
+ text_format.Parse(self.file_desc, desc)
+ pool.Add(desc)
+ self.proto_cls = message_factory.MessageFactory(pool).GetPrototype(
+ pool.FindMessageTypeByName('f.msg2'))
+ self.p = self.proto_cls()
+ self.p.field.payload = 'c' * (1024 * 1024 * 64 + 1)
+ self.p_serialized = self.p.SerializeToString()
+
+ def testAssertOversizeProto(self):
+ from google.protobuf.pyext._message import SetAllowOversizeProtos
+ SetAllowOversizeProtos(False)
+ q = self.proto_cls()
+ try:
+ q.ParseFromString(self.p_serialized)
+ except message.DecodeError as e:
+ self.assertEqual(str(e), 'Error parsing message')
+
+ def testSucceedOversizeProto(self):
+ from google.protobuf.pyext._message import SetAllowOversizeProtos
+ SetAllowOversizeProtos(True)
+ q = self.proto_cls()
+ q.ParseFromString(self.p_serialized)
+ self.assertEqual(self.p.field.payload, q.field.payload)
+
if __name__ == '__main__':
unittest.main()
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 60ec9c1..7d3e8c3 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -1911,6 +1911,30 @@
Py_RETURN_NONE;
}
+// Protobuf has a 64MB limit built in, this variable will override this. Please
+// do not enable this unless you fully understand the implications: protobufs
+// must all be kept in memory at the same time, so if they grow too big you may
+// get OOM errors. The protobuf APIs do not provide any tools for processing
+// protobufs in chunks. If you have protos this big you should break them up if
+// it is at all convenient to do so.
+static bool allow_oversize_protos = false;
+
+// Provide a method in the module to set allow_oversize_protos to a boolean
+// value. This method returns the newly value of allow_oversize_protos.
+static PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) {
+ if (!arg || !PyBool_Check(arg)) {
+ PyErr_SetString(PyExc_TypeError,
+ "Argument to SetAllowOversizeProtos must be boolean");
+ return NULL;
+ }
+ allow_oversize_protos = PyObject_IsTrue(arg);
+ if (allow_oversize_protos) {
+ Py_RETURN_TRUE;
+ } else {
+ Py_RETURN_FALSE;
+ }
+}
+
static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
const void* data;
Py_ssize_t data_length;
@@ -1921,15 +1945,9 @@
AssureWritable(self);
io::CodedInputStream input(
reinterpret_cast<const uint8*>(data), data_length);
-#if PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
- // Protobuf has a 64MB limit built in, this code will override this. Please do
- // not enable this unless you fully understand the implications: protobufs
- // must all be kept in memory at the same time, so if they grow too big you
- // may get OOM errors. The protobuf APIs do not provide any tools for
- // processing protobufs in chunks. If you have protos this big you should
- // break them up if it is at all convenient to do so.
- input.SetTotalBytesLimit(INT_MAX, INT_MAX);
-#endif // PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
+ if (allow_oversize_protos) {
+ input.SetTotalBytesLimit(INT_MAX, INT_MAX);
+ }
PyDescriptorPool* pool = GetDescriptorPoolForMessage(self);
input.SetExtensionRegistry(pool->pool, pool->message_factory);
bool success = self->message->MergePartialFromCodedStream(&input);
@@ -3046,6 +3064,11 @@
} // namespace python
} // namespace protobuf
+static PyMethodDef ModuleMethods[] = {
+ {"SetAllowOversizeProtos",
+ (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos,
+ METH_O, "Enable/disable oversize proto parsing."},
+};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef _module = {
@@ -3053,7 +3076,7 @@
"_message",
google::protobuf::python::module_docstring,
-1,
- NULL,
+ ModuleMethods, /* m_methods */
NULL,
NULL,
NULL,
@@ -3072,7 +3095,8 @@
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&_module);
#else
- m = Py_InitModule3("_message", NULL, google::protobuf::python::module_docstring);
+ m = Py_InitModule3("_message", ModuleMethods,
+ google::protobuf::python::module_docstring);
#endif
if (m == NULL) {
return INITFUNC_ERRORVAL;
diff --git a/src/google/protobuf/stubs/int128.cc b/src/google/protobuf/stubs/int128.cc
index d80c64f..3a36b4b 100644
--- a/src/google/protobuf/stubs/int128.cc
+++ b/src/google/protobuf/stubs/int128.cc
@@ -31,7 +31,7 @@
#include <google/protobuf/stubs/int128.h>
#include <iomanip>
-#include <iostream> // NOLINT(readability/streams)
+#include <ostream> // NOLINT(readability/streams)
#include <sstream>
namespace google {
diff --git a/src/google/protobuf/stubs/statusor.h b/src/google/protobuf/stubs/statusor.h
index ad84870..29f869a 100644
--- a/src/google/protobuf/stubs/statusor.h
+++ b/src/google/protobuf/stubs/statusor.h
@@ -224,7 +224,7 @@
template<typename T>
template<typename U>
inline StatusOr<T>::StatusOr(const StatusOr<U>& other)
- : status_(other.status_), value_(other.status_.ok() ? other.value_ : NULL) {
+ : status_(other.status_), value_(other.status_.ok() ? other.value_ : T()) {
}
template<typename T>
diff --git a/travis.sh b/tests.sh
similarity index 75%
rename from travis.sh
rename to tests.sh
index f039b39..fd81b76 100755
--- a/travis.sh
+++ b/tests.sh
@@ -1,17 +1,24 @@
-#!/usr/bin/env bash
+#!/bin/bash
+#
+# Build and runs tests for the protobuf project. The tests as written here are
+# used by both Jenkins and Travis, though some specialized logic is required to
+# handle the differences between them.
-# Note: travis currently does not support testing more than one language so the
-# .travis.yml cheats and claims to only be cpp. If they add multiple language
-# support, this should probably get updated to install steps and/or
-# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
-
-# .travis.yml uses matrix.exclude to block the cases where app-get can't be
-# use to install things.
+on_travis() {
+ if [ "$TRAVIS" == "true" ]; then
+ "$@"
+ fi
+}
# For when some other test needs the C++ main build, including protoc and
# libprotobuf.
internal_build_cpp() {
- if [ $(uname -s) == "Linux" ]; then
+ if [ -f src/protoc ]; then
+ # Already built.
+ return
+ fi
+
+ if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then
# Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more
# decent C++ 11 support in order to compile conformance tests.
sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
@@ -42,16 +49,20 @@
# need to really build protoc, but it's simplest to keep with the
# conventions of the other builds.
internal_build_cpp
+ NUGET=/usr/local/bin/nuget.exe
- # Install latest version of Mono
- sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
- echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
- echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list
- sudo apt-get update -qq
- sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
- wget www.nuget.org/NuGet.exe -O nuget.exe
+ if [ "$TRAVIS" == "true" ]; then
+ # Install latest version of Mono
+ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+ echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
+ echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list
+ sudo apt-get update -qq
+ sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
+ wget www.nuget.org/NuGet.exe -O nuget.exe
+ NUGET=../../nuget.exe
+ fi
- (cd csharp/src; mono ../../nuget.exe restore)
+ (cd csharp/src; mono $NUGET restore)
csharp/buildall.sh
cd conformance && make test_csharp && cd ..
}
@@ -78,40 +89,54 @@
version=$1
case "$version" in
jdk6)
- sudo apt-get install openjdk-6-jdk
+ on_travis sudo apt-get install openjdk-6-jdk
export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH
;;
jdk7)
- sudo apt-get install openjdk-7-jdk
+ on_travis sudo apt-get install openjdk-7-jdk
export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
;;
oracle7)
- sudo apt-get install python-software-properties # for apt-add-repository
- echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \
- sudo debconf-set-selections
- yes | sudo apt-add-repository ppa:webupd8team/java
- yes | sudo apt-get install oracle-java7-installer
+ if [ "$TRAVIS" == "true" ]; then
+ sudo apt-get install python-software-properties # for apt-add-repository
+ echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \
+ sudo debconf-set-selections
+ yes | sudo apt-add-repository ppa:webupd8team/java
+ yes | sudo apt-get install oracle-java7-installer
+ fi;
export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
;;
esac
+ if [ "$TRAVIS" != "true" ]; then
+ MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
+ MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
+ fi;
+
which java
java -version
}
+# --batch-mode supresses download progress output that spams the logs.
+MVN="mvn --batch-mode"
+
build_java() {
+ version=$1
+ dir=java_$version
# Java build needs `protoc`.
internal_build_cpp
- cd java && mvn test && mvn install
- cd util && mvn test
+ cp -r java $dir
+ cd $dir && $MVN clean && $MVN test
cd ../..
}
+# The conformance tests are hard-coded to work with the $ROOT/java directory.
+# So this can't run in parallel with two different sets of tests.
build_java_with_conformance_tests() {
# Java build needs `protoc`.
internal_build_cpp
- cd java && mvn test && mvn install
- cd util && mvn test && mvn assembly:single
+ cd java && $MVN test && $MVN install
+ cd util && $MVN package assembly:single
cd ../..
cd conformance && make test_java && cd ..
}
@@ -119,12 +144,12 @@
build_javanano() {
# Java build needs `protoc`.
internal_build_cpp
- cd javanano && mvn test && cd ..
+ cd javanano && $MVN test && cd ..
}
build_java_jdk6() {
use_java jdk6
- build_java
+ build_java jdk6
}
build_java_jdk7() {
use_java jdk7
@@ -132,7 +157,7 @@
}
build_java_oracle7() {
use_java oracle7
- build_java
+ build_java oracle7
}
build_javanano_jdk6() {
@@ -149,6 +174,9 @@
}
internal_install_python_deps() {
+ if [ "$TRAVIS" != "true" ]; then
+ return;
+ fi
# Install tox (OS X doesn't have pip).
if [ $(uname -s) == "Darwin" ]; then
sudo easy_install tox
@@ -278,6 +306,14 @@
cd js && npm install && npm test && cd ..
}
+# Note: travis currently does not support testing more than one language so the
+# .travis.yml cheats and claims to only be cpp. If they add multiple language
+# support, this should probably get updated to install steps and/or
+# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
+
+# .travis.yml uses matrix.exclude to block the cases where app-get can't be
+# use to install things.
+
# -------- main --------
if [ "$#" -ne 1 ]; then
@@ -294,10 +330,10 @@
objectivec_osx |
python |
python_cpp |
- ruby_19 |
- ruby_20 |
- ruby_21 |
- ruby_22 |
+ ruby19 |
+ ruby20 |
+ ruby21 |
+ ruby22 |
jruby }
"
exit 1