Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 1 | // Protocol Buffers - Google's data interchange format |
| 2 | // Copyright 2008 Google Inc. All rights reserved. |
| 3 | // https://developers.google.com/protocol-buffers/ |
| 4 | // |
| 5 | // Redistribution and use in source and binary forms, with or without |
| 6 | // modification, are permitted provided that the following conditions are |
| 7 | // met: |
| 8 | // |
| 9 | // * Redistributions of source code must retain the above copyright |
| 10 | // notice, this list of conditions and the following disclaimer. |
| 11 | // * Redistributions in binary form must reproduce the above |
| 12 | // copyright notice, this list of conditions and the following disclaimer |
| 13 | // in the documentation and/or other materials provided with the |
| 14 | // distribution. |
| 15 | // * Neither the name of Google Inc. nor the names of its |
| 16 | // contributors may be used to endorse or promote products derived from |
| 17 | // this software without specific prior written permission. |
| 18 | // |
| 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
| 31 | |
| 32 | // This file defines a protocol for running the conformance test suite |
| 33 | // in-process. In other words, the suite itself will run in the same process as |
| 34 | // the code under test. |
| 35 | // |
| 36 | // For pros and cons of this approach, please see conformance.proto. |
| 37 | |
| 38 | #ifndef CONFORMANCE_CONFORMANCE_TEST_H |
| 39 | #define CONFORMANCE_CONFORMANCE_TEST_H |
| 40 | |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 41 | #include <functional> |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 42 | #include <string> |
Josh Haberman | b0500b3 | 2015-07-10 16:36:59 -0700 | [diff] [blame] | 43 | #include <google/protobuf/stubs/common.h> |
| 44 | #include <google/protobuf/util/type_resolver.h> |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 45 | #include <google/protobuf/wire_format_lite.h> |
| 46 | |
Feng Xiao | a0cecfd | 2015-12-14 18:33:38 -0800 | [diff] [blame] | 47 | #include "third_party/jsoncpp/json.h" |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 48 | |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 49 | namespace conformance { |
| 50 | class ConformanceRequest; |
| 51 | class ConformanceResponse; |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 52 | class TestAllTypes; |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 53 | } // namespace conformance |
| 54 | |
| 55 | namespace google { |
| 56 | namespace protobuf { |
| 57 | |
| 58 | class ConformanceTestRunner { |
| 59 | public: |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 60 | virtual ~ConformanceTestRunner() {} |
| 61 | |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 62 | // Call to run a single conformance test. |
| 63 | // |
| 64 | // "input" is a serialized conformance.ConformanceRequest. |
| 65 | // "output" should be set to a serialized conformance.ConformanceResponse. |
| 66 | // |
| 67 | // If there is any error in running the test itself, set "runtime_error" in |
| 68 | // the response. |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 69 | virtual void RunTest(const std::string& test_name, |
| 70 | const std::string& input, |
| 71 | std::string* output) = 0; |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 72 | }; |
| 73 | |
| 74 | // Class representing the test suite itself. To run it, implement your own |
| 75 | // class derived from ConformanceTestRunner and then write code like: |
| 76 | // |
| 77 | // class MyConformanceTestRunner : public ConformanceTestRunner { |
| 78 | // public: |
| 79 | // virtual void RunTest(...) { |
| 80 | // // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. |
| 81 | // } |
| 82 | // }; |
| 83 | // |
| 84 | // int main() { |
| 85 | // MyConformanceTestRunner runner; |
| 86 | // google::protobuf::ConformanceTestSuite suite; |
| 87 | // |
| 88 | // std::string output; |
| 89 | // suite.RunSuite(&runner, &output); |
| 90 | // } |
| 91 | // |
| 92 | class ConformanceTestSuite { |
| 93 | public: |
| 94 | ConformanceTestSuite() : verbose_(false) {} |
| 95 | |
Josh Haberman | 181c7f2 | 2015-07-15 11:05:10 -0700 | [diff] [blame] | 96 | void SetVerbose(bool verbose) { verbose_ = verbose; } |
| 97 | |
Josh Haberman | d2b6738 | 2015-06-03 12:04:35 -0700 | [diff] [blame] | 98 | // Sets the list of tests that are expected to fail when RunSuite() is called. |
| 99 | // RunSuite() will fail unless the set of failing tests is exactly the same |
| 100 | // as this list. |
| 101 | void SetFailureList(const std::vector<std::string>& failure_list); |
| 102 | |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 103 | // Run all the conformance tests against the given test runner. |
| 104 | // Test output will be stored in "output". |
Josh Haberman | d2b6738 | 2015-06-03 12:04:35 -0700 | [diff] [blame] | 105 | // |
| 106 | // Returns true if the set of failing tests was exactly the same as the |
| 107 | // failure list. If SetFailureList() was not called, returns true if all |
| 108 | // tests passed. |
| 109 | bool RunSuite(ConformanceTestRunner* runner, std::string* output); |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 110 | |
| 111 | private: |
Josh Haberman | d2b6738 | 2015-06-03 12:04:35 -0700 | [diff] [blame] | 112 | void ReportSuccess(const std::string& test_name); |
Josh Haberman | b0500b3 | 2015-07-10 16:36:59 -0700 | [diff] [blame] | 113 | void ReportFailure(const string& test_name, |
| 114 | const conformance::ConformanceRequest& request, |
| 115 | const conformance::ConformanceResponse& response, |
| 116 | const char* fmt, ...); |
| 117 | void ReportSkip(const string& test_name, |
| 118 | const conformance::ConformanceRequest& request, |
| 119 | const conformance::ConformanceResponse& response); |
Josh Haberman | 23bf3b5 | 2015-06-04 15:04:00 -0700 | [diff] [blame] | 120 | void RunTest(const std::string& test_name, |
| 121 | const conformance::ConformanceRequest& request, |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 122 | conformance::ConformanceResponse* response); |
Josh Haberman | b0500b3 | 2015-07-10 16:36:59 -0700 | [diff] [blame] | 123 | void RunValidInputTest(const string& test_name, const string& input, |
| 124 | conformance::WireFormat input_format, |
| 125 | const string& equivalent_text_format, |
| 126 | conformance::WireFormat requested_output); |
| 127 | void RunValidJsonTest(const string& test_name, const string& input_json, |
| 128 | const string& equivalent_text_format); |
Feng Xiao | e841bac | 2015-12-11 17:09:20 -0800 | [diff] [blame] | 129 | void RunValidJsonTestWithProtobufInput(const string& test_name, |
| 130 | const conformance::TestAllTypes& input, |
| 131 | const string& equivalent_text_format); |
| 132 | |
| 133 | typedef std::function<bool(const Json::Value&)> Validator; |
| 134 | void RunValidJsonTestWithValidator(const string& test_name, |
| 135 | const string& input_json, |
| 136 | const Validator& validator); |
| 137 | void ExpectParseFailureForJson(const string& test_name, |
| 138 | const string& input_json); |
| 139 | void ExpectSerializeFailureForJson(const string& test_name, |
| 140 | const string& text_format); |
Josh Haberman | d2b6738 | 2015-06-03 12:04:35 -0700 | [diff] [blame] | 141 | void ExpectParseFailureForProto(const std::string& proto, |
| 142 | const std::string& test_name); |
| 143 | void ExpectHardParseFailureForProto(const std::string& proto, |
| 144 | const std::string& test_name); |
| 145 | void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); |
| 146 | bool CheckSetEmpty(const set<string>& set_to_check, const char* msg); |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 147 | ConformanceTestRunner* runner_; |
| 148 | int successes_; |
Josh Haberman | b0500b3 | 2015-07-10 16:36:59 -0700 | [diff] [blame] | 149 | int expected_failures_; |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 150 | bool verbose_; |
| 151 | std::string output_; |
Josh Haberman | d2b6738 | 2015-06-03 12:04:35 -0700 | [diff] [blame] | 152 | |
| 153 | // The set of test names that are expected to fail in this run, but haven't |
| 154 | // failed yet. |
| 155 | std::set<std::string> expected_to_fail_; |
| 156 | |
| 157 | // The set of test names that have been run. Used to ensure that there are no |
| 158 | // duplicate names in the suite. |
| 159 | std::set<std::string> test_names_; |
| 160 | |
| 161 | // The set of tests that failed, but weren't expected to. |
| 162 | std::set<std::string> unexpected_failing_tests_; |
| 163 | |
| 164 | // The set of tests that succeeded, but weren't expected to. |
| 165 | std::set<std::string> unexpected_succeeding_tests_; |
Josh Haberman | b0500b3 | 2015-07-10 16:36:59 -0700 | [diff] [blame] | 166 | |
| 167 | // The set of tests that the testee opted out of; |
| 168 | std::set<std::string> skipped_; |
| 169 | |
| 170 | google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver> |
| 171 | type_resolver_; |
| 172 | std::string type_url_; |
Josh Haberman | 4e63b52 | 2015-04-14 13:45:39 -0700 | [diff] [blame] | 173 | }; |
| 174 | |
| 175 | } // namespace protobuf |
| 176 | } // namespace google |
| 177 | |
| 178 | #endif // CONFORMANCE_CONFORMANCE_TEST_H |