Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | /* Copyright (c) 2015, Google Inc. |
| 2 | * |
| 3 | * Permission to use, copy, modify, and/or distribute this software for any |
| 4 | * purpose with or without fee is hereby granted, provided that the above |
| 5 | * copyright notice and this permission notice appear in all copies. |
| 6 | * |
| 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 | |
| 15 | #ifndef OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H |
| 16 | #define OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H |
| 17 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 18 | #include <openssl/base.h> |
| 19 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 20 | #include <stdint.h> |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 21 | |
David Benjamin | 6e899c7 | 2016-06-09 18:02:18 -0400 | [diff] [blame] | 22 | OPENSSL_MSVC_PRAGMA(warning(push)) |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 23 | OPENSSL_MSVC_PRAGMA(warning(disable : 4702)) |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 24 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 25 | #include <functional> |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 26 | #include <map> |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 27 | #include <memory> |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 28 | #include <set> |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 29 | #include <string> |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 30 | #include <vector> |
| 31 | |
David Benjamin | 6e899c7 | 2016-06-09 18:02:18 -0400 | [diff] [blame] | 32 | OPENSSL_MSVC_PRAGMA(warning(pop)) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 33 | |
| 34 | // File-based test framework. |
| 35 | // |
| 36 | // This module provides a file-based test framework. The file format is based on |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 37 | // that of OpenSSL upstream's evp_test and BoringSSL's aead_test. NIST CAVP test |
| 38 | // vector files are also supported. Each input file is a sequence of attributes, |
| 39 | // instructions and blank lines. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 40 | // |
| 41 | // Each attribute has the form: |
| 42 | // |
| 43 | // Name = Value |
| 44 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 45 | // Instructions are enclosed in square brackets and may appear without a value: |
| 46 | // |
| 47 | // [Name = Value] |
| 48 | // |
| 49 | // or |
| 50 | // |
| 51 | // [Name] |
| 52 | // |
| 53 | // Commas in instruction lines are treated as separate instructions. Thus this: |
| 54 | // |
| 55 | // [Name1,Name2] |
| 56 | // |
| 57 | // is the same as: |
| 58 | // |
| 59 | // [Name1] |
| 60 | // [Name2] |
| 61 | // |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 62 | // Either '=' or ':' may be used to delimit the name from the value. Both the |
| 63 | // name and value have leading and trailing spaces stripped. |
| 64 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 65 | // Each file contains a number of instruction blocks and test cases. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 66 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 67 | // An instruction block is a sequence of instructions followed by a blank line. |
| 68 | // Instructions apply to all test cases following its appearance, until the next |
| 69 | // instruction block. Instructions are unordered. |
| 70 | // |
| 71 | // A test is a sequence of one or more attributes followed by a blank line. For |
| 72 | // tests that process multiple kinds of test cases, the first attribute is |
| 73 | // parsed out as the test's type and parameter. Otherwise, attributes are |
| 74 | // unordered. The first attribute is also included in the set of attributes, so |
| 75 | // tests which do not dispatch may ignore this mechanism. |
| 76 | // |
| 77 | // Additional blank lines and lines beginning with # are ignored. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 78 | // |
| 79 | // Functions in this module freely output to |stderr| on failure. Tests should |
| 80 | // also do so, and it is recommended they include the corresponding test's line |
| 81 | // number in any output. |PrintLine| does this automatically. |
| 82 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 83 | // Each attribute in a test and all instructions applying to it must be |
| 84 | // consumed. When a test completes, if any attributes or insturctions haven't |
| 85 | // been processed, the framework reports an error. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 86 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 87 | class FileTest; |
| 88 | typedef bool (*FileTestFunc)(FileTest *t, void *arg); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 89 | |
| 90 | class FileTest { |
| 91 | public: |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 92 | enum ReadResult { |
| 93 | kReadSuccess, |
| 94 | kReadEOF, |
| 95 | kReadError, |
| 96 | }; |
| 97 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 98 | class LineReader { |
| 99 | public: |
| 100 | virtual ~LineReader() {} |
| 101 | virtual ReadResult ReadLine(char *out, size_t len) = 0; |
| 102 | }; |
| 103 | |
| 104 | struct Options { |
| 105 | // path is the path to the input file. |
| 106 | const char *path = nullptr; |
| 107 | // callback is called for each test. It should get the parameters from this |
| 108 | // object and signal any errors by returning false. |
| 109 | FileTestFunc callback = nullptr; |
| 110 | // arg is an opaque pointer that is passed to |callback|. |
| 111 | void *arg = nullptr; |
| 112 | // silent suppressed the "PASS" string that is otherwise printed after |
| 113 | // successful runs. |
| 114 | bool silent = false; |
| 115 | // comment_callback is called after each comment in the input is parsed. |
| 116 | std::function<void(const std::string&)> comment_callback; |
Robert Sloan | 0db7f54 | 2018-01-16 15:48:33 -0800 | [diff] [blame^] | 117 | // is_kas_test is true if a NIST “KAS” test is being parsed. These tests |
| 118 | // are inconsistent with the other NIST files to such a degree that they |
| 119 | // need their own boolean. |
| 120 | bool is_kas_test = false; |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 121 | }; |
| 122 | |
| 123 | explicit FileTest(std::unique_ptr<LineReader> reader, |
Robert Sloan | 0db7f54 | 2018-01-16 15:48:33 -0800 | [diff] [blame^] | 124 | std::function<void(const std::string &)> comment_callback, |
| 125 | bool is_kas_test); |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 126 | ~FileTest(); |
| 127 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 128 | // ReadNext reads the next test from the file. It returns |kReadSuccess| if |
| 129 | // successfully reading a test and |kReadEOF| at the end of the file. On |
| 130 | // error or if the previous test had unconsumed attributes, it returns |
| 131 | // |kReadError|. |
| 132 | ReadResult ReadNext(); |
| 133 | |
| 134 | // PrintLine is a variant of printf which prepends the line number and appends |
| 135 | // a trailing newline. |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 136 | void PrintLine(const char *format, ...) OPENSSL_PRINTF_FORMAT_FUNC(2, 3); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 137 | |
| 138 | unsigned start_line() const { return start_line_; } |
| 139 | |
| 140 | // GetType returns the name of the first attribute of the current test. |
| 141 | const std::string &GetType(); |
| 142 | // GetParameter returns the value of the first attribute of the current test. |
| 143 | const std::string &GetParameter(); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 144 | |
| 145 | // HasAttribute returns true if the current test has an attribute named |key|. |
| 146 | bool HasAttribute(const std::string &key); |
| 147 | |
| 148 | // GetAttribute looks up the attribute with key |key|. It sets |*out_value| to |
| 149 | // the value and returns true if it exists and returns false with an error to |
| 150 | // |stderr| otherwise. |
| 151 | bool GetAttribute(std::string *out_value, const std::string &key); |
| 152 | |
| 153 | // GetAttributeOrDie looks up the attribute with key |key| and aborts if it is |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 154 | // missing. It should only be used after a |HasAttribute| call. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 155 | const std::string &GetAttributeOrDie(const std::string &key); |
| 156 | |
| 157 | // GetBytes looks up the attribute with key |key| and decodes it as a byte |
| 158 | // string. On success, it writes the result to |*out| and returns |
| 159 | // true. Otherwise it returns false with an error to |stderr|. The value may |
| 160 | // be either a hexadecimal string or a quoted ASCII string. It returns true on |
| 161 | // success and returns false with an error to |stderr| on failure. |
| 162 | bool GetBytes(std::vector<uint8_t> *out, const std::string &key); |
| 163 | |
| 164 | // ExpectBytesEqual returns true if |expected| and |actual| are equal. |
| 165 | // Otherwise, it returns false and prints a message to |stderr|. |
| 166 | bool ExpectBytesEqual(const uint8_t *expected, size_t expected_len, |
| 167 | const uint8_t *actual, size_t actual_len); |
| 168 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 169 | // AtNewInstructionBlock returns true if the current test was immediately |
| 170 | // preceded by an instruction block. |
| 171 | bool IsAtNewInstructionBlock() const; |
| 172 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 173 | // HasInstruction returns true if the current test has an instruction. |
| 174 | bool HasInstruction(const std::string &key); |
| 175 | |
| 176 | // GetInstruction looks up the instruction with key |key|. It sets |
| 177 | // |*out_value| to the value (empty string if the instruction has no value) |
| 178 | // and returns true if it exists and returns false with an error to |stderr| |
| 179 | // otherwise. |
| 180 | bool GetInstruction(std::string *out_value, const std::string &key); |
| 181 | |
| 182 | // CurrentTestToString returns the file content parsed for the current test. |
| 183 | // If the current test was preceded by an instruction block, the return test |
| 184 | // case is preceded by the instruction block and a single blank line. All |
| 185 | // other blank or comment lines are omitted. |
| 186 | const std::string &CurrentTestToString() const; |
| 187 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 188 | // InjectInstruction adds a key value pair to the most recently parsed set of |
| 189 | // instructions. |
| 190 | void InjectInstruction(const std::string &key, const std::string &value); |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 191 | |
Robert Sloan | 927a495 | 2017-07-03 11:25:09 -0700 | [diff] [blame] | 192 | // SkipCurrent passes the current test case. Unused attributes are ignored. |
| 193 | void SkipCurrent(); |
| 194 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 195 | private: |
| 196 | void ClearTest(); |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 197 | void ClearInstructions(); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 198 | void OnKeyUsed(const std::string &key); |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 199 | void OnInstructionUsed(const std::string &key); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 200 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 201 | std::unique_ptr<LineReader> reader_; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 202 | // line_ is the number of lines read. |
| 203 | unsigned line_ = 0; |
| 204 | |
| 205 | // start_line_ is the line number of the first attribute of the test. |
| 206 | unsigned start_line_ = 0; |
| 207 | // type_ is the name of the first attribute of the test. |
| 208 | std::string type_; |
| 209 | // parameter_ is the value of the first attribute. |
| 210 | std::string parameter_; |
| 211 | // attributes_ contains all attributes in the test, including the first. |
| 212 | std::map<std::string, std::string> attributes_; |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 213 | // instructions_ contains all instructions in scope for the test. |
| 214 | std::map<std::string, std::string> instructions_; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 215 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 216 | // unused_attributes_ is the set of attributes that have not been queried. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 217 | std::set<std::string> unused_attributes_; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 218 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 219 | // unused_instructions_ is the set of instructions that have not been queried. |
| 220 | std::set<std::string> unused_instructions_; |
| 221 | |
| 222 | std::string current_test_; |
| 223 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 224 | bool is_at_new_instruction_block_ = false; |
Robert Sloan | 0db7f54 | 2018-01-16 15:48:33 -0800 | [diff] [blame^] | 225 | bool seen_non_comment_ = false; |
| 226 | bool is_kas_test_ = false; |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 227 | |
| 228 | // comment_callback_, if set, is a callback function that is called with the |
| 229 | // contents of each comment as they are parsed. |
| 230 | std::function<void(const std::string&)> comment_callback_; |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 231 | |
| 232 | FileTest(const FileTest &) = delete; |
| 233 | FileTest &operator=(const FileTest &) = delete; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 234 | }; |
| 235 | |
| 236 | // FileTestMain runs a file-based test out of |path| and returns an exit code |
| 237 | // suitable to return out of |main|. |run_test| should return true on pass and |
| 238 | // false on failure. FileTestMain also implements common handling of the 'Error' |
| 239 | // attribute. A test with that attribute is expected to fail. The value of the |
| 240 | // attribute is the reason string of the expected OpenSSL error code. |
| 241 | // |
| 242 | // Tests are guaranteed to run serially and may affect global state if need be. |
| 243 | // It is legal to use "tests" which, for example, import a private key into a |
| 244 | // list of keys. This may be used to initialize a shared set of keys for many |
| 245 | // tests. However, if one test fails, the framework will continue to run |
| 246 | // subsequent tests. |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 247 | int FileTestMain(FileTestFunc run_test, void *arg, const char *path); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 248 | |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame] | 249 | // FileTestMain accepts a larger number of options via a struct. |
| 250 | int FileTestMain(const FileTest::Options &opts); |
| 251 | |
| 252 | // FileTestGTest behaves like FileTestMain, but for GTest. |path| must be the |
| 253 | // name of a test file embedded in the test binary. |
| 254 | void FileTestGTest(const char *path, std::function<void(FileTest *)> run_test); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 255 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 256 | #endif // OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H |