Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | /* Copyright (c) 2015, Google Inc. |
| 2 | * |
| 3 | * Permission to use, copy, modify, and/or distribute this software for any |
| 4 | * purpose with or without fee is hereby granted, provided that the above |
| 5 | * copyright notice and this permission notice appear in all copies. |
| 6 | * |
| 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 | |
| 15 | #ifndef OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H |
| 16 | #define OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H |
| 17 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 18 | #include <openssl/base.h> |
| 19 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 20 | #include <stdint.h> |
| 21 | #include <stdio.h> |
| 22 | |
David Benjamin | 6e899c7 | 2016-06-09 18:02:18 -0400 | [diff] [blame] | 23 | OPENSSL_MSVC_PRAGMA(warning(push)) |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 24 | OPENSSL_MSVC_PRAGMA(warning(disable : 4702)) |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 25 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 26 | #include <map> |
| 27 | #include <set> |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 28 | #include <string> |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 29 | #include <vector> |
| 30 | |
David Benjamin | 6e899c7 | 2016-06-09 18:02:18 -0400 | [diff] [blame] | 31 | OPENSSL_MSVC_PRAGMA(warning(pop)) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 32 | |
| 33 | // File-based test framework. |
| 34 | // |
| 35 | // This module provides a file-based test framework. The file format is based on |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 36 | // that of OpenSSL upstream's evp_test and BoringSSL's aead_test. NIST CAVP test |
| 37 | // vector files are also supported. Each input file is a sequence of attributes, |
| 38 | // instructions and blank lines. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 39 | // |
| 40 | // Each attribute has the form: |
| 41 | // |
| 42 | // Name = Value |
| 43 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 44 | // Instructions are enclosed in square brackets and may appear without a value: |
| 45 | // |
| 46 | // [Name = Value] |
| 47 | // |
| 48 | // or |
| 49 | // |
| 50 | // [Name] |
| 51 | // |
| 52 | // Commas in instruction lines are treated as separate instructions. Thus this: |
| 53 | // |
| 54 | // [Name1,Name2] |
| 55 | // |
| 56 | // is the same as: |
| 57 | // |
| 58 | // [Name1] |
| 59 | // [Name2] |
| 60 | // |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 61 | // Either '=' or ':' may be used to delimit the name from the value. Both the |
| 62 | // name and value have leading and trailing spaces stripped. |
| 63 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 64 | // Each file contains a number of instruction blocks and test cases. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 65 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 66 | // An instruction block is a sequence of instructions followed by a blank line. |
| 67 | // Instructions apply to all test cases following its appearance, until the next |
| 68 | // instruction block. Instructions are unordered. |
| 69 | // |
| 70 | // A test is a sequence of one or more attributes followed by a blank line. For |
| 71 | // tests that process multiple kinds of test cases, the first attribute is |
| 72 | // parsed out as the test's type and parameter. Otherwise, attributes are |
| 73 | // unordered. The first attribute is also included in the set of attributes, so |
| 74 | // tests which do not dispatch may ignore this mechanism. |
| 75 | // |
| 76 | // Additional blank lines and lines beginning with # are ignored. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 77 | // |
| 78 | // Functions in this module freely output to |stderr| on failure. Tests should |
| 79 | // also do so, and it is recommended they include the corresponding test's line |
| 80 | // number in any output. |PrintLine| does this automatically. |
| 81 | // |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 82 | // Each attribute in a test and all instructions applying to it must be |
| 83 | // consumed. When a test completes, if any attributes or insturctions haven't |
| 84 | // been processed, the framework reports an error. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 85 | |
| 86 | |
| 87 | class FileTest { |
| 88 | public: |
| 89 | explicit FileTest(const char *path); |
| 90 | ~FileTest(); |
| 91 | |
| 92 | // is_open returns true if the file was successfully opened. |
| 93 | bool is_open() const { return file_ != nullptr; } |
| 94 | |
| 95 | enum ReadResult { |
| 96 | kReadSuccess, |
| 97 | kReadEOF, |
| 98 | kReadError, |
| 99 | }; |
| 100 | |
| 101 | // ReadNext reads the next test from the file. It returns |kReadSuccess| if |
| 102 | // successfully reading a test and |kReadEOF| at the end of the file. On |
| 103 | // error or if the previous test had unconsumed attributes, it returns |
| 104 | // |kReadError|. |
| 105 | ReadResult ReadNext(); |
| 106 | |
| 107 | // PrintLine is a variant of printf which prepends the line number and appends |
| 108 | // a trailing newline. |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 109 | void PrintLine(const char *format, ...) OPENSSL_PRINTF_FORMAT_FUNC(2, 3); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 110 | |
| 111 | unsigned start_line() const { return start_line_; } |
| 112 | |
| 113 | // GetType returns the name of the first attribute of the current test. |
| 114 | const std::string &GetType(); |
| 115 | // GetParameter returns the value of the first attribute of the current test. |
| 116 | const std::string &GetParameter(); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 117 | |
| 118 | // HasAttribute returns true if the current test has an attribute named |key|. |
| 119 | bool HasAttribute(const std::string &key); |
| 120 | |
| 121 | // GetAttribute looks up the attribute with key |key|. It sets |*out_value| to |
| 122 | // the value and returns true if it exists and returns false with an error to |
| 123 | // |stderr| otherwise. |
| 124 | bool GetAttribute(std::string *out_value, const std::string &key); |
| 125 | |
| 126 | // GetAttributeOrDie looks up the attribute with key |key| and aborts if it is |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 127 | // missing. It should only be used after a |HasAttribute| call. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 128 | const std::string &GetAttributeOrDie(const std::string &key); |
| 129 | |
| 130 | // GetBytes looks up the attribute with key |key| and decodes it as a byte |
| 131 | // string. On success, it writes the result to |*out| and returns |
| 132 | // true. Otherwise it returns false with an error to |stderr|. The value may |
| 133 | // be either a hexadecimal string or a quoted ASCII string. It returns true on |
| 134 | // success and returns false with an error to |stderr| on failure. |
| 135 | bool GetBytes(std::vector<uint8_t> *out, const std::string &key); |
| 136 | |
| 137 | // ExpectBytesEqual returns true if |expected| and |actual| are equal. |
| 138 | // Otherwise, it returns false and prints a message to |stderr|. |
| 139 | bool ExpectBytesEqual(const uint8_t *expected, size_t expected_len, |
| 140 | const uint8_t *actual, size_t actual_len); |
| 141 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 142 | // HasInstruction returns true if the current test has an instruction. |
| 143 | bool HasInstruction(const std::string &key); |
| 144 | |
| 145 | // GetInstruction looks up the instruction with key |key|. It sets |
| 146 | // |*out_value| to the value (empty string if the instruction has no value) |
| 147 | // and returns true if it exists and returns false with an error to |stderr| |
| 148 | // otherwise. |
| 149 | bool GetInstruction(std::string *out_value, const std::string &key); |
| 150 | |
| 151 | // CurrentTestToString returns the file content parsed for the current test. |
| 152 | // If the current test was preceded by an instruction block, the return test |
| 153 | // case is preceded by the instruction block and a single blank line. All |
| 154 | // other blank or comment lines are omitted. |
| 155 | const std::string &CurrentTestToString() const; |
| 156 | |
| 157 | void SetIgnoreUnusedAttributes(bool ignore); |
| 158 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 159 | private: |
| 160 | void ClearTest(); |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 161 | void ClearInstructions(); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 162 | void OnKeyUsed(const std::string &key); |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 163 | void OnInstructionUsed(const std::string &key); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 164 | |
| 165 | FILE *file_ = nullptr; |
| 166 | // line_ is the number of lines read. |
| 167 | unsigned line_ = 0; |
| 168 | |
| 169 | // start_line_ is the line number of the first attribute of the test. |
| 170 | unsigned start_line_ = 0; |
| 171 | // type_ is the name of the first attribute of the test. |
| 172 | std::string type_; |
| 173 | // parameter_ is the value of the first attribute. |
| 174 | std::string parameter_; |
| 175 | // attributes_ contains all attributes in the test, including the first. |
| 176 | std::map<std::string, std::string> attributes_; |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 177 | // instructions_ contains all instructions in scope for the test. |
| 178 | std::map<std::string, std::string> instructions_; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 179 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 180 | // unused_attributes_ is the set of attributes that have not been queried. |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 181 | std::set<std::string> unused_attributes_; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 182 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 183 | // unused_instructions_ is the set of instructions that have not been queried. |
| 184 | std::set<std::string> unused_instructions_; |
| 185 | |
| 186 | std::string current_test_; |
| 187 | |
| 188 | bool ignore_unused_attributes_ = false; |
| 189 | |
| 190 | FileTest(const FileTest &) = delete; |
| 191 | FileTest &operator=(const FileTest &) = delete; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 192 | }; |
| 193 | |
| 194 | // FileTestMain runs a file-based test out of |path| and returns an exit code |
| 195 | // suitable to return out of |main|. |run_test| should return true on pass and |
| 196 | // false on failure. FileTestMain also implements common handling of the 'Error' |
| 197 | // attribute. A test with that attribute is expected to fail. The value of the |
| 198 | // attribute is the reason string of the expected OpenSSL error code. |
| 199 | // |
| 200 | // Tests are guaranteed to run serially and may affect global state if need be. |
| 201 | // It is legal to use "tests" which, for example, import a private key into a |
| 202 | // list of keys. This may be used to initialize a shared set of keys for many |
| 203 | // tests. However, if one test fails, the framework will continue to run |
| 204 | // subsequent tests. |
| 205 | int FileTestMain(bool (*run_test)(FileTest *t, void *arg), void *arg, |
| 206 | const char *path); |
| 207 | |
Robert Sloan | 2424d84 | 2017-05-01 07:46:28 -0700 | [diff] [blame] | 208 | // FileTestMainSilent behaves like FileTestMain but does not print a final |
| 209 | // FAIL/PASS message to stdout. |
| 210 | int FileTestMainSilent(bool (*run_test)(FileTest *t, void *arg), void *arg, |
| 211 | const char *path); |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 212 | |
| 213 | #endif /* OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H */ |