Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 1 | /*------------------------------------------------------------------------- |
| 2 | * drawElements Quality Program Test Executor |
| 3 | * ------------------------------------------ |
| 4 | * |
| 5 | * Copyright 2014 The Android Open Source Project |
| 6 | * |
| 7 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 8 | * you may not use this file except in compliance with the License. |
| 9 | * You may obtain a copy of the License at |
| 10 | * |
| 11 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | * |
| 13 | * Unless required by applicable law or agreed to in writing, software |
| 14 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | * See the License for the specific language governing permissions and |
| 17 | * limitations under the License. |
| 18 | * |
| 19 | *//*! |
| 20 | * \file |
| 21 | * \brief Command line test executor. |
| 22 | *//*--------------------------------------------------------------------*/ |
| 23 | |
| 24 | #include "xeBatchExecutor.hpp" |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 25 | #include "xeLocalTcpIpLink.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 26 | #include "xeTcpIpLink.hpp" |
| 27 | #include "xeTestCaseListParser.hpp" |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 28 | #include "xeTestLogWriter.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 29 | #include "xeTestResultParser.hpp" |
| 30 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 31 | #include "deCommandLine.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 32 | #include "deDirectoryIterator.hpp" |
| 33 | #include "deStringUtil.hpp" |
Pyry Haulos | 674ddbb | 2017-03-30 16:42:23 -0700 | [diff] [blame] | 34 | #include "deUniquePtr.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 35 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 36 | #include "deString.h" |
| 37 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 38 | #include <algorithm> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 39 | #include <cstdio> |
| 40 | #include <cstdlib> |
| 41 | #include <fstream> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 42 | #include <iostream> |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 43 | #include <memory> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 44 | #include <sstream> |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 45 | #include <string> |
| 46 | #include <vector> |
| 47 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 48 | #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32) |
| 49 | # include <signal.h> |
| 50 | #endif |
| 51 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 52 | using std::vector; |
| 53 | using std::string; |
| 54 | |
| 55 | namespace |
| 56 | { |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 57 | |
| 58 | // Command line arguments. |
| 59 | namespace opt |
| 60 | { |
| 61 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 62 | DE_DECLARE_COMMAND_LINE_OPT(StartServer, string); |
| 63 | DE_DECLARE_COMMAND_LINE_OPT(Host, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 64 | DE_DECLARE_COMMAND_LINE_OPT(Port, int); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 65 | DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string); |
| 66 | DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>); |
| 67 | DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>); |
| 68 | DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string); |
| 69 | DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string); |
| 70 | DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 71 | DE_DECLARE_COMMAND_LINE_OPT(Summary, bool); |
| 72 | |
| 73 | // TargetConfiguration |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 74 | DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string); |
| 75 | DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string); |
| 76 | DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 77 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 78 | void parseCommaSeparatedList (const char* src, vector<string>* dst) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 79 | { |
| 80 | std::istringstream inStr (src); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 81 | string comp; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 82 | |
| 83 | while (std::getline(inStr, comp, ',')) |
| 84 | dst->push_back(comp); |
| 85 | } |
| 86 | |
| 87 | void registerOptions (de::cmdline::Parser& parser) |
| 88 | { |
| 89 | using de::cmdline::Option; |
| 90 | using de::cmdline::NamedValue; |
| 91 | |
| 92 | static const NamedValue<bool> s_yesNo[] = |
| 93 | { |
| 94 | { "yes", true }, |
| 95 | { "no", false } |
| 96 | }; |
| 97 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 98 | parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.") |
| 99 | << Option<Host> ("c", "connect", "Connect to host. Address of the execserver.") |
| 100 | << Option<Port> ("p", "port", "TCP port of the execserver.", "50016") |
| 101 | << Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".") |
| 102 | << Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList) |
| 103 | << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "") |
| 104 | << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.") |
| 105 | << Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa") |
| 106 | << Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt") |
| 107 | << Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes") |
| 108 | << Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>") |
| 109 | << Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".") |
| 110 | << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", ""); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 111 | } |
| 112 | |
| 113 | } // opt |
| 114 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 115 | enum RunMode |
| 116 | { |
| 117 | RUNMODE_CONNECT, |
| 118 | RUNMODE_START_SERVER |
| 119 | }; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 120 | |
| 121 | struct CommandLine |
| 122 | { |
| 123 | CommandLine (void) |
| 124 | : port (0) |
| 125 | , summary (false) |
| 126 | { |
| 127 | } |
| 128 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 129 | xe::TargetConfiguration targetCfg; |
| 130 | RunMode runMode; |
| 131 | string serverBinOrAddress; |
| 132 | int port; |
| 133 | string caseListDir; |
| 134 | vector<string> testset; |
| 135 | vector<string> exclude; |
| 136 | string inFile; |
| 137 | string outFile; |
| 138 | string infoFile; |
| 139 | bool summary; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 140 | }; |
| 141 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 142 | bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 143 | { |
| 144 | de::cmdline::Parser parser; |
| 145 | de::cmdline::CommandLine opts; |
| 146 | |
| 147 | XE_CHECK(argc >= 1); |
| 148 | |
| 149 | opt::registerOptions(parser); |
| 150 | |
| 151 | if (!parser.parse(argc-1, argv+1, &opts, std::cerr)) |
| 152 | { |
| 153 | std::cout << argv[0] << " [options]\n"; |
| 154 | parser.help(std::cout); |
| 155 | return false; |
| 156 | } |
| 157 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 158 | if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>()) |
| 159 | { |
| 160 | std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl; |
| 161 | return false; |
| 162 | } |
| 163 | else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>()) |
| 164 | { |
| 165 | std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl; |
| 166 | return false; |
| 167 | } |
| 168 | |
| 169 | if (!opts.hasOption<opt::TestSet>()) |
| 170 | { |
| 171 | std::cout << "Invalid command line arguments. --testset not defined." << std::endl; |
| 172 | return false; |
| 173 | } |
| 174 | |
| 175 | if (opts.hasOption<opt::StartServer>()) |
| 176 | { |
| 177 | cmdLine.runMode = RUNMODE_START_SERVER; |
| 178 | cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>(); |
| 179 | } |
| 180 | else |
| 181 | { |
| 182 | cmdLine.runMode = RUNMODE_CONNECT; |
| 183 | cmdLine.serverBinOrAddress = opts.getOption<opt::Host>(); |
| 184 | } |
| 185 | |
| 186 | if (opts.hasOption<opt::ContinueFile>()) |
| 187 | { |
| 188 | cmdLine.inFile = opts.getOption<opt::ContinueFile>(); |
| 189 | |
| 190 | if (cmdLine.inFile.empty()) |
| 191 | { |
| 192 | std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl; |
| 193 | return false; |
| 194 | } |
| 195 | } |
| 196 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 197 | cmdLine.port = opts.getOption<opt::Port>(); |
| 198 | cmdLine.caseListDir = opts.getOption<opt::CaseListDir>(); |
| 199 | cmdLine.testset = opts.getOption<opt::TestSet>(); |
| 200 | cmdLine.exclude = opts.getOption<opt::ExcludeSet>(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 201 | cmdLine.outFile = opts.getOption<opt::TestLogFile>(); |
| 202 | cmdLine.infoFile = opts.getOption<opt::InfoLogFile>(); |
| 203 | cmdLine.summary = opts.getOption<opt::Summary>(); |
| 204 | cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>(); |
| 205 | cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>(); |
| 206 | cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>(); |
| 207 | |
| 208 | return true; |
| 209 | } |
| 210 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 211 | bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 212 | { |
| 213 | int ptrnPos = 0; |
| 214 | int casePos = 0; |
| 215 | |
| 216 | for (;;) |
| 217 | { |
| 218 | char c = casePath[casePos]; |
| 219 | char p = pattern[ptrnPos]; |
| 220 | |
| 221 | if (p == '*') |
| 222 | { |
| 223 | /* Recurse to rest of positions. */ |
| 224 | int next = casePos; |
| 225 | for (;;) |
| 226 | { |
| 227 | if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup)) |
| 228 | return DE_TRUE; |
| 229 | |
| 230 | if (casePath[next] == 0) |
| 231 | return DE_FALSE; /* No match found. */ |
| 232 | else |
| 233 | next += 1; |
| 234 | } |
| 235 | DE_ASSERT(DE_FALSE); |
| 236 | } |
| 237 | else if (c == 0 && p == 0) |
| 238 | return true; |
| 239 | else if (c == 0) |
| 240 | { |
| 241 | /* Incomplete match is ok for test groups. */ |
| 242 | return isTestGroup; |
| 243 | } |
| 244 | else if (c != p) |
| 245 | return false; |
| 246 | |
| 247 | casePos += 1; |
| 248 | ptrnPos += 1; |
| 249 | } |
| 250 | |
| 251 | DE_ASSERT(false); |
| 252 | return false; |
| 253 | } |
| 254 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 255 | void readCaseList (xe::TestGroup* root, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 256 | { |
| 257 | xe::TestCaseListParser caseListParser; |
| 258 | std::ifstream in (filename, std::ios_base::binary); |
| 259 | deUint8 buf[1024]; |
| 260 | |
| 261 | XE_CHECK(in.good()); |
| 262 | |
| 263 | caseListParser.init(root); |
| 264 | |
| 265 | for (;;) |
| 266 | { |
| 267 | in.read((char*)&buf[0], sizeof(buf)); |
| 268 | int numRead = (int)in.gcount(); |
| 269 | |
| 270 | if (numRead > 0) |
| 271 | caseListParser.parse(&buf[0], numRead); |
| 272 | |
| 273 | if (numRead < (int)sizeof(buf)) |
| 274 | break; // EOF |
| 275 | } |
| 276 | } |
| 277 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 278 | void readCaseLists (xe::TestRoot& root, const char* caseListDir) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 279 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 280 | int testCaseListCount = 0; |
| 281 | de::DirectoryIterator iter (caseListDir); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 282 | |
| 283 | for (; iter.hasItem(); iter.next()) |
| 284 | { |
| 285 | de::FilePath item = iter.getItem(); |
| 286 | |
| 287 | if (item.getType() == de::FilePath::TYPE_FILE) |
| 288 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 289 | string baseName = item.getBaseName(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 290 | if (baseName.find("-cases.xml") == baseName.length()-10) |
| 291 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 292 | string packageName = baseName.substr(0, baseName.length()-10); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 293 | xe::TestGroup* package = root.createGroup(packageName.c_str(), ""); |
| 294 | |
| 295 | readCaseList(package, item.getPath()); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 296 | testCaseListCount++; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 297 | } |
| 298 | } |
| 299 | } |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 300 | |
| 301 | if (testCaseListCount == 0) |
| 302 | throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'"); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 303 | } |
| 304 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 305 | void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 306 | { |
| 307 | for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) |
| 308 | { |
| 309 | const xe::TestNode* child = group.getChild(childNdx); |
| 310 | const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; |
Pyry Haulos | 653ad0e | 2016-08-16 12:23:58 +0200 | [diff] [blame] | 311 | const string fullPath = child->getFullPath(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 312 | |
| 313 | if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) |
| 314 | { |
| 315 | if (isGroup) |
| 316 | { |
| 317 | // Recurse into group. |
| 318 | addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); |
| 319 | } |
| 320 | else |
| 321 | { |
| 322 | DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); |
| 323 | testSet.add(child); |
| 324 | } |
| 325 | } |
| 326 | } |
| 327 | } |
| 328 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 329 | void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 330 | { |
| 331 | for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) |
| 332 | { |
| 333 | const xe::TestNode* child = group.getChild(childNdx); |
| 334 | const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 335 | const string fullPath = child->getFullPath(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 336 | |
| 337 | if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) |
| 338 | { |
| 339 | if (isGroup) |
| 340 | { |
| 341 | // Recurse into group. |
| 342 | removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); |
| 343 | } |
| 344 | else |
| 345 | { |
| 346 | DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); |
| 347 | testSet.remove(child); |
| 348 | } |
| 349 | } |
| 350 | } |
| 351 | } |
| 352 | |
| 353 | class BatchResultHandler : public xe::TestLogHandler |
| 354 | { |
| 355 | public: |
| 356 | BatchResultHandler (xe::BatchResult* batchResult) |
| 357 | : m_batchResult(batchResult) |
| 358 | { |
| 359 | } |
| 360 | |
| 361 | void setSessionInfo (const xe::SessionInfo& sessionInfo) |
| 362 | { |
| 363 | m_batchResult->getSessionInfo() = sessionInfo; |
| 364 | } |
| 365 | |
| 366 | xe::TestCaseResultPtr startTestCaseResult (const char* casePath) |
| 367 | { |
| 368 | // \todo [2012-11-01 pyry] What to do with duplicate results? |
| 369 | if (m_batchResult->hasTestCaseResult(casePath)) |
| 370 | return m_batchResult->getTestCaseResult(casePath); |
| 371 | else |
| 372 | return m_batchResult->createTestCaseResult(casePath); |
| 373 | } |
| 374 | |
| 375 | void testCaseResultUpdated (const xe::TestCaseResultPtr&) |
| 376 | { |
| 377 | } |
| 378 | |
| 379 | void testCaseResultComplete (const xe::TestCaseResultPtr&) |
| 380 | { |
| 381 | } |
| 382 | |
| 383 | private: |
| 384 | xe::BatchResult* m_batchResult; |
| 385 | }; |
| 386 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 387 | void readLogFile (xe::BatchResult* batchResult, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 388 | { |
| 389 | std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); |
| 390 | BatchResultHandler handler (batchResult); |
| 391 | xe::TestLogParser parser (&handler); |
| 392 | deUint8 buf [1024]; |
| 393 | int numRead = 0; |
| 394 | |
| 395 | for (;;) |
| 396 | { |
| 397 | in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); |
| 398 | numRead = (int)in.gcount(); |
| 399 | |
| 400 | if (numRead <= 0) |
| 401 | break; |
| 402 | |
| 403 | parser.parse(&buf[0], numRead); |
| 404 | } |
| 405 | |
| 406 | in.close(); |
| 407 | } |
| 408 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 409 | void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 410 | { |
| 411 | int countByStatusCode[xe::TESTSTATUSCODE_LAST]; |
| 412 | std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0); |
| 413 | |
| 414 | for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter) |
| 415 | { |
| 416 | const xe::TestNode* node = *iter; |
| 417 | if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node)) |
| 418 | { |
| 419 | const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 420 | string fullPath; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 421 | xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING; |
| 422 | testCase->getFullPath(fullPath); |
| 423 | |
| 424 | // Parse result data if such exists. |
| 425 | if (batchResult.hasTestCaseResult(fullPath.c_str())) |
| 426 | { |
| 427 | xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str()); |
| 428 | xe::TestCaseResult result; |
| 429 | xe::TestResultParser parser; |
| 430 | |
| 431 | xe::parseTestCaseResultFromData(&parser, &result, *resultData.get()); |
| 432 | statusCode = result.statusCode; |
| 433 | } |
| 434 | |
| 435 | countByStatusCode[statusCode] += 1; |
| 436 | } |
| 437 | } |
| 438 | |
| 439 | printf("\nTest run summary:\n"); |
| 440 | int totalCases = 0; |
| 441 | for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++) |
| 442 | { |
| 443 | if (countByStatusCode[code] > 0) |
| 444 | printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]); |
| 445 | |
| 446 | totalCases += countByStatusCode[code]; |
| 447 | } |
| 448 | printf(" %20s: %5d\n", "Total", totalCases); |
| 449 | } |
| 450 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 451 | void writeInfoLog (const xe::InfoLog& log, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 452 | { |
| 453 | std::ofstream out(filename, std::ios_base::binary); |
| 454 | XE_CHECK(out.good()); |
| 455 | out.write((const char*)log.getBytes(), log.getSize()); |
| 456 | out.close(); |
| 457 | } |
| 458 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 459 | xe::CommLink* createCommLink (const CommandLine& cmdLine) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 460 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 461 | if (cmdLine.runMode == RUNMODE_START_SERVER) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 462 | { |
| 463 | xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink(); |
| 464 | try |
| 465 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 466 | link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 467 | return link; |
| 468 | } |
| 469 | catch (...) |
| 470 | { |
| 471 | delete link; |
| 472 | throw; |
| 473 | } |
| 474 | } |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 475 | else if (cmdLine.runMode == RUNMODE_CONNECT) |
| 476 | { |
| 477 | de::SocketAddress address; |
| 478 | |
| 479 | address.setFamily(DE_SOCKETFAMILY_INET4); |
| 480 | address.setProtocol(DE_SOCKETPROTOCOL_TCP); |
| 481 | address.setHost(cmdLine.serverBinOrAddress.c_str()); |
| 482 | address.setPort(cmdLine.port); |
| 483 | |
| 484 | xe::TcpIpLink* link = new xe::TcpIpLink(); |
| 485 | try |
| 486 | { |
| 487 | std::string error; |
| 488 | |
| 489 | link->connect(address); |
| 490 | return link; |
| 491 | } |
| 492 | catch (const std::exception& error) |
| 493 | { |
| 494 | delete link; |
| 495 | throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what()); |
| 496 | } |
| 497 | catch (...) |
| 498 | { |
| 499 | delete link; |
| 500 | throw; |
| 501 | } |
| 502 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 503 | else |
| 504 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 505 | DE_ASSERT(false); |
| 506 | return DE_NULL; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 507 | } |
| 508 | } |
| 509 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 510 | #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) |
| 511 | |
| 512 | static xe::BatchExecutor* s_executor = DE_NULL; |
| 513 | |
| 514 | void signalHandler (int, siginfo_t*, void*) |
| 515 | { |
| 516 | if (s_executor) |
| 517 | s_executor->cancel(); |
| 518 | } |
| 519 | |
| 520 | void setupSignalHandler (xe::BatchExecutor* executor) |
| 521 | { |
| 522 | s_executor = executor; |
| 523 | struct sigaction sa; |
| 524 | |
| 525 | sa.sa_sigaction = signalHandler; |
| 526 | sa.sa_flags = SA_SIGINFO | SA_RESTART; |
| 527 | sigfillset(&sa.sa_mask); |
| 528 | |
| 529 | sigaction(SIGINT, &sa, DE_NULL); |
| 530 | } |
| 531 | |
| 532 | void resetSignalHandler (void) |
| 533 | { |
| 534 | struct sigaction sa; |
| 535 | |
| 536 | sa.sa_handler = SIG_DFL; |
| 537 | sa.sa_flags = SA_RESTART; |
| 538 | sigfillset(&sa.sa_mask); |
| 539 | |
| 540 | sigaction(SIGINT, &sa, DE_NULL); |
| 541 | s_executor = DE_NULL; |
| 542 | } |
| 543 | |
| 544 | #elif (DE_OS == DE_OS_WIN32) |
| 545 | |
| 546 | static xe::BatchExecutor* s_executor = DE_NULL; |
| 547 | |
| 548 | void signalHandler (int) |
| 549 | { |
| 550 | if (s_executor) |
| 551 | s_executor->cancel(); |
| 552 | } |
| 553 | |
| 554 | void setupSignalHandler (xe::BatchExecutor* executor) |
| 555 | { |
| 556 | s_executor = executor; |
| 557 | signal(SIGINT, signalHandler); |
| 558 | } |
| 559 | |
| 560 | void resetSignalHandler (void) |
| 561 | { |
| 562 | signal(SIGINT, SIG_DFL); |
| 563 | s_executor = DE_NULL; |
| 564 | } |
| 565 | |
| 566 | #else |
| 567 | |
| 568 | void setupSignalHandler (xe::BatchExecutor*) |
| 569 | { |
| 570 | } |
| 571 | |
| 572 | void resetSignalHandler (void) |
| 573 | { |
| 574 | } |
| 575 | |
| 576 | #endif |
| 577 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 578 | void runExecutor (const CommandLine& cmdLine) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 579 | { |
| 580 | xe::TestRoot root; |
| 581 | |
| 582 | // Read case list definitions. |
| 583 | readCaseLists(root, cmdLine.caseListDir.c_str()); |
| 584 | |
| 585 | // Build test set. |
| 586 | xe::TestSet testSet; |
| 587 | |
| 588 | // Build test set. |
| 589 | for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter) |
| 590 | addMatchingCases(root, testSet, filterIter->c_str()); |
| 591 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 592 | if (testSet.empty()) |
| 593 | throw xe::Error("None of the test case lists contains tests matching any of the test sets."); |
| 594 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 595 | // Remove excluded cases. |
| 596 | for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter) |
| 597 | removeMatchingCases(root, testSet, filterIter->c_str()); |
| 598 | |
| 599 | // Initialize batch result. |
| 600 | xe::BatchResult batchResult; |
| 601 | xe::InfoLog infoLog; |
| 602 | |
| 603 | // Read existing results from input file (if supplied). |
| 604 | if (!cmdLine.inFile.empty()) |
| 605 | readLogFile(&batchResult, cmdLine.inFile.c_str()); |
| 606 | |
| 607 | // Initialize commLink. |
Pyry Haulos | 674ddbb | 2017-03-30 16:42:23 -0700 | [diff] [blame] | 608 | de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine)); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 609 | |
| 610 | xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 611 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 612 | try |
| 613 | { |
| 614 | setupSignalHandler(&executor); |
| 615 | executor.run(); |
| 616 | resetSignalHandler(); |
| 617 | } |
| 618 | catch (...) |
| 619 | { |
| 620 | resetSignalHandler(); |
| 621 | |
| 622 | if (!cmdLine.outFile.empty()) |
| 623 | { |
| 624 | xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); |
| 625 | printf("Test log written to %s\n", cmdLine.outFile.c_str()); |
| 626 | } |
| 627 | |
| 628 | if (!cmdLine.infoFile.empty()) |
| 629 | { |
| 630 | writeInfoLog(infoLog, cmdLine.infoFile.c_str()); |
| 631 | printf("Info log written to %s\n", cmdLine.infoFile.c_str()); |
| 632 | } |
| 633 | |
| 634 | if (cmdLine.summary) |
| 635 | printBatchResultSummary(&root, testSet, batchResult); |
| 636 | |
| 637 | throw; |
| 638 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 639 | |
| 640 | if (!cmdLine.outFile.empty()) |
| 641 | { |
| 642 | xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); |
| 643 | printf("Test log written to %s\n", cmdLine.outFile.c_str()); |
| 644 | } |
| 645 | |
| 646 | if (!cmdLine.infoFile.empty()) |
| 647 | { |
| 648 | writeInfoLog(infoLog, cmdLine.infoFile.c_str()); |
| 649 | printf("Info log written to %s\n", cmdLine.infoFile.c_str()); |
| 650 | } |
| 651 | |
| 652 | if (cmdLine.summary) |
| 653 | printBatchResultSummary(&root, testSet, batchResult); |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 654 | |
| 655 | { |
| 656 | string err; |
| 657 | |
| 658 | if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR) |
| 659 | throw xe::Error(err); |
| 660 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 661 | } |
| 662 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 663 | } // anonymous |
| 664 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 665 | int main (int argc, const char* const* argv) |
| 666 | { |
| 667 | CommandLine cmdLine; |
| 668 | |
| 669 | if (!parseCommandLine(cmdLine, argc, argv)) |
| 670 | return -1; |
| 671 | |
| 672 | try |
| 673 | { |
| 674 | runExecutor(cmdLine); |
| 675 | } |
| 676 | catch (const std::exception& e) |
| 677 | { |
| 678 | printf("%s\n", e.what()); |
| 679 | return -1; |
| 680 | } |
| 681 | |
| 682 | return 0; |
| 683 | } |