Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 1 | /*------------------------------------------------------------------------- |
| 2 | * drawElements Quality Program Test Executor |
| 3 | * ------------------------------------------ |
| 4 | * |
| 5 | * Copyright 2014 The Android Open Source Project |
| 6 | * |
| 7 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 8 | * you may not use this file except in compliance with the License. |
| 9 | * You may obtain a copy of the License at |
| 10 | * |
| 11 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | * |
| 13 | * Unless required by applicable law or agreed to in writing, software |
| 14 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | * See the License for the specific language governing permissions and |
| 17 | * limitations under the License. |
| 18 | * |
| 19 | *//*! |
| 20 | * \file |
| 21 | * \brief Command line test executor. |
| 22 | *//*--------------------------------------------------------------------*/ |
| 23 | |
| 24 | #include "xeBatchExecutor.hpp" |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 25 | #include "xeLocalTcpIpLink.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 26 | #include "xeTcpIpLink.hpp" |
| 27 | #include "xeTestCaseListParser.hpp" |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 28 | #include "xeTestLogWriter.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 29 | #include "xeTestResultParser.hpp" |
| 30 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 31 | #include "deCommandLine.hpp" |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 32 | #include "deDirectoryIterator.hpp" |
| 33 | #include "deStringUtil.hpp" |
| 34 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 35 | #include "deString.h" |
| 36 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 37 | #include <algorithm> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 38 | #include <cstdio> |
| 39 | #include <cstdlib> |
| 40 | #include <fstream> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 41 | #include <iostream> |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 42 | #include <memory> |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 43 | #include <sstream> |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 44 | #include <string> |
| 45 | #include <vector> |
| 46 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 47 | #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32) |
| 48 | # include <signal.h> |
| 49 | #endif |
| 50 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 51 | using std::vector; |
| 52 | using std::string; |
| 53 | |
| 54 | namespace |
| 55 | { |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 56 | |
| 57 | // Command line arguments. |
| 58 | namespace opt |
| 59 | { |
| 60 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 61 | DE_DECLARE_COMMAND_LINE_OPT(StartServer, string); |
| 62 | DE_DECLARE_COMMAND_LINE_OPT(Host, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 63 | DE_DECLARE_COMMAND_LINE_OPT(Port, int); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 64 | DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string); |
| 65 | DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>); |
| 66 | DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>); |
| 67 | DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string); |
| 68 | DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string); |
| 69 | DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 70 | DE_DECLARE_COMMAND_LINE_OPT(Summary, bool); |
| 71 | |
| 72 | // TargetConfiguration |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 73 | DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string); |
| 74 | DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string); |
| 75 | DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 76 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 77 | void parseCommaSeparatedList (const char* src, vector<string>* dst) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 78 | { |
| 79 | std::istringstream inStr (src); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 80 | string comp; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 81 | |
| 82 | while (std::getline(inStr, comp, ',')) |
| 83 | dst->push_back(comp); |
| 84 | } |
| 85 | |
| 86 | void registerOptions (de::cmdline::Parser& parser) |
| 87 | { |
| 88 | using de::cmdline::Option; |
| 89 | using de::cmdline::NamedValue; |
| 90 | |
| 91 | static const NamedValue<bool> s_yesNo[] = |
| 92 | { |
| 93 | { "yes", true }, |
| 94 | { "no", false } |
| 95 | }; |
| 96 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 97 | parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.") |
| 98 | << Option<Host> ("c", "connect", "Connect to host. Address of the execserver.") |
| 99 | << Option<Port> ("p", "port", "TCP port of the execserver.", "50016") |
| 100 | << Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".") |
| 101 | << Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList) |
| 102 | << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "") |
| 103 | << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.") |
| 104 | << Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa") |
| 105 | << Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt") |
| 106 | << Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes") |
| 107 | << Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>") |
| 108 | << Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".") |
| 109 | << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", ""); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | } // opt |
| 113 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 114 | enum RunMode |
| 115 | { |
| 116 | RUNMODE_CONNECT, |
| 117 | RUNMODE_START_SERVER |
| 118 | }; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 119 | |
| 120 | struct CommandLine |
| 121 | { |
| 122 | CommandLine (void) |
| 123 | : port (0) |
| 124 | , summary (false) |
| 125 | { |
| 126 | } |
| 127 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 128 | xe::TargetConfiguration targetCfg; |
| 129 | RunMode runMode; |
| 130 | string serverBinOrAddress; |
| 131 | int port; |
| 132 | string caseListDir; |
| 133 | vector<string> testset; |
| 134 | vector<string> exclude; |
| 135 | string inFile; |
| 136 | string outFile; |
| 137 | string infoFile; |
| 138 | bool summary; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 139 | }; |
| 140 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 141 | bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 142 | { |
| 143 | de::cmdline::Parser parser; |
| 144 | de::cmdline::CommandLine opts; |
| 145 | |
| 146 | XE_CHECK(argc >= 1); |
| 147 | |
| 148 | opt::registerOptions(parser); |
| 149 | |
| 150 | if (!parser.parse(argc-1, argv+1, &opts, std::cerr)) |
| 151 | { |
| 152 | std::cout << argv[0] << " [options]\n"; |
| 153 | parser.help(std::cout); |
| 154 | return false; |
| 155 | } |
| 156 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 157 | if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>()) |
| 158 | { |
| 159 | std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl; |
| 160 | return false; |
| 161 | } |
| 162 | else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>()) |
| 163 | { |
| 164 | std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl; |
| 165 | return false; |
| 166 | } |
| 167 | |
| 168 | if (!opts.hasOption<opt::TestSet>()) |
| 169 | { |
| 170 | std::cout << "Invalid command line arguments. --testset not defined." << std::endl; |
| 171 | return false; |
| 172 | } |
| 173 | |
| 174 | if (opts.hasOption<opt::StartServer>()) |
| 175 | { |
| 176 | cmdLine.runMode = RUNMODE_START_SERVER; |
| 177 | cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>(); |
| 178 | } |
| 179 | else |
| 180 | { |
| 181 | cmdLine.runMode = RUNMODE_CONNECT; |
| 182 | cmdLine.serverBinOrAddress = opts.getOption<opt::Host>(); |
| 183 | } |
| 184 | |
| 185 | if (opts.hasOption<opt::ContinueFile>()) |
| 186 | { |
| 187 | cmdLine.inFile = opts.getOption<opt::ContinueFile>(); |
| 188 | |
| 189 | if (cmdLine.inFile.empty()) |
| 190 | { |
| 191 | std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl; |
| 192 | return false; |
| 193 | } |
| 194 | } |
| 195 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 196 | cmdLine.port = opts.getOption<opt::Port>(); |
| 197 | cmdLine.caseListDir = opts.getOption<opt::CaseListDir>(); |
| 198 | cmdLine.testset = opts.getOption<opt::TestSet>(); |
| 199 | cmdLine.exclude = opts.getOption<opt::ExcludeSet>(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 200 | cmdLine.outFile = opts.getOption<opt::TestLogFile>(); |
| 201 | cmdLine.infoFile = opts.getOption<opt::InfoLogFile>(); |
| 202 | cmdLine.summary = opts.getOption<opt::Summary>(); |
| 203 | cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>(); |
| 204 | cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>(); |
| 205 | cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>(); |
| 206 | |
| 207 | return true; |
| 208 | } |
| 209 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 210 | bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 211 | { |
| 212 | int ptrnPos = 0; |
| 213 | int casePos = 0; |
| 214 | |
| 215 | for (;;) |
| 216 | { |
| 217 | char c = casePath[casePos]; |
| 218 | char p = pattern[ptrnPos]; |
| 219 | |
| 220 | if (p == '*') |
| 221 | { |
| 222 | /* Recurse to rest of positions. */ |
| 223 | int next = casePos; |
| 224 | for (;;) |
| 225 | { |
| 226 | if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup)) |
| 227 | return DE_TRUE; |
| 228 | |
| 229 | if (casePath[next] == 0) |
| 230 | return DE_FALSE; /* No match found. */ |
| 231 | else |
| 232 | next += 1; |
| 233 | } |
| 234 | DE_ASSERT(DE_FALSE); |
| 235 | } |
| 236 | else if (c == 0 && p == 0) |
| 237 | return true; |
| 238 | else if (c == 0) |
| 239 | { |
| 240 | /* Incomplete match is ok for test groups. */ |
| 241 | return isTestGroup; |
| 242 | } |
| 243 | else if (c != p) |
| 244 | return false; |
| 245 | |
| 246 | casePos += 1; |
| 247 | ptrnPos += 1; |
| 248 | } |
| 249 | |
| 250 | DE_ASSERT(false); |
| 251 | return false; |
| 252 | } |
| 253 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 254 | void readCaseList (xe::TestGroup* root, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 255 | { |
| 256 | xe::TestCaseListParser caseListParser; |
| 257 | std::ifstream in (filename, std::ios_base::binary); |
| 258 | deUint8 buf[1024]; |
| 259 | |
| 260 | XE_CHECK(in.good()); |
| 261 | |
| 262 | caseListParser.init(root); |
| 263 | |
| 264 | for (;;) |
| 265 | { |
| 266 | in.read((char*)&buf[0], sizeof(buf)); |
| 267 | int numRead = (int)in.gcount(); |
| 268 | |
| 269 | if (numRead > 0) |
| 270 | caseListParser.parse(&buf[0], numRead); |
| 271 | |
| 272 | if (numRead < (int)sizeof(buf)) |
| 273 | break; // EOF |
| 274 | } |
| 275 | } |
| 276 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 277 | void readCaseLists (xe::TestRoot& root, const char* caseListDir) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 278 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 279 | int testCaseListCount = 0; |
| 280 | de::DirectoryIterator iter (caseListDir); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 281 | |
| 282 | for (; iter.hasItem(); iter.next()) |
| 283 | { |
| 284 | de::FilePath item = iter.getItem(); |
| 285 | |
| 286 | if (item.getType() == de::FilePath::TYPE_FILE) |
| 287 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 288 | string baseName = item.getBaseName(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 289 | if (baseName.find("-cases.xml") == baseName.length()-10) |
| 290 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 291 | string packageName = baseName.substr(0, baseName.length()-10); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 292 | xe::TestGroup* package = root.createGroup(packageName.c_str(), ""); |
| 293 | |
| 294 | readCaseList(package, item.getPath()); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 295 | testCaseListCount++; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 296 | } |
| 297 | } |
| 298 | } |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 299 | |
| 300 | if (testCaseListCount == 0) |
| 301 | throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'"); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 302 | } |
| 303 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 304 | void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 305 | { |
| 306 | for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) |
| 307 | { |
| 308 | const xe::TestNode* child = group.getChild(childNdx); |
| 309 | const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 310 | const string fullPath = child->getFullPath(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 311 | |
| 312 | if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) |
| 313 | { |
| 314 | if (isGroup) |
| 315 | { |
| 316 | // Recurse into group. |
| 317 | addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); |
| 318 | } |
| 319 | else |
| 320 | { |
| 321 | DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); |
| 322 | testSet.add(child); |
| 323 | } |
| 324 | } |
| 325 | } |
| 326 | } |
| 327 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 328 | void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 329 | { |
| 330 | for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) |
| 331 | { |
| 332 | const xe::TestNode* child = group.getChild(childNdx); |
| 333 | const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 334 | const string fullPath = child->getFullPath(); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 335 | |
| 336 | if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) |
| 337 | { |
| 338 | if (isGroup) |
| 339 | { |
| 340 | // Recurse into group. |
| 341 | removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); |
| 342 | } |
| 343 | else |
| 344 | { |
| 345 | DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); |
| 346 | testSet.remove(child); |
| 347 | } |
| 348 | } |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | class BatchResultHandler : public xe::TestLogHandler |
| 353 | { |
| 354 | public: |
| 355 | BatchResultHandler (xe::BatchResult* batchResult) |
| 356 | : m_batchResult(batchResult) |
| 357 | { |
| 358 | } |
| 359 | |
| 360 | void setSessionInfo (const xe::SessionInfo& sessionInfo) |
| 361 | { |
| 362 | m_batchResult->getSessionInfo() = sessionInfo; |
| 363 | } |
| 364 | |
| 365 | xe::TestCaseResultPtr startTestCaseResult (const char* casePath) |
| 366 | { |
| 367 | // \todo [2012-11-01 pyry] What to do with duplicate results? |
| 368 | if (m_batchResult->hasTestCaseResult(casePath)) |
| 369 | return m_batchResult->getTestCaseResult(casePath); |
| 370 | else |
| 371 | return m_batchResult->createTestCaseResult(casePath); |
| 372 | } |
| 373 | |
| 374 | void testCaseResultUpdated (const xe::TestCaseResultPtr&) |
| 375 | { |
| 376 | } |
| 377 | |
| 378 | void testCaseResultComplete (const xe::TestCaseResultPtr&) |
| 379 | { |
| 380 | } |
| 381 | |
| 382 | private: |
| 383 | xe::BatchResult* m_batchResult; |
| 384 | }; |
| 385 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 386 | void readLogFile (xe::BatchResult* batchResult, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 387 | { |
| 388 | std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); |
| 389 | BatchResultHandler handler (batchResult); |
| 390 | xe::TestLogParser parser (&handler); |
| 391 | deUint8 buf [1024]; |
| 392 | int numRead = 0; |
| 393 | |
| 394 | for (;;) |
| 395 | { |
| 396 | in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); |
| 397 | numRead = (int)in.gcount(); |
| 398 | |
| 399 | if (numRead <= 0) |
| 400 | break; |
| 401 | |
| 402 | parser.parse(&buf[0], numRead); |
| 403 | } |
| 404 | |
| 405 | in.close(); |
| 406 | } |
| 407 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 408 | void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 409 | { |
| 410 | int countByStatusCode[xe::TESTSTATUSCODE_LAST]; |
| 411 | std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0); |
| 412 | |
| 413 | for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter) |
| 414 | { |
| 415 | const xe::TestNode* node = *iter; |
| 416 | if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node)) |
| 417 | { |
| 418 | const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node); |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 419 | string fullPath; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 420 | xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING; |
| 421 | testCase->getFullPath(fullPath); |
| 422 | |
| 423 | // Parse result data if such exists. |
| 424 | if (batchResult.hasTestCaseResult(fullPath.c_str())) |
| 425 | { |
| 426 | xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str()); |
| 427 | xe::TestCaseResult result; |
| 428 | xe::TestResultParser parser; |
| 429 | |
| 430 | xe::parseTestCaseResultFromData(&parser, &result, *resultData.get()); |
| 431 | statusCode = result.statusCode; |
| 432 | } |
| 433 | |
| 434 | countByStatusCode[statusCode] += 1; |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | printf("\nTest run summary:\n"); |
| 439 | int totalCases = 0; |
| 440 | for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++) |
| 441 | { |
| 442 | if (countByStatusCode[code] > 0) |
| 443 | printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]); |
| 444 | |
| 445 | totalCases += countByStatusCode[code]; |
| 446 | } |
| 447 | printf(" %20s: %5d\n", "Total", totalCases); |
| 448 | } |
| 449 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 450 | void writeInfoLog (const xe::InfoLog& log, const char* filename) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 451 | { |
| 452 | std::ofstream out(filename, std::ios_base::binary); |
| 453 | XE_CHECK(out.good()); |
| 454 | out.write((const char*)log.getBytes(), log.getSize()); |
| 455 | out.close(); |
| 456 | } |
| 457 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 458 | xe::CommLink* createCommLink (const CommandLine& cmdLine) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 459 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 460 | if (cmdLine.runMode == RUNMODE_START_SERVER) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 461 | { |
| 462 | xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink(); |
| 463 | try |
| 464 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 465 | link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 466 | return link; |
| 467 | } |
| 468 | catch (...) |
| 469 | { |
| 470 | delete link; |
| 471 | throw; |
| 472 | } |
| 473 | } |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 474 | else if (cmdLine.runMode == RUNMODE_CONNECT) |
| 475 | { |
| 476 | de::SocketAddress address; |
| 477 | |
| 478 | address.setFamily(DE_SOCKETFAMILY_INET4); |
| 479 | address.setProtocol(DE_SOCKETPROTOCOL_TCP); |
| 480 | address.setHost(cmdLine.serverBinOrAddress.c_str()); |
| 481 | address.setPort(cmdLine.port); |
| 482 | |
| 483 | xe::TcpIpLink* link = new xe::TcpIpLink(); |
| 484 | try |
| 485 | { |
| 486 | std::string error; |
| 487 | |
| 488 | link->connect(address); |
| 489 | return link; |
| 490 | } |
| 491 | catch (const std::exception& error) |
| 492 | { |
| 493 | delete link; |
| 494 | throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what()); |
| 495 | } |
| 496 | catch (...) |
| 497 | { |
| 498 | delete link; |
| 499 | throw; |
| 500 | } |
| 501 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 502 | else |
| 503 | { |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 504 | DE_ASSERT(false); |
| 505 | return DE_NULL; |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 506 | } |
| 507 | } |
| 508 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 509 | #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) |
| 510 | |
| 511 | static xe::BatchExecutor* s_executor = DE_NULL; |
| 512 | |
| 513 | void signalHandler (int, siginfo_t*, void*) |
| 514 | { |
| 515 | if (s_executor) |
| 516 | s_executor->cancel(); |
| 517 | } |
| 518 | |
| 519 | void setupSignalHandler (xe::BatchExecutor* executor) |
| 520 | { |
| 521 | s_executor = executor; |
| 522 | struct sigaction sa; |
| 523 | |
| 524 | sa.sa_sigaction = signalHandler; |
| 525 | sa.sa_flags = SA_SIGINFO | SA_RESTART; |
| 526 | sigfillset(&sa.sa_mask); |
| 527 | |
| 528 | sigaction(SIGINT, &sa, DE_NULL); |
| 529 | } |
| 530 | |
| 531 | void resetSignalHandler (void) |
| 532 | { |
| 533 | struct sigaction sa; |
| 534 | |
| 535 | sa.sa_handler = SIG_DFL; |
| 536 | sa.sa_flags = SA_RESTART; |
| 537 | sigfillset(&sa.sa_mask); |
| 538 | |
| 539 | sigaction(SIGINT, &sa, DE_NULL); |
| 540 | s_executor = DE_NULL; |
| 541 | } |
| 542 | |
| 543 | #elif (DE_OS == DE_OS_WIN32) |
| 544 | |
| 545 | static xe::BatchExecutor* s_executor = DE_NULL; |
| 546 | |
| 547 | void signalHandler (int) |
| 548 | { |
| 549 | if (s_executor) |
| 550 | s_executor->cancel(); |
| 551 | } |
| 552 | |
| 553 | void setupSignalHandler (xe::BatchExecutor* executor) |
| 554 | { |
| 555 | s_executor = executor; |
| 556 | signal(SIGINT, signalHandler); |
| 557 | } |
| 558 | |
| 559 | void resetSignalHandler (void) |
| 560 | { |
| 561 | signal(SIGINT, SIG_DFL); |
| 562 | s_executor = DE_NULL; |
| 563 | } |
| 564 | |
| 565 | #else |
| 566 | |
| 567 | void setupSignalHandler (xe::BatchExecutor*) |
| 568 | { |
| 569 | } |
| 570 | |
| 571 | void resetSignalHandler (void) |
| 572 | { |
| 573 | } |
| 574 | |
| 575 | #endif |
| 576 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 577 | void runExecutor (const CommandLine& cmdLine) |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 578 | { |
| 579 | xe::TestRoot root; |
| 580 | |
| 581 | // Read case list definitions. |
| 582 | readCaseLists(root, cmdLine.caseListDir.c_str()); |
| 583 | |
| 584 | // Build test set. |
| 585 | xe::TestSet testSet; |
| 586 | |
| 587 | // Build test set. |
| 588 | for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter) |
| 589 | addMatchingCases(root, testSet, filterIter->c_str()); |
| 590 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 591 | if (testSet.empty()) |
| 592 | throw xe::Error("None of the test case lists contains tests matching any of the test sets."); |
| 593 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 594 | // Remove excluded cases. |
| 595 | for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter) |
| 596 | removeMatchingCases(root, testSet, filterIter->c_str()); |
| 597 | |
| 598 | // Initialize batch result. |
| 599 | xe::BatchResult batchResult; |
| 600 | xe::InfoLog infoLog; |
| 601 | |
| 602 | // Read existing results from input file (if supplied). |
| 603 | if (!cmdLine.inFile.empty()) |
| 604 | readLogFile(&batchResult, cmdLine.inFile.c_str()); |
| 605 | |
| 606 | // Initialize commLink. |
| 607 | std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine)); |
| 608 | |
| 609 | xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog); |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 610 | |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 611 | try |
| 612 | { |
| 613 | setupSignalHandler(&executor); |
| 614 | executor.run(); |
| 615 | resetSignalHandler(); |
| 616 | } |
| 617 | catch (...) |
| 618 | { |
| 619 | resetSignalHandler(); |
| 620 | |
| 621 | if (!cmdLine.outFile.empty()) |
| 622 | { |
| 623 | xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); |
| 624 | printf("Test log written to %s\n", cmdLine.outFile.c_str()); |
| 625 | } |
| 626 | |
| 627 | if (!cmdLine.infoFile.empty()) |
| 628 | { |
| 629 | writeInfoLog(infoLog, cmdLine.infoFile.c_str()); |
| 630 | printf("Info log written to %s\n", cmdLine.infoFile.c_str()); |
| 631 | } |
| 632 | |
| 633 | if (cmdLine.summary) |
| 634 | printBatchResultSummary(&root, testSet, batchResult); |
| 635 | |
| 636 | throw; |
| 637 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 638 | |
| 639 | if (!cmdLine.outFile.empty()) |
| 640 | { |
| 641 | xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); |
| 642 | printf("Test log written to %s\n", cmdLine.outFile.c_str()); |
| 643 | } |
| 644 | |
| 645 | if (!cmdLine.infoFile.empty()) |
| 646 | { |
| 647 | writeInfoLog(infoLog, cmdLine.infoFile.c_str()); |
| 648 | printf("Info log written to %s\n", cmdLine.infoFile.c_str()); |
| 649 | } |
| 650 | |
| 651 | if (cmdLine.summary) |
| 652 | printBatchResultSummary(&root, testSet, batchResult); |
Mika Isojärvi | 5717785 | 2015-05-19 13:44:08 -0700 | [diff] [blame] | 653 | |
| 654 | { |
| 655 | string err; |
| 656 | |
| 657 | if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR) |
| 658 | throw xe::Error(err); |
| 659 | } |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 660 | } |
| 661 | |
Mika Isojärvi | e16b0f5 | 2015-05-18 15:32:39 -0700 | [diff] [blame] | 662 | } // anonymous |
| 663 | |
Jarkko Poyry | 3c82736 | 2014-09-02 11:48:52 +0300 | [diff] [blame] | 664 | int main (int argc, const char* const* argv) |
| 665 | { |
| 666 | CommandLine cmdLine; |
| 667 | |
| 668 | if (!parseCommandLine(cmdLine, argc, argv)) |
| 669 | return -1; |
| 670 | |
| 671 | try |
| 672 | { |
| 673 | runExecutor(cmdLine); |
| 674 | } |
| 675 | catch (const std::exception& e) |
| 676 | { |
| 677 | printf("%s\n", e.what()); |
| 678 | return -1; |
| 679 | } |
| 680 | |
| 681 | return 0; |
| 682 | } |