automated/linux/gst-validate: Add support for ignore tests
There are some gst-validate tests that fails depending on the board
so add a way to use a file stored in a git repository to ignore the
result.
The gst-validate-launcher dosen't provide a command line option to
exclude certain tests by design depends on write your own testsuite.py
module so this is the better implementation for now.
Change-Id: I8e05de42892a58c1ccb344eb72932217de23921d
Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
diff --git a/automated/linux/gst-validate/gst-validate.yaml b/automated/linux/gst-validate/gst-validate.yaml
index 39d6066..7a3e765 100644
--- a/automated/linux/gst-validate/gst-validate.yaml
+++ b/automated/linux/gst-validate/gst-validate.yaml
@@ -26,17 +26,22 @@
MAIN_DIR: "/gst-validate"
OPTIONS: "-nd -v -j 1"
GST_INTEGRATION_SUITES: "http://testdata.validation.linaro.org/gst-validate/gst-integration-testsuites_1.12.tar.gz"
+ GST_IGNORE_TESTS_REPO: ""
+ GST_IGNORE_TESTS_BRANCH: "master"
+ GST_IGNORE_TESTS_FILE: ""
run:
steps:
+ - . ./automated/lib/sh-test-lib
+ - cd ./automated/linux/gst-validate
- wget ${GST_INTEGRATION_SUITES}
- tar -xzf $(basename ${GST_INTEGRATION_SUITES}) -C /
- - gst-validate-launcher -M ${MAIN_DIR} ${OPTIONS} || true
-
-parse:
- pattern: '(?P<test_case_id>validate\..*):\s+(?P<result>(Failed|Passed|Skipped|Timeout))'
- fixupdict:
- Failed: fail
- Passed: pass
- Skipped: skip
- Timeout: fail
+ - IGNORE_FILE=""
+ - if [ ! -z "${GST_IGNORE_TESTS_REPO}" ] && [ ! -z "${GST_IGNORE_TESTS_FILE}" ]; then
+ - repo_path=${PWD}/$(basename ${GST_IGNORE_TESTS_REPO})
+ - git clone -b ${GST_IGNORE_TESTS_BRANCH} ${GST_IGNORE_TESTS_REPO} $repo_path
+ - IGNORE_FILE=${repo_path}/${GST_IGNORE_TESTS_FILE}
+ - fi
+ - gst-validate-launcher -M ${MAIN_DIR} ${OPTIONS} 2>&1 | tee ./gst-validate-raw.log
+ - ./gst_validate_lava_parse.py ./gst-validate-raw.log ${IGNORE_FILE} > ./result.txt
+ - ../../utils/send-to-lava.sh ./result.txt
diff --git a/automated/linux/gst-validate/gst_validate_lava_parse.py b/automated/linux/gst-validate/gst_validate_lava_parse.py
new file mode 100755
index 0000000..8845fe6
--- /dev/null
+++ b/automated/linux/gst-validate/gst_validate_lava_parse.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# LAVA/OE gst-validate results parse script
+#
+# Copyright (C) 2018, Linaro Limited.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# Author: Aníbal Limón <anibal.limon@linaro.org>
+#
+
+import json
+import os
+import re
+import sys
+
+
+def map_result_to_lava(result):
+ if result == 'Passed':
+ result = 'pass'
+ elif result == 'Failed':
+ result = 'fail'
+ elif result == 'Skipped':
+ result = 'skip'
+ elif result == 'Timeout':
+ result = 'fail'
+
+ return result
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print("Usage: %s <result_file> [ignore_file]" % sys.argv[0])
+ sys.exit(1)
+
+ ignore_tests = []
+ if len(sys.argv) == 3:
+ with open(sys.argv[2], 'r') as f:
+ ignore_tests = f.read().split()
+
+ rex = re.compile('^(?P<test_case_id>validate\..*):\s+(?P<result>(Failed|Passed|Skipped|Timeout))')
+ with open(sys.argv[1], 'r') as f:
+ for line in f.readlines():
+ s = rex.search(line)
+ if s:
+ test_case_id = s.group('test_case_id')
+ result = s.group('result')
+
+ if test_case_id not in ignore_tests:
+ print("%s %s" % (test_case_id, map_result_to_lava(result)))