.gitlab-ci/tracie/tests/test.py

import logging
import pytest
import re
import shutil
import xml.etree.ElementTree as ET

from os import environ, chdir
from os.path import dirname, exists, realpath

import tracie


RESULTS_YAML = "results/results.yml"
JUNIT_XML = "results/junit.xml"
TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
TRACIE_DIR = dirname(realpath(__file__)) + "/.."

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()


def write_to(content, filename):
    with open(filename, 'w') as f:
        f.write(content)


def read_from(filename):
    with open(filename) as f:
        content = f.read()
    return content


def run_tracie():
    '''
    Run tests for the .testtrace types, using the "gl-test-device" and
    "vk-test-device" device names.
    '''
    result = tracie.main(["--device-name", "gl-test-device",
                          "--file", "./tests/traces.yml"])
    if not result:
        return False
    result = tracie.main(["--device-name", "vk-test-device",
                          "--file", "./tests/traces.yml"])
    return result


def prepare_for_run(tmp_path):
    '''
    Copy all the tracie scripts to the test dir for the unit tests.
    This avoids polluting the normal working dir with test result artifacts.
    '''
    test_dir = str(tmp_path) + "/run"
    shutil.copytree(TRACIE_DIR, test_dir)
    # Change the working dir to the test_dir
    chdir(test_dir)
    # Set the traces-db
    shutil.move("./tests/test-data", "./traces-db")
    # Disable trace storing
    environ["TRACIE_STORE_IMAGES"] = "0"
    environ["TRACIE_UPLOAD_TO_MINIO"] = "0"
    environ["CI_PROJECT_PATH"] = "test-project"
    environ["CI_PIPELINE_ID"] = "667"
    environ["CI_JOB_ID"] = "42"

def cleanup(tmp_path):
    '''
    Performs the clean up of the test dir.
    '''
    if exists(tmp_path):
        shutil.rmtree(tmp_path)


@pytest.fixture(autouse=True)
def run_test(tmp_path):
    '''
    Wraps the execution of each test as follows:

      prepare_for_run()
      test()
      cleanup()
    '''
    logger.debug("Working dir: %s", tmp_path)
    prepare_for_run(tmp_path)
    yield
    cleanup(tmp_path)


def check_results_yaml_content(filename, expectations):
    '''
    Checks the content of the filename with the list of expectations
    passed as parameter.

    Arguments:
        filename (str): The path of the file to check
        expectations (list): A list with the content to find in the file

    Returns:
        bool: The return value. True if the content of the filename satisfies
              the expectations, False otherwise.
    '''
    content = read_from(filename)
    for e in expectations:
        ocurrencies = re.findall(e, content)
        if not len(ocurrencies):
            logger.error("Expectation not found in %s: %s", filename, e)
            return False
    return True


def test_tracie_succeeds_if_all_images_match():
    assert run_tracie()
    expectations = [
        "actual: 5efda83854befe0155ff8517a58d5b51",
        "expected: 5efda83854befe0155ff8517a58d5b51",
    ]
    assert check_results_yaml_content(RESULTS_YAML, expectations)


def test_tracie_fails_on_image_mismatch():
    filename = "./tests/traces.yml"
    content = read_from(filename)
    content = content.replace("5efda83854befe0155ff8517a58d5b51",
                              "8e0a801367e1714463475a824dab363b")
    write_to(content, filename)
    assert not run_tracie()
    expectations = [
        "actual: 5efda83854befe0155ff8517a58d5b51",
        "expected: 8e0a801367e1714463475a824dab363b",
        "trace2/test/vk-test-device/olive.testtrace-0.png"
    ]
    assert check_results_yaml_content(RESULTS_YAML, expectations)


def test_tracie_traces_with_and_without_checksum():
    filename = "./tests/traces.yml"
    content = read_from(filename)
    content += '''  - path: trace1/red.testtrace
    expectations:
    - device: bla
      checksum: 000000000000000'''
    write_to(content, filename)

    # red.testtrace should be skipped, since it doesn't
    # have any checksums for our device
    filename = "./traces-db/trace1/red.testtrace"
    content = "ff0000ff"
    write_to(content, filename)
    assert run_tracie()


def test_tracie_only_traces_without_checksum():
    filename = "./tests/traces.yml"
    content = '''traces:
  - path: trace1/red.testtrace
    expectations:
    - device: bla
      checksum: 000000000000000'''
    write_to(content, filename)

    # red.testtrace should be skipped, since it doesn't
    # have any checksums for our device
    filename = "./traces-db/trace1/red.testtrace"
    content = "ff0000ff"
    write_to(content, filename)
    assert run_tracie()


def test_tracie_with_no_traces():
    filename = "./tests/traces.yml"
    content = 'traces:'
    write_to(content, filename)
    assert run_tracie()
    expectations = [
        "{}",
    ]
    assert check_results_yaml_content(RESULTS_YAML, expectations)


def test_tracie_fails_on_dump_image_error():
    # "invalid" should fail to parse as rgba and
    # cause an error
    filename = "./traces-db/trace1/magenta.testtrace"
    write_to("invalid\n", filename)
    run_tracie()
    expectations = [
        "actual: error",
        "expected: 8e0a801367e1714463475a824dab363b",
        "trace1/magenta.testtrace",
    ]
    assert check_results_yaml_content(RESULTS_YAML, expectations)


def test_tracie_stores_only_logs_on_checksum_match():
    assert run_tracie()
    assert exists(TRACE_LOG_TEST1)
    assert exists(TRACE_LOG_TEST2)
    assert not exists(TRACE_PNG_TEST1)
    assert not exists(TRACE_PNG_TEST2)


def test_tracie_stores_images_on_checksum_mismatch():
    filename = "./tests/traces.yml"
    content = read_from(filename)
    content = content.replace("5efda83854befe0155ff8517a58d5b51",
                              "8e0a801367e1714463475a824dab363b")
    write_to(content, filename)
    assert not run_tracie()
    assert not exists(TRACE_PNG_TEST1)
    assert exists(TRACE_PNG_TEST2)


def test_tracie_stores_images_on_request():
    environ["TRACIE_STORE_IMAGES"] = "1"
    assert run_tracie()
    assert exists(TRACE_PNG_TEST1)
    assert exists(TRACE_PNG_TEST2)

def test_tracie_writes_junit_xml():
    assert run_tracie()
    junit_xml = ET.parse(JUNIT_XML)
    assert junit_xml.getroot().tag == 'testsuites'
    testsuites = junit_xml.findall("./testsuite")
    testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase")
    testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase")

    assert len(testsuites) == 2
    assert len(testcases_gl) == 1
    assert len(testcases_vk) == 1
    assert testcases_gl[0].get("name") == "trace1/magenta.testtrace"
    assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device"
    assert testcases_vk[0].get("name") == "trace2/olive.testtrace"
    assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device"

def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag():
    filename = "./tests/traces.yml"
    content = read_from(filename)
    content = content.replace("5efda83854befe0155ff8517a58d5b51",
                              "8e0a801367e1714463475a824dab363b")
    write_to(content, filename)

    assert not run_tracie()

    junit_xml = ET.parse(JUNIT_XML)
    failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure")
    failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure")

    assert len(failures_gl) == 0
    assert len(failures_vk) == 1
    dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace"
    assert dashboard_url in failures_vk[0].text