lava/lava_config_definitions.py

# Copyright 2019-2020 Fairphone B.V.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""LAVA configuration deployment classes."""

from contextlib import contextmanager
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Dict, Generator, Optional

import jinja2
import yaml

from lava_interface import Lavacli, LavaError, LavaDevice, LavaDeviceHealth


@contextmanager
def processed_source_file(
    path: Path, operation: str = "pass-through", **template_kwargs: Any
) -> Generator[Path, None, None]:
    """Return the path to an optionally processed input file.

    This function facilitates using input files that may or may not need to be
    processed before being used by other functions. It takes in a path to a
    file, executes the requested operation and provides the path to the
    resulting file within the block of the context in that this function used.

    The resulting path is only valid within the `with` block that is executed in
    the created context. Any temporary files generated by this function are
    guaranteed to be deleted once the context is closed.

    Following operations are supported:
        pass-through: The input file is used as is.
        render-jinja2:
            The input file is treated as jinja2 template and rendered
            accordingly.

    Arguments:
        path: Path to the input file.
        operation: Operation to perform on the file.
        template_kwargs:
            In case of templating operations, these arguments are made available
            to the template while it gets rendered.

    Raises:
        ValueError: If the requested operation is invalid.

    Returns:
        Generator for a `with` context that produces a Path object pointing to
        the generated output file.

    """
    if operation == "pass-through":
        # Use input file as is
        yield path
        return

    if operation == "render-jinja2":
        # Render jinja template first.
        env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(str(path.parent))
        )
        template = env.get_template(path.name)

        # Render into a temporary file.
        tmp_file = NamedTemporaryFile(mode="w+", delete=False)
        with tmp_file as rendered_file:
            rendered_file.writelines(template.render(**template_kwargs))

        tmp_file_path = Path(tmp_file.name)
        try:
            yield tmp_file_path
        finally:
            tmp_file_path.unlink()

        return

    raise ValueError("Unknown file operation requested: {}".format(operation))


class DeviceTypeDefinition:
    """Device type definitions for LAVA configuration deployment."""

    def __init__(self, yaml_path: Path, lavacli: Lavacli):
        """Initialize an instance from a definition YAML file.

        Arguments:
            yaml_path: Path pointing to the source YAML file.
            lavacli: lavacli wrapper instance.

        """
        self._yaml_path = yaml_path
        self.device_type_name = self._yaml_path.stem
        self._config_yaml = yaml.safe_load(self._yaml_path.read_text())
        self._lavacli = lavacli

    @contextmanager
    def get_config_file(
        self, config_key: str, **extra_template_kwargs: Any
    ) -> Generator[Path, None, None]:
        """Get a (rendered) configuration file for the device type.

        This produces a device type dictionary template, device dictionary of
        health check in context of the current device type.

        This is a context manager. The returned path is only guaranteed to be
        valid within the `with` block.

        Arguments:
            config_key:
                Configuration file to be produced. This must be one of the
                top-level keys defined by a device type deployment YAML file:
                device_type_dict, device_dict or health_check.
            extra_template_kwargs:
                Extra arguments used for template rendering. This is only used
                if the selected file in the current device type requests
                template rendering as an operation on the input file.

        Returns:
            pathlib.Path object pointing to the (temporarily) generated file.

        """
        config = self._config_yaml[config_key]
        with processed_source_file(
            path=self._yaml_path.parent / config["source"],
            operation=config["operation"],
            device_type=self.device_type_name,
            **extra_template_kwargs,
        ) as config_file:
            yield config_file

    def _type_exists(self) -> bool:
        """Check whether current device type already exists in LAVA.

        Returns: True if the type exists already, False otherwise.

        """
        all_types_dict = self._lavacli.device_types_cmd_yaml("list", "--yaml")
        return bool(
            [
                type
                for type in all_types_dict
                if type["name"] == self.device_type_name
            ]
        )

    def _deploy_config_file(self, config_key: str) -> None:
        """Deploy configuration files for existing device types.

        This allows to deploy the device type dictionary or the health check.
        Both have very similar subcommands in `lavacli` and just differ in the
        command names for what is done here.

        The device type must be added to LAVA before adding configurations
        through this function.

        """
        # Fetch the file if it exists. Silently ignore if this fails due to a
        # not yet existing configuration.
        if config_key == "device_type_dict":
            lavacli_cmd = "template"
        elif config_key == "health_check":
            lavacli_cmd = "health-check"
        else:
            raise RuntimeError(
                "Invalid configuration key: {}".format(config_key)
            )

        current_file_content: Optional[str]
        try:
            current_file_content = self._lavacli.device_types_cmd(
                lavacli_cmd, "get", self.device_type_name
            )
        except LavaError as e:
            # The error is the same for not-yet added device types and missing
            # configurations. We must assume that the device type was added to
            # LAVA prior to calling the function here.
            expected_error = (
                '''Fault 404: "Device-type '{}' was not found."'''
                "".format(self.device_type_name)
            )
            if expected_error not in e.called_process_error.stdout:
                raise e
            current_file_content = None

        # Update if necessary.
        with self.get_config_file(config_key=config_key) as new_file:
            new_file_content = new_file.read_text()
            if new_file_content != current_file_content:
                self._lavacli.device_types_cmd(
                    lavacli_cmd, "set", self.device_type_name, str(new_file)
                )

    def deploy(self) -> None:
        """High-level interface for deploying this definition.

        This deploys the wrapped device type including device type dictionary
        and health check to the current LAVA instance.

        Raises:
            LavaError: If a call to lavacli fails for some reason.

        """
        if not self._type_exists():
            self._lavacli.device_types_cmd("add", self.device_type_name)

        self._deploy_config_file("device_type_dict")
        self._deploy_config_file("health_check")


class DevicesDefinition:
    # pylint: disable=too-few-public-methods
    """Device definitions for LAVA configuration deployment."""

    def __init__(
        self,
        yaml_path: Path,
        device_type_defs: Dict[str, DeviceTypeDefinition],
        lavacli: Lavacli,
    ):
        """Initialize an instance from a definition YAML file.

        Arguments:
            yaml_path: Path to the source YAML file.
            device_type_defs:
                DeviceTypeDefinition objects mapped by their device type names.
            lavacli: lavacli wrapper instance.

        """
        self._config_yaml = yaml.safe_load(yaml_path.read_text())
        if not self._config_yaml:
            self._config_yaml = {}  # Guard against empty input files.
        self._device_type_defs = device_type_defs
        self._lavacli = lavacli

    @staticmethod
    @contextmanager
    def _get_device_dict(
        device_type_def: DeviceTypeDefinition, **extra_template_kwargs: Any
    ) -> Generator[Path, None, None]:
        with device_type_def.get_config_file(
            config_key="device_dict", **extra_template_kwargs
        ) as device_dictionary_path:
            yield device_dictionary_path

    def deploy(self) -> None:
        """High-level interface for deploying this definition.

        This deploys a list of devices to LAVA. For each device it ensures that:
        It is added to the LAVA instance if it doesn't exist yet and that the
        list of tags and the device dictionary comply with the requested
        definition.

        Devices currently present in the LAVA instance but not listed in the new
        definition will be set to "RETIRED" in order to make them unavailable
        for test run. Device configurations are never removed from the LAVA
        instance, as all data associated to those devices would needed to be
        delete, including previous test results.
        """
        previous_devices = {
            d["hostname"]: LavaDevice(
                hostname=d["hostname"],
                health=LavaDeviceHealth(d["health"]),
                lavacli=self._lavacli,
            )
            for d in self._lavacli.devices_cmd_yaml("list", "--all", "--yaml")
        }

        for device_type, devices_yaml in self._config_yaml.items():
            device_type_def = self._device_type_defs[device_type]

            for device_yaml in devices_yaml:
                hostname = device_yaml["hostname"]
                try:
                    device = previous_devices.pop(hostname)
                except KeyError:
                    device = LavaDevice(
                        hostname=hostname, lavacli=self._lavacli
                    )

                with self._get_device_dict(
                    device_type_def=device_type_def,
                    **device_yaml["template_args"],
                ) as device_dict_file:
                    device.add_update_config(
                        lava_worker=device_yaml["lava-worker"],
                        device_type_name=device_type_def.device_type_name,
                        tags=device_yaml["tags"],
                        device_dict_file=device_dict_file,
                    )

        # Retire stale devices.
        for device in previous_devices.values():
            device.health = LavaDeviceHealth.RETIRED


class LAVAInstanceConfigDeployer:
    """LAVA instance configuration deployment helper.

    This class encapsulates the deployment of LAVA device tags, device types and
    devices into a LAVA instance. It starts off from the configuration
    repository and a configuration YAML in its root for the LAVA instance to
    configure.
    """

    def __init__(self, instance_definition_file: Path, lavacli_identity: str):
        """Initialize a LAVAInstanceConfigDeployer instance.

        Arguments:
            instance_definition_file:
                Path to the YAML file that define the instance to which
                configuration are to be deployed. It is the entry point for
                parsing the deployment definition.
            lavacli_identity:
                Name of the lavacli identity for connecting and authorizing at
                the LAVA instance to configure. This identity must be set up
                before using this class.

        """
        self._instace_def_file = instance_definition_file
        self._instance_defs = yaml.safe_load(self._instace_def_file.read_text())
        self._lavacli = Lavacli(identity=lavacli_identity)
        self._device_type_defs = {
            d.device_type_name: d
            for d in (
                DeviceTypeDefinition(
                    yaml_path=self._instace_def_file.parent / device_type_src,
                    lavacli=self._lavacli,
                )
                for device_type_src in self._instance_defs["device_types"]
            )
        }

    def configure(self) -> None:
        """Run the whole configuration deployment."""
        try:
            self.configure_tags()
            self.configure_device_types()
            self.configure_devices()
            print("Done.")
        except LavaError as e:
            print(
                "ERROR: LAVA instance configuration did not succeed: {}".format(
                    e
                )
            )
            return

    def configure_tags(self) -> None:
        """Configure device tags global to the LAVA instance."""
        print("Configuring device tags…")
        new_tags: Dict[str, str] = {}
        for tags_source in self._instance_defs["device_tags"]:
            new_tags_from_source = yaml.safe_load(
                (self._instace_def_file.parent / tags_source).read_text()
            )
            for tag in new_tags_from_source:
                name = tag["name"]
                if name in new_tags:
                    raise RuntimeError(
                        "LAVA device tags must be unique. At least following "
                        'tag was defined multiple times: "{}". Second '
                        "occurrence in {}.".format(
                            name, self._instace_def_file.parent / tags_source
                        )
                    )
                new_tags[name] = tag["description"]

        current_tags = self._lavacli.tags_cmd_yaml("list", "--yaml")
        current_tags = {t["name"]: t["description"] for t in current_tags}
        for name, description in new_tags.items():
            current_descr = current_tags.get(name)
            if current_descr is None:
                self._lavacli.tags_cmd(
                    "add", name, "--description", description
                )
            elif current_descr != description:
                print(
                    'WARNING: Tag "{}" already exists but was a different '
                    "description. This needs to be fixed manually.\n"
                    'Previous: "{}"\n'
                    'New:      "{}"'.format(name, current_descr, description)
                )

    def configure_device_types(self) -> None:
        """Add and configure a device type in LAVA."""
        print("Configuring device types…")
        for _, definition in self._device_type_defs.items():
            definition.deploy()

    def configure_devices(self) -> None:
        """Add and configure devices of a specific type in LAVA."""
        print("Configuring devices…")
        for devices_src in self._instance_defs["devices"]:
            src_path = self._instace_def_file.parent / devices_src
            definition = DevicesDefinition(
                yaml_path=src_path,
                device_type_defs=self._device_type_defs,
                lavacli=self._lavacli,
            )
            definition.deploy()