apps/CameraITS/pymodules/its/cv2image.py

# Copyright 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import unittest

import cv2
import its.caps
import its.device
import its.error
import its.image
import numpy

CHART_FILE = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules', 'its',
                          'test_images', 'ISO12233.png')
CHART_HEIGHT = 13.5  # cm
CHART_DISTANCE_RFOV = 31.0  # cm
CHART_DISTANCE_WFOV = 22.0  # cm
CHART_SCALE_START = 0.65
CHART_SCALE_STOP = 1.35
CHART_SCALE_STEP = 0.025

FOV_THRESH_SUPER_TELE = 40
FOV_THRESH_TELE = 60
FOV_THRESH_WFOV = 91

SCALE_RFOV_IN_WFOV_BOX = 0.67
SCALE_TELE_IN_RFOV_BOX = 0.67
SCALE_TELE_IN_WFOV_BOX = 0.5
SCALE_SUPER_TELE_IN_RFOV_BOX = 0.5

VGA_HEIGHT = 480
VGA_WIDTH = 640


def calc_chart_scaling(chart_distance, camera_fov):
    chart_scaling = 1.0
    camera_fov = float(camera_fov)
    if (FOV_THRESH_TELE < camera_fov < FOV_THRESH_WFOV and
                numpy.isclose(chart_distance, CHART_DISTANCE_WFOV, rtol=0.1)):
        chart_scaling = SCALE_RFOV_IN_WFOV_BOX
    elif (camera_fov <= FOV_THRESH_TELE and
          numpy.isclose(chart_distance, CHART_DISTANCE_WFOV, rtol=0.1)):
        chart_scaling = SCALE_TELE_IN_WFOV_BOX
    elif (camera_fov <= FOV_THRESH_SUPER_TELE and
          numpy.isclose(chart_distance, CHART_DISTANCE_RFOV, rtol=0.1)):
        chart_scaling = SCALE_SUPER_TELE_IN_RFOV_BOX
    elif (camera_fov <= FOV_THRESH_TELE and
          numpy.isclose(chart_distance, CHART_DISTANCE_RFOV, rtol=0.1)):
        chart_scaling = SCALE_TELE_IN_RFOV_BOX
    return chart_scaling


def scale_img(img, scale=1.0):
    """Scale and image based on a real number scale factor."""
    dim = (int(img.shape[1]*scale), int(img.shape[0]*scale))
    return cv2.resize(img.copy(), dim, interpolation=cv2.INTER_AREA)


def gray_scale_img(img):
    """Return gray scale version of image."""
    if len(img.shape) == 2:
        img_gray = img.copy()
    elif len(img.shape) == 3:
        if img.shape[2] == 1:
            img_gray = img[:, :, 0].copy()
        else:
            img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    return img_gray


class Chart(object):
    """Definition for chart object.

    Defines PNG reference file, chart size and distance, and scaling range.
    """

    def __init__(self, chart_file=None, height=None, distance=None,
                 scale_start=None, scale_stop=None, scale_step=None,
                 camera_id=None):
        """Initial constructor for class.

        Args:
            chart_file:     str; absolute path to png file of chart
            height:         float; height in cm of displayed chart
            distance:       float; distance in cm from camera of displayed chart
            scale_start:    float; start value for scaling for chart search
            scale_stop:     float; stop value for scaling for chart search
            scale_step:     float; step value for scaling for chart search
            camera_id:      int; camera used for extractor
        """
        self._file = chart_file or CHART_FILE
        self._height = height or CHART_HEIGHT
        self._distance = distance or CHART_DISTANCE_RFOV
        self._scale_start = scale_start or CHART_SCALE_START
        self._scale_stop = scale_stop or CHART_SCALE_STOP
        self._scale_step = scale_step or CHART_SCALE_STEP
        self.xnorm, self.ynorm, self.wnorm, self.hnorm, self.scale = its.image.chart_located_per_argv()
        if not self.xnorm:
            with its.device.ItsSession(camera_id) as cam:
                props = cam.get_camera_properties()
                props = cam.override_with_hidden_physical_camera_props(props)
                if its.caps.read_3a(props):
                    self.locate(cam, props)
                else:
                    print 'Chart locator skipped.'
                    self._set_scale_factors_to_one()

    def _set_scale_factors_to_one(self):
        """Set scale factors to 1.0 for skipped tests."""
        self.wnorm = 1.0
        self.hnorm = 1.0
        self.xnorm = 0.0
        self.ynorm = 0.0
        self.scale = 1.0

    def _calc_scale_factors(self, cam, props, fmt, s, e, fd):
        """Take an image with s, e, & fd to find the chart location.

        Args:
            cam:            An open device session.
            props:          Properties of cam
            fmt:            Image format for the capture
            s:              Sensitivity for the AF request as defined in
                            android.sensor.sensitivity
            e:              Exposure time for the AF request as defined in
                            android.sensor.exposureTime
            fd:             float; autofocus lens position
        Returns:
            template:       numpy array; chart template for locator
            img_3a:         numpy array; RGB image for chart location
            scale_factor:   float; scaling factor for chart search
        """
        req = its.objects.manual_capture_request(s, e)
        req['android.lens.focusDistance'] = fd
        cap_chart = its.image.stationary_lens_cap(cam, req, fmt)
        img_3a = its.image.convert_capture_to_rgb_image(cap_chart, props)
        img_3a = its.image.rotate_img_per_argv(img_3a)
        its.image.write_image(img_3a, 'af_scene.jpg')
        template = cv2.imread(self._file, cv2.IMREAD_ANYDEPTH)
        focal_l = cap_chart['metadata']['android.lens.focalLength']
        pixel_pitch = (props['android.sensor.info.physicalSize']['height'] /
                       img_3a.shape[0])
        print ' Chart distance: %.2fcm' % self._distance
        print ' Chart height: %.2fcm' % self._height
        print ' Focal length: %.2fmm' % focal_l
        print ' Pixel pitch: %.2fum' % (pixel_pitch*1E3)
        print ' Template height: %dpixels' % template.shape[0]
        chart_pixel_h = self._height * focal_l / (self._distance * pixel_pitch)
        scale_factor = template.shape[0] / chart_pixel_h
        print 'Chart/image scale factor = %.2f' % scale_factor
        return template, img_3a, scale_factor

    def locate(self, cam, props):
        """Find the chart in the image, and append location to chart object.

        The values appended are:
            xnorm:          float; [0, 1] left loc of chart in scene
            ynorm:          float; [0, 1] top loc of chart in scene
            wnorm:          float; [0, 1] width of chart in scene
            hnorm:          float; [0, 1] height of chart in scene
            scale:          float; scale factor to extract chart

        Args:
            cam:            An open device session
            props:          Camera properties
        """
        if its.caps.read_3a(props):
            s, e, _, _, fd = cam.do_3a(get_results=True)
            fmt = {'format': 'yuv', 'width': VGA_WIDTH, 'height': VGA_HEIGHT}
            chart, scene, s_factor = self._calc_scale_factors(cam, props, fmt,
                                                              s, e, fd)
        else:
            print 'Chart locator skipped.'
            self._set_scale_factors_to_one()
            return
        scale_start = self._scale_start * s_factor
        scale_stop = self._scale_stop * s_factor
        scale_step = self._scale_step * s_factor
        self.scale = s_factor
        max_match = []
        # check for normalized image
        if numpy.amax(scene) <= 1.0:
            scene = (scene * 255.0).astype(numpy.uint8)
        scene_gray = gray_scale_img(scene)
        print 'Finding chart in scene...'
        for scale in numpy.arange(scale_start, scale_stop, scale_step):
            scene_scaled = scale_img(scene_gray, scale)
            if (scene_scaled.shape[0] < chart.shape[0] or
                        scene_scaled.shape[1] < chart.shape[1]):
                continue
            result = cv2.matchTemplate(scene_scaled, chart, cv2.TM_CCOEFF)
            _, opt_val, _, top_left_scaled = cv2.minMaxLoc(result)
            # print out scale and match
            print ' scale factor: %.3f, opt val: %.f' % (scale, opt_val)
            max_match.append((opt_val, top_left_scaled))

        # determine if optimization results are valid
        opt_values = [x[0] for x in max_match]
        if 2.0*min(opt_values) > max(opt_values):
            estring = ('Warning: unable to find chart in scene!\n'
                       'Check camera distance and self-reported '
                       'pixel pitch, focal length and hyperfocal distance.')
            print estring
            self._set_scale_factors_to_one()
        else:
            if (max(opt_values) == opt_values[0] or
                        max(opt_values) == opt_values[len(opt_values)-1]):
                estring = ('Warning: chart is at extreme range of locator '
                           'check.\n')
                print estring
            # find max and draw bbox
            match_index = max_match.index(max(max_match, key=lambda x: x[0]))
            self.scale = scale_start + scale_step * match_index
            print 'Optimum scale factor: %.3f' %  self.scale
            top_left_scaled = max_match[match_index][1]
            h, w = chart.shape
            bottom_right_scaled = (top_left_scaled[0] + w,
                                   top_left_scaled[1] + h)
            top_left = (int(top_left_scaled[0]/self.scale),
                        int(top_left_scaled[1]/self.scale))
            bottom_right = (int(bottom_right_scaled[0]/self.scale),
                            int(bottom_right_scaled[1]/self.scale))
            self.wnorm = float((bottom_right[0]) - top_left[0]) / scene.shape[1]
            self.hnorm = float((bottom_right[1]) - top_left[1]) / scene.shape[0]
            self.xnorm = float(top_left[0]) / scene.shape[1]
            self.ynorm = float(top_left[1]) / scene.shape[0]


def get_angle(input_img):
    """Computes anglular inclination of chessboard in input_img.

    Angle estimation algoritm description:
        Input: 2D grayscale image of chessboard.
        Output: Angle of rotation of chessboard perpendicular to
            chessboard. Assumes chessboard and camera are parallel to
            each other.

        1) Use adaptive threshold to make image binary
        2) Find countours
        3) Filter out small contours
        4) Filter out all non-square contours
        5) Compute most common square shape.
            The assumption here is that the most common square instances
            are the chessboard squares. We've shown that with our current
            tuning, we can robustly identify the squares on the sensor fusion
            chessboard.
        6) Return median angle of most common square shape.

    USAGE NOTE: This function has been tuned to work for the chessboard used in
    the sensor_fusion tests. See images in test_images/rotated_chessboard/ for
    sample captures. If this function is used with other chessboards, it may not
    work as expected.

    TODO: Make algorithm more robust so it works on any type of
    chessboard.

    Args:
        input_img (2D numpy.ndarray): Grayscale image stored as a 2D
            numpy array.

    Returns:
        Median angle of squares in degrees identified in the image.
    """
    # Tuning parameters
    min_square_area = (float)(input_img.shape[1] * 0.05)

    # Creates copy of image to avoid modifying original.
    img = numpy.array(input_img, copy=True)

    # Scale pixel values from 0-1 to 0-255
    img *= 255
    img = img.astype(numpy.uint8)

    thresh = cv2.adaptiveThreshold(
            img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 201, 2)

    # Find all contours
    contours = []
    cv2_version = cv2.__version__
    if cv2_version.startswith('3.'): # OpenCV 3.x
        _, contours, _ = cv2.findContours(
                thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    else: # OpenCV 2.x and 4.x
        contours, _ = cv2.findContours(
                thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Filter contours to squares only.
    square_contours = []

    for contour in contours:
        rect = cv2.minAreaRect(contour)
        _, (width, height), angle = rect

        # Skip non-squares (with 0.1 tolerance)
        tolerance = 0.1
        if width < height * (1 - tolerance) or width > height * (1 + tolerance):
            continue

        # Remove very small contours.
        # These are usually just tiny dots due to noise.
        area = cv2.contourArea(contour)
        if area < min_square_area:
            continue

        if cv2_version.startswith('2.4.'):
            box = numpy.int0(cv2.cv.BoxPoints(rect))
        elif cv2_version.startswith('3.2.'):
            box = numpy.int0(cv2.boxPoints(rect))
        square_contours.append(contour)

    areas = []
    for contour in square_contours:
        area = cv2.contourArea(contour)
        areas.append(area)

    median_area = numpy.median(areas)

    filtered_squares = []
    filtered_angles = []
    for square in square_contours:
        area = cv2.contourArea(square)
        if area < median_area * 0.90 or area > median_area * 1.10:
            continue

        filtered_squares.append(square)
        _, (width, height), angle = cv2.minAreaRect(square)
        filtered_angles.append(angle)

    if len(filtered_angles) < 10:
        return None

    return numpy.median(filtered_angles)


class __UnitTest(unittest.TestCase):
    """Run a suite of unit tests on this module.
    """

    def test_compute_image_sharpness(self):
        """Unit test for compute_img_sharpness.

        Test by using PNG of ISO12233 chart and blurring intentionally.
        'sharpness' should drop off by sqrt(2) for 2x blur of image.

        We do one level of blur as PNG image is not perfect.
        """
        yuv_full_scale = 1023.0
        chart_file = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules',
                                  'its', 'test_images', 'ISO12233.png')
        chart = cv2.imread(chart_file, cv2.IMREAD_ANYDEPTH)
        white_level = numpy.amax(chart).astype(float)
        sharpness = {}
        for j in [2, 4, 8]:
            blur = cv2.blur(chart, (j, j))
            blur = blur[:, :, numpy.newaxis]
            sharpness[j] = (yuv_full_scale *
                            its.image.compute_image_sharpness(blur /
                                                              white_level))
        self.assertTrue(numpy.isclose(sharpness[2]/sharpness[4],
                                      numpy.sqrt(2), atol=0.1))
        self.assertTrue(numpy.isclose(sharpness[4]/sharpness[8],
                                      numpy.sqrt(2), atol=0.1))

    def test_get_angle_identify_unrotated_chessboard_angle(self):
        basedir = os.path.join(
                os.path.dirname(__file__), 'test_images/rotated_chessboards/')

        normal_img_path = os.path.join(basedir, 'normal.jpg')
        wide_img_path = os.path.join(basedir, 'wide.jpg')

        normal_img = cv2.cvtColor(
                cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
        wide_img = cv2.cvtColor(
                cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)

        assert get_angle(normal_img) == 0
        assert get_angle(wide_img) == 0

    def test_get_angle_identify_rotated_chessboard_angle(self):
        basedir = os.path.join(
                os.path.dirname(__file__), 'test_images/rotated_chessboards/')

        # Array of the image files and angles containing rotated chessboards.
        test_cases = [
                ('_15_ccw', 15),
                ('_30_ccw', 30),
                ('_45_ccw', 45),
                ('_60_ccw', 60),
                ('_75_ccw', 75),
                ('_90_ccw', 90)
        ]

        # For each rotated image pair (normal, wide). Check if angle is
        # identified as expected.
        for suffix, angle in test_cases:
            # Define image paths
            normal_img_path = os.path.join(
                    basedir, 'normal{}.jpg'.format(suffix))
            wide_img_path = os.path.join(
                    basedir, 'wide{}.jpg'.format(suffix))

            # Load and color convert images
            normal_img = cv2.cvtColor(
                    cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
            wide_img = cv2.cvtColor(
                    cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)

            # Assert angle is as expected up to 2.0 degrees of accuracy.
            assert numpy.isclose(
                    abs(get_angle(normal_img)), angle, 2.0)
            assert numpy.isclose(
                    abs(get_angle(wide_img)), angle, 2.0)


def component_shape(contour):
    """Measure the shape of a connected component.

    Args:
        contour: return from cv2.findContours. A list of pixel coordinates of
        the contour.

    Returns:
        The most left, right, top, bottom pixel location, height, width, and
        the center pixel location of the contour.
    """
    shape = {'left': numpy.inf, 'right': 0, 'top': numpy.inf, 'bottom': 0,
             'width': 0, 'height': 0, 'ctx': 0, 'cty': 0}
    for pt in contour:
        if pt[0][0] < shape['left']:
            shape['left'] = pt[0][0]
        if pt[0][0] > shape['right']:
            shape['right'] = pt[0][0]
        if pt[0][1] < shape['top']:
            shape['top'] = pt[0][1]
        if pt[0][1] > shape['bottom']:
            shape['bottom'] = pt[0][1]
    shape['width'] = shape['right'] - shape['left'] + 1
    shape['height'] = shape['bottom'] - shape['top'] + 1
    shape['ctx'] = (shape['left'] + shape['right']) / 2
    shape['cty'] = (shape['top'] + shape['bottom']) / 2
    return shape


if __name__ == '__main__':
    unittest.main()