CameraITS: multi camera frame sync test.
This is the first of a series of CLs. This CL contains the overall
structure of the test. Sensor fusion box control needs to be added.
Test: Ran sensor fusion test directly and ensured 7 captures were
executed.
Bug: 77539367
Change-Id: I8b93f9a06ba59338934644fa335486e05fe35a39
diff --git a/apps/CameraITS/pymodules/its/cv2image.py b/apps/CameraITS/pymodules/its/cv2image.py
index 3020548..58da2c8 100644
--- a/apps/CameraITS/pymodules/its/cv2image.py
+++ b/apps/CameraITS/pymodules/its/cv2image.py
@@ -14,11 +14,13 @@
import os
import unittest
+import time
import cv2
import its.caps
import its.device
import its.error
+import its.image
import numpy
VGA_HEIGHT = 480
@@ -195,6 +197,104 @@
self.ynorm = float(top_left[1]) / scene.shape[0]
+def get_angle(input_img):
+ """Computes anglular inclination of chessboard in input_img.
+
+ Angle estimation algoritm description:
+ Input: 2D grayscale image of chessboard.
+ Output: Angle of rotation of chessboard perpendicular to
+ chessboard. Assumes chessboard and camera are parallel to
+ each other.
+
+ 1) Use adaptive threshold to make image binary
+ 2) Find countours
+ 3) Filter out small contours
+ 4) Filter out all non-square contours
+ 5) Compute most common square shape.
+ The assumption here is that the most common square instances
+ are the chessboard squares. We've shown that with our current
+ tuning, we can robustly identify the squares on the sensor fusion
+ chessboard.
+ 6) Return median angle of most common square shape.
+
+ USAGE NOTE: This function has been tuned to work for the chessboard used in
+ the sensor_fusion tests. See images in test_images/rotated_chessboard/ for
+ sample captures. If this function is used with other chessboards, it may not
+ work as expected.
+
+ TODO: Make algorithm more robust so it works on any type of
+ chessboard.
+
+ Args:
+ input_img (2D numpy.ndarray): Grayscale image stored as a 2D
+ numpy array.
+
+ Returns:
+ Median angle of squares in degrees identified in the image.
+ """
+ # Tuning parameters
+ min_square_area = (float) (input_img.shape[1] * 0.05)
+
+ # Creates copy of image to avoid modifying original.
+ img = numpy.array(input_img, copy=True)
+
+ # Scale pixel values from 0-1 to 0-255
+ img = img * 255
+ img = img.astype(numpy.uint8)
+
+ thresh = cv2.adaptiveThreshold(
+ img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 201, 2)
+
+ # Find all contours
+ _, contours, _ = cv2.findContours(
+ thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+ # Filter contours to squares only.
+ square_contours = []
+
+ for contour in contours:
+ rect = cv2.minAreaRect(contour)
+ _, (width, height), angle = rect
+
+ # Skip non-squares (with 0.1 tolerance)
+ tolerance = 0.1
+ if width < height * (1 - tolerance) or width > height * (1 + tolerance):
+ continue
+
+ # Remove very small contours.
+ # These are usually just tiny dots due to noise.
+ area = cv2.contourArea(contour)
+ if area < min_square_area:
+ continue
+
+ box = cv2.boxPoints(rect)
+ box = numpy.int0(box)
+ square_contours.append(contour)
+
+ areas = []
+ for contour in square_contours:
+ area = cv2.contourArea(contour)
+ areas.append(area)
+
+ median_area = numpy.median(areas)
+
+ filtered_squares = []
+ filtered_angles = []
+ for square in square_contours:
+ area = cv2.contourArea(square)
+ if area < median_area * 0.90 or area > median_area * 1.10:
+ continue
+
+ filtered_squares.append(square)
+ _, (width, height), angle = cv2.minAreaRect(square)
+ filtered_angles.append(angle)
+
+ if len(filtered_angles) < 10:
+ return None
+
+ return numpy.median(filtered_angles)
+
+
class __UnitTest(unittest.TestCase):
"""Run a suite of unit tests on this module.
"""
@@ -224,6 +324,56 @@
self.assertTrue(numpy.isclose(sharpness[4]/sharpness[8],
numpy.sqrt(2), atol=0.1))
+ def test_get_angle_identify_unrotated_chessboard_angle(self):
+ basedir = os.path.join(
+ os.path.dirname(__file__), 'test_images/rotated_chessboards/')
+
+ normal_img_path = os.path.join(basedir, 'normal.jpg')
+ wide_img_path = os.path.join(basedir, 'wide.jpg')
+
+ normal_img = cv2.cvtColor(
+ cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
+ wide_img = cv2.cvtColor(
+ cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)
+
+ assert get_angle(normal_img) == 0
+ assert get_angle(wide_img) == 0
+
+ def test_get_angle_identify_rotated_chessboard_angle(self):
+ basedir = os.path.join(
+ os.path.dirname(__file__), 'test_images/rotated_chessboards/')
+
+ # Array of the image files and angles containing rotated chessboards.
+ test_cases = [
+ ('_15_ccw', 15),
+ ('_30_ccw', 30),
+ ('_45_ccw', 45),
+ ('_60_ccw', 60),
+ ('_75_ccw', 75),
+ ('_90_ccw', 90)
+ ]
+
+ # For each rotated image pair (normal, wide). Check if angle is
+ # identified as expected.
+ for suffix, angle in test_cases:
+ # Define image paths
+ normal_img_path = os.path.join(
+ basedir, 'normal{}.jpg'.format(suffix))
+ wide_img_path = os.path.join(
+ basedir, 'wide{}.jpg'.format(suffix))
+
+ # Load and color convert images
+ normal_img = cv2.cvtColor(
+ cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
+ wide_img = cv2.cvtColor(
+ cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)
+
+ # Assert angle is as expected up to 2.0 degrees of accuracy.
+ assert numpy.isclose(
+ abs(get_angle(normal_img)), angle, 2.0)
+ assert numpy.isclose(
+ abs(get_angle(wide_img)), angle, 2.0)
+
if __name__ == '__main__':
unittest.main()
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal.jpg
new file mode 100644
index 0000000..e6418be
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_15_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_15_ccw.jpg
new file mode 100644
index 0000000..fa921c2
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_15_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_30_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_30_ccw.jpg
new file mode 100644
index 0000000..907f0d2
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_30_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_45_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_45_ccw.jpg
new file mode 100644
index 0000000..59dc939
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_45_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_60_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_60_ccw.jpg
new file mode 100644
index 0000000..7d11c40
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_60_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_75_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_75_ccw.jpg
new file mode 100644
index 0000000..1193bb1
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_75_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_90_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_90_ccw.jpg
new file mode 100644
index 0000000..ea233ae
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/normal_90_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide.jpg
new file mode 100644
index 0000000..a790506
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_15_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_15_ccw.jpg
new file mode 100644
index 0000000..1871bab
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_15_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_30_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_30_ccw.jpg
new file mode 100644
index 0000000..d3bff2a
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_30_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_45_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_45_ccw.jpg
new file mode 100644
index 0000000..1298752
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_45_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_60_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_60_ccw.jpg
new file mode 100644
index 0000000..642aeea
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_60_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_75_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_75_ccw.jpg
new file mode 100644
index 0000000..b224ae4
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_75_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_90_ccw.jpg b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_90_ccw.jpg
new file mode 100644
index 0000000..c2ad401
--- /dev/null
+++ b/apps/CameraITS/pymodules/its/test_images/rotated_chessboards/wide_90_ccw.jpg
Binary files differ
diff --git a/apps/CameraITS/tests/sensor_fusion/test_multi_camera_frame_sync.py b/apps/CameraITS/tests/sensor_fusion/test_multi_camera_frame_sync.py
new file mode 100644
index 0000000..2998c88
--- /dev/null
+++ b/apps/CameraITS/tests/sensor_fusion/test_multi_camera_frame_sync.py
@@ -0,0 +1,182 @@
+# Copyright 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import its.caps
+from its.cv2image import get_angle
+import its.device
+import its.image
+import its.objects
+import its.target
+
+import cv2
+import matplotlib
+from matplotlib import pylab
+import numpy
+import os
+
+ANGLE_MASK = 10 # degrees
+ANGULAR_DIFF_THRESHOLD = 5 # degrees
+ANGULAR_MOVEMENT_THRESHOLD = 45 # degrees
+NAME = os.path.basename(__file__).split(".")[0]
+NUM_CAPTURES = 100
+W = 640
+H = 480
+
+
+def _check_available_capabilities(props):
+ """Returns True if all required test capabilities are present."""
+ return all([
+ its.caps.compute_target_exposure(props),
+ its.caps.per_frame_control(props),
+ its.caps.logical_multi_camera(props),
+ its.caps.raw16(props),
+ its.caps.manual_sensor(props),
+ its.caps.sensor_fusion(props)])
+
+
+def _assert_camera_movement(frame_pairs_angles):
+ """Assert the angles between each frame pair are sufficiently different.
+
+ Different angles is an indication of camera movement.
+ """
+ angles = [i for i, j in frame_pairs_angles]
+ max_angle = numpy.amax(angles)
+ min_angle = numpy.amin(angles)
+ emsg = "Not enough phone movement!\n"
+ emsg += "min angle: %.2f, max angle: %.2f deg, THRESH: %d deg" % (
+ min_angle, max_angle, ANGULAR_MOVEMENT_THRESHOLD)
+ assert max_angle - min_angle > ANGULAR_MOVEMENT_THRESHOLD, emsg
+
+
+def _assert_angular_difference(angle_1, angle_2):
+ """Assert angular difference is within threshold."""
+ diff = abs(angle_2 - angle_1)
+
+ # Assert difference is less than threshold
+ emsg = "Diff between frame pair: %.1f. Threshold: %d deg." % (
+ diff, ANGULAR_DIFF_THRESHOLD)
+ assert diff < ANGULAR_DIFF_THRESHOLD, emsg
+
+
+def _mask_angles_near_extremes(frame_pairs_angles):
+ """Mask out the data near the top and bottom of angle range."""
+ masked_pairs_angles = [[i, j] for i, j in frame_pairs_angles
+ if ANGLE_MASK <= abs(i) <= 90-ANGLE_MASK]
+ return masked_pairs_angles
+
+
+def _plot_frame_pairs_angles(frame_pairs_angles, ids):
+ """Plot the extracted angles."""
+ matplotlib.pyplot.figure("Camera Rotation Angle")
+ cam0_angles = [i for i, j in frame_pairs_angles]
+ cam1_angles = [j for i, j in frame_pairs_angles]
+ pylab.plot(range(len(cam0_angles)), cam0_angles, "r", label="%s" % ids[0])
+ pylab.plot(range(len(cam1_angles)), cam1_angles, "g", label="%s" % ids[1])
+ pylab.legend()
+ pylab.xlabel("Camera frame number")
+ pylab.ylabel("Rotation angle (degrees)")
+ matplotlib.pyplot.savefig("%s_angles_plot.png" % (NAME))
+
+ matplotlib.pyplot.figure("Angle Diffs")
+ angle_diffs = [j-i for i, j in frame_pairs_angles]
+ pylab.plot(range(len(angle_diffs)), angle_diffs, "b",
+ label="cam%s-%s" % (ids[1], ids[0]))
+ pylab.legend()
+ pylab.xlabel("Camera frame number")
+ pylab.ylabel("Rotation angle difference (degrees)")
+ matplotlib.pyplot.savefig("%s_angle_diffs_plot.png" % (NAME))
+
+def _collect_data():
+ """Returns list of pair of gray frames and camera ids used for captures."""
+ yuv_sizes = {}
+ with its.device.ItsSession() as cam:
+ props = cam.get_camera_properties()
+
+ # If capabilities not present, skip.
+ its.caps.skip_unless(_check_available_capabilities(props))
+
+ # Determine return parameters
+ debug = its.caps.debug_mode()
+ ids = its.caps.logical_multi_camera_physical_ids(props)
+
+ # Define capture request
+ s, e, _, _, f = cam.do_3a(get_results=True)
+ req = its.objects.manual_capture_request(s, e, f)
+
+ # capture YUVs
+ out_surfaces = [{"format": "yuv", "width": W, "height": H,
+ "physicalCamera": ids[0]},
+ {"format": "yuv", "width": W, "height": H,
+ "physicalCamera": ids[1]}]
+
+ capture_1_list, capture_2_list = cam.do_capture(
+ [req]*NUM_CAPTURES, out_surfaces)
+
+ # Create list of capture pairs. [[cap1A, cap1B], [cap2A, cap2B], ...]
+ frame_pairs = zip(capture_1_list, capture_2_list)
+
+ # Convert captures to grayscale
+ frame_pairs_gray = [
+ [
+ cv2.cvtColor(its.image.convert_capture_to_rgb_image(f, props=props), cv2.COLOR_RGB2GRAY) for f in pair
+ ] for pair in frame_pairs]
+
+ # Save images for debugging
+ if debug:
+ for i, imgs in enumerate(frame_pairs_gray):
+ for j in [0, 1]:
+ file_name = "%s_%s_%03d.png" % (NAME, ids[j], i)
+ cv2.imwrite(file_name, imgs[j]*255)
+
+ return frame_pairs_gray, ids
+
+def main():
+ """Test frame timestamps captured by logical camera are within 10ms."""
+ frame_pairs_gray, ids = _collect_data()
+
+ # Compute angles in frame pairs
+ frame_pairs_angles = [
+ [get_angle(p[0]), get_angle(p[1])] for p in frame_pairs_gray]
+
+ # Remove frames where not enough squares were detected.
+ filtered_pairs_angles = []
+ for angle_1, angle_2 in frame_pairs_angles:
+ if angle_1 == None or angle_2 == None:
+ continue
+ filtered_pairs_angles.append([angle_1, angle_2])
+
+ print 'Using {} image pairs to compute angular difference.'.format(
+ len(filtered_pairs_angles))
+
+ assert len(filtered_pairs_angles) > 20, (
+ "Unable to identify enough frames with detected squares.")
+
+ # Mask out data near 90 degrees.
+ # The chessboard angles we compute go from 0 to 89. Meaning,
+ # 90 degrees equals to 0 degrees.
+ # In order to avoid this jump, we ignore any frames at these extremeties.
+ masked_pairs_angles = _mask_angles_near_extremes(filtered_pairs_angles)
+
+ # Plot angles and differences
+ _plot_frame_pairs_angles(filtered_pairs_angles, ids)
+
+ # Ensure camera moved
+ _assert_camera_movement(filtered_pairs_angles)
+
+ # Ensure angle between images from each camera does not change appreciably
+ for cam_1_angle, cam_2_angle in masked_pairs_angles:
+ _assert_angular_difference(cam_1_angle, cam_2_angle)
+
+if __name__ == "__main__":
+ main()
