[autotest] Add a delete_file_throttler to throttle result size
This throttler deletes result files from largest first to reduce result size.
BUG=chromium:716218
TEST=unittest
Change-Id: Ib8a8e9727bed84a1a7143eacab3b70aba9ec1ff9
Reviewed-on: https://chromium-review.googlesource.com/557306
Commit-Ready: Dan Shi <dshi@google.com>
Tested-by: Dan Shi <dshi@google.com>
Reviewed-by: Dan Shi <dshi@google.com>
diff --git a/client/bin/result_tools/delete_file_throttler.py b/client/bin/result_tools/delete_file_throttler.py
new file mode 100644
index 0000000..5b55942
--- /dev/null
+++ b/client/bin/result_tools/delete_file_throttler.py
@@ -0,0 +1,58 @@
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This throttler reduces result size by deleting files permanently."""
+
+import os
+
+import throttler_lib
+import utils_lib
+
+
+# Default threshold of file size in KB for a file to be qualified for deletion.
+DEFAULT_FILE_SIZE_THRESHOLD_BYTE = 1024 * 1024
+
+def _delete_file(file_info):
+ """Delete the given file and update the summary.
+
+ @param file_info: A ResultInfo object containing summary for the file to be
+ shrunk.
+ """
+ utils_lib.LOG('Deleting file %s.' % file_info.path)
+ try:
+ os.remove(file_info.path)
+ except OSError as e:
+ utils_lib.LOG('Failed to delete file %s Error: %s' %
+ (file_info.path, e))
+
+ # Update the trimmed_size in ResultInfo.
+ file_info.trimmed_size = 0
+
+
+def throttle(summary, max_result_size_KB,
+ file_size_threshold_byte=DEFAULT_FILE_SIZE_THRESHOLD_BYTE,
+ exclude_file_patterns=[]):
+ """Throttle the files in summary by trimming file content.
+
+ Stop throttling until all files are processed or the result size is already
+ reduced to be under the given max_result_size_KB.
+
+ @param summary: A ResultInfo object containing result summary.
+ @param max_result_size_KB: Maximum test result size in KB.
+ @param file_size_threshold_byte: Threshold of file size in byte for a file
+ to be qualified for deletion. All qualified files will be deleted,
+ until all files are processed or the result size is under the given
+ max_result_size_KB.
+ @param exclude_file_patterns: A list of regex pattern for files not to be
+ throttled. Default is an empty list.
+ """
+ file_infos, _ = throttler_lib.sort_result_files(summary)
+ file_infos = throttler_lib.get_throttleable_files(
+ file_infos, exclude_file_patterns)
+
+ for info in file_infos:
+ if info.trimmed_size > file_size_threshold_byte:
+ _delete_file(info)
+ if throttler_lib.check_throttle_limit(summary, max_result_size_KB):
+ return
diff --git a/client/bin/result_tools/delete_file_throttler_unittest.py b/client/bin/result_tools/delete_file_throttler_unittest.py
new file mode 100644
index 0000000..0856289
--- /dev/null
+++ b/client/bin/result_tools/delete_file_throttler_unittest.py
@@ -0,0 +1,111 @@
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import common
+from autotest_lib.client.bin.result_tools import delete_file_throttler
+from autotest_lib.client.bin.result_tools import result_info
+from autotest_lib.client.bin.result_tools import unittest_lib
+from autotest_lib.client.bin.result_tools import utils_lib
+
+
+LARGE_SIZE_BYTE = 1000
+MEDIUM_SIZE_BYTE = 800
+SMALL_SIZE_BYTE = 100
+# Maximum result size is set to 2KB so the file with MEDIUM_SIZE_BYTE will be
+# kept.
+MAX_RESULT_SIZE_KB = 2
+# Any file with size above the threshold is qualified to be deleted.
+FILE_SIZE_THRESHOLD_BYTE = 512
+
+SUMMARY_AFTER_THROTTLE = {
+ '': {utils_lib.DIRS: [
+ {'file1.xml': {utils_lib.ORIGINAL_SIZE_BYTES: LARGE_SIZE_BYTE,
+ utils_lib.TRIMMED_SIZE_BYTES: 0}},
+ {'file2.jpg': {utils_lib.ORIGINAL_SIZE_BYTES: LARGE_SIZE_BYTE,
+ utils_lib.TRIMMED_SIZE_BYTES: 0}},
+ {'file3.log': {utils_lib.ORIGINAL_SIZE_BYTES: SMALL_SIZE_BYTE}},
+ {'file_to_keep': {utils_lib.ORIGINAL_SIZE_BYTES: MEDIUM_SIZE_BYTE}},
+ {'folder1': {
+ utils_lib.DIRS: [
+ {'file4': {utils_lib.ORIGINAL_SIZE_BYTES: LARGE_SIZE_BYTE,
+ utils_lib.TRIMMED_SIZE_BYTES: 0}},
+ {'keyval':
+ {utils_lib.ORIGINAL_SIZE_BYTES: LARGE_SIZE_BYTE}},
+ ],
+ utils_lib.ORIGINAL_SIZE_BYTES: 2 * LARGE_SIZE_BYTE,
+ utils_lib.TRIMMED_SIZE_BYTES: LARGE_SIZE_BYTE}}],
+ utils_lib.ORIGINAL_SIZE_BYTES:
+ 4 * LARGE_SIZE_BYTE + SMALL_SIZE_BYTE + MEDIUM_SIZE_BYTE,
+ utils_lib.TRIMMED_SIZE_BYTES:
+ LARGE_SIZE_BYTE + SMALL_SIZE_BYTE + MEDIUM_SIZE_BYTE}
+ }
+
+class ThrottleTest(unittest.TestCase):
+ """Test class for shrink_file_throttler.throttle method."""
+
+ def setUp(self):
+ """Setup directory for test."""
+ self.test_dir = tempfile.mkdtemp()
+ self.files_not_deleted = []
+ self.files_to_delete = []
+
+ file1 = os.path.join(self.test_dir, 'file1.xml')
+ unittest_lib.create_file(file1, LARGE_SIZE_BYTE)
+ self.files_to_delete.append(file1)
+
+ file2 = os.path.join(self.test_dir, 'file2.jpg')
+ unittest_lib.create_file(file2, LARGE_SIZE_BYTE)
+ self.files_to_delete.append(file2)
+
+ file_to_keep = os.path.join(self.test_dir, 'file_to_keep')
+ unittest_lib.create_file(file_to_keep, MEDIUM_SIZE_BYTE)
+ self.files_not_deleted.append(file_to_keep)
+
+ file3 = os.path.join(self.test_dir, 'file3.log')
+ unittest_lib.create_file(file3, SMALL_SIZE_BYTE)
+ self.files_not_deleted.append(file3)
+
+ folder1 = os.path.join(self.test_dir, 'folder1')
+ os.mkdir(folder1)
+ file4 = os.path.join(folder1, 'file4')
+ unittest_lib.create_file(file4, LARGE_SIZE_BYTE)
+ self.files_to_delete.append(file4)
+
+ protected_file = os.path.join(folder1, 'keyval')
+ unittest_lib.create_file(protected_file, LARGE_SIZE_BYTE)
+ self.files_not_deleted.append(protected_file)
+
+ def tearDown(self):
+ """Cleanup the test directory."""
+ shutil.rmtree(self.test_dir, ignore_errors=True)
+
+ def testTrim(self):
+ """Test throttle method."""
+ summary = result_info.ResultInfo.build_from_path(self.test_dir)
+ delete_file_throttler.throttle(
+ summary,
+ max_result_size_KB=MAX_RESULT_SIZE_KB,
+ file_size_threshold_byte=FILE_SIZE_THRESHOLD_BYTE)
+
+ self.assertEqual(SUMMARY_AFTER_THROTTLE, summary)
+
+ # Verify files that should not be deleted still exists.
+ for f in self.files_not_deleted:
+ self.assertTrue(os.stat(f).st_size > 0,
+ 'File %s should not be deleted!' % f)
+
+ # Verify files that should be deleted no longer exists.
+ for f in self.files_to_delete:
+ self.assertFalse(os.path.exists(f), 'File %s is not deleted!' % f)
+
+
+# this is so the test can be run in standalone mode
+if __name__ == '__main__':
+ """Main"""
+ unittest.main()
\ No newline at end of file