Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 2 | """A crontab script to delete night test data.""" |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 3 | |
| 4 | from __future__ import print_function |
| 5 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 6 | __author__ = 'shenhan@google.com (Han Shen)' |
| 7 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 8 | import argparse |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 9 | import datetime |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 10 | import os |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 11 | import re |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 12 | import sys |
| 13 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 14 | from cros_utils import command_executer |
| 15 | from cros_utils import constants |
| 16 | from cros_utils import misc |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 17 | |
| 18 | DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') |
| 19 | |
| 20 | |
| 21 | def CleanNumberedDir(s, dry_run=False): |
| 22 | """Deleted directories under each dated_dir.""" |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 23 | chromeos_dirs = [ |
| 24 | os.path.join(s, x) for x in os.listdir(s) |
| 25 | if misc.IsChromeOsTree(os.path.join(s, x)) |
| 26 | ] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 27 | ce = command_executer.GetCommandExecuter(log_level='none') |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 28 | all_succeeded = True |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 29 | for cd in chromeos_dirs: |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 30 | if misc.DeleteChromeOsTree(cd, dry_run=dry_run): |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 31 | print('Successfully removed chromeos tree "{0}".'.format(cd)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 32 | else: |
| 33 | all_succeeded = False |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 34 | print('Failed to remove chromeos tree "{0}", please check.'.format(cd)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 35 | |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 36 | if not all_succeeded: |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 37 | print('Failed to delete at least one chromeos tree, please check.') |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 38 | return False |
| 39 | |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 40 | ## Now delete the numbered dir Before forcibly removing the directory, just |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 41 | ## check 's' to make sure it is sane. A valid dir to be removed must be |
| 42 | ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 43 | valid_dir_pattern = ( |
| 44 | '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 45 | if not re.search(valid_dir_pattern, s): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 46 | print('Trying to delete an invalid dir "{0}" (must match "{1}"), ' |
| 47 | 'please check.'.format(s, valid_dir_pattern)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 48 | return False |
| 49 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 50 | cmd = 'rm -fr {0}'.format(s) |
| 51 | if dry_run: |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 52 | print(cmd) |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 53 | else: |
Luis Lozano | 036c923 | 2015-12-10 10:47:01 -0800 | [diff] [blame] | 54 | if ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) == 0: |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 55 | print('Successfully removed "{0}".'.format(s)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 56 | else: |
| 57 | all_succeeded = False |
Yunlian Jiang | d97422a | 2015-12-16 11:06:13 -0800 | [diff] [blame] | 58 | print('Failed to remove "{0}", please check.'.format(s)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 59 | return all_succeeded |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 60 | |
| 61 | |
| 62 | def CleanDatedDir(dated_dir, dry_run=False): |
| 63 | # List subdirs under dir |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 64 | subdirs = [ |
| 65 | os.path.join(dated_dir, x) for x in os.listdir(dated_dir) |
| 66 | if os.path.isdir(os.path.join(dated_dir, x)) |
| 67 | ] |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 68 | all_succeeded = True |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 69 | for s in subdirs: |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 70 | if not CleanNumberedDir(s, dry_run): |
| 71 | all_succeeded = False |
| 72 | return all_succeeded |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 73 | |
| 74 | |
| 75 | def ProcessArguments(argv): |
| 76 | """Process arguments.""" |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 77 | parser = argparse.ArgumentParser( |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 78 | description='Automatically delete nightly test data directories.', |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 79 | usage='auto_delete_nightly_test_data.py options') |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 80 | parser.add_argument( |
| 81 | '-d', |
| 82 | '--dry_run', |
| 83 | dest='dry_run', |
| 84 | default=False, |
| 85 | action='store_true', |
| 86 | help='Only print command line, do not execute anything.') |
| 87 | parser.add_argument( |
| 88 | '--days_to_preserve', |
| 89 | dest='days_to_preserve', |
| 90 | default=3, |
| 91 | help=('Specify the number of days (not including today),' |
| 92 | ' test data generated on these days will *NOT* be ' |
| 93 | 'deleted. Defaults to 3.')) |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 94 | options = parser.parse_args(argv) |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 95 | return options |
| 96 | |
| 97 | |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 98 | def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run): |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 99 | rv = 0 |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 100 | ce = command_executer.GetCommandExecuter() |
| 101 | # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 102 | # accessed more than specified time. |
| 103 | minutes = 1440 * days_to_preserve |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 104 | cmd = (r'find {0} -maxdepth 1 -type d ' |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 105 | r'\( -name "test_that_*" -amin +{1} -o ' |
| 106 | r' -name "cros-update*" -amin +{1} -o ' |
| 107 | r' -regex "{0}/tmp......" -amin +{1} \) ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 108 | r'-exec bash -c "echo rm -fr {{}}" \; ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 109 | r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, minutes) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 110 | if dry_run: |
| 111 | print('Going to execute:\n%s' % cmd) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 112 | else: |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 113 | rv = ce.RunCommand(cmd, print_to_console=False) |
| 114 | if rv == 0: |
| 115 | print('Successfully cleaned chromeos tree tmp directory ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 116 | '"{0}".'.format(chroot_tmp)) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 117 | else: |
| 118 | print('Some directories were not removed under chromeos tree ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 119 | 'tmp directory -"{0}".'.format(chroot_tmp)) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 120 | |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 121 | return rv |
| 122 | |
| 123 | |
| 124 | def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve, |
| 125 | dry_run): |
| 126 | rv = 0 |
| 127 | rv2 = 0 |
| 128 | ce = command_executer.GetCommandExecuter() |
| 129 | minutes = 1440 * days_to_preserve |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 130 | # Clean image tar files, which were last accessed 1 hour ago and clean image |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 131 | # bin files that were last accessed more than specified time. |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 132 | cmd = ('find {0}/*{1} -type f ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 133 | r'\( -name "chromiumos_test_image.tar" -amin +60 -o ' |
| 134 | r' -name "chromiumos_test_image.tar.xz" -amin +60 -o ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 135 | r' -name "chromiumos_test_image.bin" -amin +{2} \) ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 136 | r'-exec bash -c "echo rm -f {{}}" \; ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 137 | r'-exec bash -c "rm -f {{}}" \;').format(chroot_tmp, subdir_suffix, |
| 138 | minutes) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 139 | |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 140 | if dry_run: |
| 141 | print('Going to execute:\n%s' % cmd) |
| 142 | else: |
| 143 | rv2 = ce.RunCommand(cmd, print_to_console=False) |
| 144 | if rv2 == 0: |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 145 | print('Successfully cleaned chromeos images from ' |
| 146 | '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 147 | else: |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 148 | print('Some chromeos images were not removed from ' |
| 149 | '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 150 | |
| 151 | rv += rv2 |
| 152 | |
| 153 | # Clean autotest files that were last accessed more than specified time. |
| 154 | rv2 = 0 |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 155 | cmd = (r'find {0}/*{1} -maxdepth 2 -type d ' |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 156 | r'\( -name "autotest_files" \) ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 157 | r'-amin +{2} ' |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 158 | r'-exec bash -c "echo rm -fr {{}}" \; ' |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 159 | r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, subdir_suffix, |
| 160 | minutes) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 161 | if dry_run: |
| 162 | print('Going to execute:\n%s' % cmd) |
| 163 | else: |
| 164 | rv2 = ce.RunCommand(cmd, print_to_console=False) |
| 165 | if rv2 == 0: |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 166 | print('Successfully cleaned chromeos image autotest directories from ' |
| 167 | '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 168 | else: |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 169 | print('Some image autotest directories were not removed from ' |
| 170 | '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 171 | |
| 172 | rv += rv2 |
| 173 | return rv |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 174 | |
| 175 | |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 176 | def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): |
| 177 | """Delete temporaries, images under crostc/chromeos.""" |
| 178 | chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', |
| 179 | 'chroot', 'tmp') |
| 180 | # Clean files in tmp directory |
| 181 | rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run) |
| 182 | # Clean image files in *-release directories |
| 183 | rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-release', |
| 184 | days_to_preserve, dry_run) |
| 185 | # Clean image files in *-pfq directories |
| 186 | rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve, |
| 187 | dry_run) |
| 188 | |
| 189 | return rv |
| 190 | |
| 191 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 192 | def Main(argv): |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 193 | """Delete nightly test data directories, tmps and test images.""" |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 194 | options = ProcessArguments(argv) |
| 195 | # Function 'isoweekday' returns 1(Monday) - 7 (Sunday). |
| 196 | d = datetime.datetime.today().isoweekday() |
| 197 | # We go back 1 week, delete from that day till we are |
| 198 | # options.days_to_preserve away from today. |
| 199 | s = d - 7 |
cmtice | 798a8fa | 2014-05-12 13:56:42 -0700 | [diff] [blame] | 200 | e = d - int(options.days_to_preserve) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 201 | rv = 0 |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 202 | for i in range(s + 1, e): |
| 203 | if i <= 0: |
| 204 | ## Wrap around if index is negative. 6 is from i + 7 - 1, because |
| 205 | ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 206 | dated_dir = DIR_BY_WEEKDAY[i + 6] |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 207 | else: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 208 | dated_dir = DIR_BY_WEEKDAY[i - 1] |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 209 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 210 | rv += 0 if CleanDatedDir( |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 211 | os.path.join(constants.CROSTC_WORKSPACE, dated_dir), |
| 212 | options.dry_run) else 1 |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 213 | |
Manoj Gupta | 4762fbb | 2017-01-23 12:56:45 -0800 | [diff] [blame] | 214 | ## Finally clean temporaries, images under crostc/chromeos |
Manoj Gupta | a8fd4f7 | 2016-12-06 11:40:58 -0800 | [diff] [blame] | 215 | rv2 = CleanChromeOsTmpAndImages( |
| 216 | int(options.days_to_preserve), options.dry_run) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 217 | |
| 218 | return rv + rv2 |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 219 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 220 | if __name__ == '__main__': |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 221 | retval = Main(sys.argv[1:]) |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 222 | sys.exit(retval) |