Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | """A crontab script to delete night test data.""" |
| 4 | __author__ = 'shenhan@google.com (Han Shen)' |
| 5 | |
| 6 | import datetime |
| 7 | import optparse |
| 8 | import os |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 9 | import re |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 10 | import sys |
| 11 | |
| 12 | from utils import command_executer |
| 13 | from utils import constants |
| 14 | from utils import misc |
| 15 | |
| 16 | DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') |
| 17 | |
| 18 | |
| 19 | def CleanNumberedDir(s, dry_run=False): |
| 20 | """Deleted directories under each dated_dir.""" |
| 21 | chromeos_dirs = [os.path.join(s, x) for x in os.listdir(s) |
| 22 | if misc.IsChromeOsTree(os.path.join(s, x))] |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 23 | ce = command_executer.GetCommandExecuter(log_level="none") |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 24 | all_succeeded = True |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 25 | for cd in chromeos_dirs: |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 26 | if misc.DeleteChromeOsTree(cd, dry_run=dry_run): |
| 27 | print 'Successfully removed chromeos tree "{0}".'.format(cd) |
| 28 | else: |
| 29 | all_succeeded = False |
| 30 | print 'Failed to remove chromeos tree "{0}", please check.'.format(cd) |
| 31 | |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 32 | if not all_succeeded: |
| 33 | print 'Failed to delete at least one chromeos tree, please check.' |
| 34 | return False |
| 35 | |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 36 | ## Now delete the numbered dir Before forcibly removing the directory, just |
Han Shen | 4375f76 | 2014-04-25 10:35:29 -0700 | [diff] [blame] | 37 | ## check 's' to make sure it is sane. A valid dir to be removed must be |
| 38 | ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. |
| 39 | valid_dir_pattern = ('^' + constants.CROSTC_WORKSPACE + '/(' + |
| 40 | '|'.join(DIR_BY_WEEKDAY) + ')') |
| 41 | if not re.search(valid_dir_pattern, s): |
| 42 | print ('Trying to delete an invalid dir "{0}" (must match "{1}"), ' |
| 43 | 'please check.'.format(s, valid_dir_pattern)) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 44 | return False |
| 45 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 46 | cmd = 'rm -fr {0}'.format(s) |
| 47 | if dry_run: |
| 48 | print cmd |
| 49 | else: |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 50 | if ce.RunCommand(cmd, return_output=False, print_to_console=False, |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 51 | terminated_timeout=480) == 0: |
| 52 | print 'Successfully removed "{0}".'.format(s) |
| 53 | else: |
| 54 | all_succeeded = False |
| 55 | print 'Failed to remove "{0}", please check.'.format(s) |
| 56 | return all_succeeded |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 57 | |
| 58 | |
| 59 | def CleanDatedDir(dated_dir, dry_run=False): |
| 60 | # List subdirs under dir |
| 61 | subdirs = [os.path.join(dated_dir, x) for x in os.listdir(dated_dir) |
| 62 | if os.path.isdir(os.path.join(dated_dir, x))] |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 63 | all_succeeded = True |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 64 | for s in subdirs: |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 65 | if not CleanNumberedDir(s, dry_run): |
| 66 | all_succeeded = False |
| 67 | return all_succeeded |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 68 | |
| 69 | |
| 70 | def ProcessArguments(argv): |
| 71 | """Process arguments.""" |
| 72 | parser = optparse.OptionParser( |
| 73 | description='Automatically delete nightly test data directories.', |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 74 | usage='auto_delete_nightly_test_data.py options') |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 75 | parser.add_option('-d', '--dry_run', dest='dry_run', |
| 76 | default=False, action='store_true', |
| 77 | help='Only print command line, do not execute anything.') |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 78 | parser.add_option('--days_to_preserve', dest='days_to_preserve', default=3, |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 79 | help=('Specify the number of days (not including today), ' |
| 80 | 'test data generated on these days will *NOT* be ' |
| 81 | 'deleted. Defaults to 3.')) |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 82 | options, _ = parser.parse_args(argv) |
| 83 | return options |
| 84 | |
| 85 | |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 86 | def CleanChromeOsTmpAndImages(): |
| 87 | """Delete temporaries, images under crostc/chromeos.""" |
| 88 | |
| 89 | chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, |
| 90 | 'chromeos', 'chroot', 'tmp') |
| 91 | |
| 92 | ce = command_executer.GetCommandExecuter() |
| 93 | # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last |
| 94 | # accessed more than 24 hours ago. |
| 95 | cmd = (r'find {0} -maxdepth 1 -type d ' |
| 96 | r'\( -name "test_that_*" -o -regex "{0}/tmp......" \) ' |
| 97 | r'-amin +1440 ' |
| 98 | r'-exec bash -c "echo rm -fr {{}}" \; ' |
| 99 | r'-exec bash -c "rm -fr {{}}" \;').format(chromeos_chroot_tmp) |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 100 | rv = ce.RunCommand(cmd, return_output=False, print_to_console=False) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 101 | if rv == 0: |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 102 | print ('Successfully cleaned chromeos tree tmp directory ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 103 | '"{0}".'.format(chromeos_chroot_tmp)) |
| 104 | else: |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 105 | print ('Some directories were not removed under chromeos tree ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 106 | 'tmp directory -"{0}".'.format(chromeos_chroot_tmp)) |
| 107 | |
| 108 | # Clean image tar files, which were last accessed 1 hour ago and clean image |
| 109 | # bin files that were last accessed more than 24 hours ago. |
| 110 | cmd = ('find {0}/*-release -type f ' |
| 111 | r'\( -name "chromiumos_test_image.tar" -amin +60 -o ' |
| 112 | r' -name "chromiumos_test_image.tar.xz" -amin +60 -o ' |
cmtice | 07eb87f | 2015-03-11 10:10:41 -0700 | [diff] [blame] | 113 | r' -name "chromiumos_test_image.bin" -amin +1440 \) ' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 114 | r'-exec bash -c "echo rm -f {{}}" \; ' |
| 115 | r'-exec bash -c "rm -f {{}}" \;').format(chromeos_chroot_tmp) |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 116 | rv2 = ce.RunCommand(cmd, return_output=False, print_to_console=False) |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 117 | if rv2 == 0: |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 118 | print 'Successfully cleaned chromeos images.' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 119 | else: |
Luis Lozano | 1af9294 | 2015-11-04 17:48:32 -0800 | [diff] [blame] | 120 | print 'Some chromeos images were not removed.' |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 121 | |
| 122 | return rv + rv2 |
| 123 | |
| 124 | |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 125 | def Main(argv): |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 126 | """Delete nightly test data directories, tmps and test images.""" |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 127 | options = ProcessArguments(argv) |
| 128 | # Function 'isoweekday' returns 1(Monday) - 7 (Sunday). |
| 129 | d = datetime.datetime.today().isoweekday() |
| 130 | # We go back 1 week, delete from that day till we are |
| 131 | # options.days_to_preserve away from today. |
| 132 | s = d - 7 |
cmtice | 798a8fa | 2014-05-12 13:56:42 -0700 | [diff] [blame] | 133 | e = d - int(options.days_to_preserve) |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 134 | rv = 0 |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 135 | for i in range(s + 1, e): |
| 136 | if i <= 0: |
| 137 | ## Wrap around if index is negative. 6 is from i + 7 - 1, because |
| 138 | ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7. |
| 139 | dated_dir = DIR_BY_WEEKDAY[i+6] |
| 140 | else: |
| 141 | dated_dir = DIR_BY_WEEKDAY[i-1] |
Han Shen | a58a824 | 2014-04-23 11:05:22 -0700 | [diff] [blame] | 142 | rv += 0 if CleanDatedDir(os.path.join( |
| 143 | constants.CROSTC_WORKSPACE, dated_dir), options.dry_run) else 1 |
Han Shen | fdd8a5b | 2015-02-18 09:37:52 -0800 | [diff] [blame] | 144 | |
| 145 | ## Finally clean temporaries, images under crostc/chromeos |
| 146 | rv2 = CleanChromeOsTmpAndImages() |
| 147 | |
| 148 | return rv + rv2 |
Han Shen | e4b6f22 | 2013-11-22 10:02:38 -0800 | [diff] [blame] | 149 | |
| 150 | |
| 151 | if __name__ == '__main__': |
| 152 | retval = Main(sys.argv) |
| 153 | sys.exit(retval) |