Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 1 | #!/usr/bin/python -B |
| 2 | |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 3 | # Copyright 2017 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """Generates the timezone data files used by Android.""" |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 18 | |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 19 | import glob |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 20 | import os |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 21 | import re |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 22 | import subprocess |
| 23 | import sys |
| 24 | import tarfile |
| 25 | import tempfile |
| 26 | |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 27 | sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 28 | import i18nutil |
Neil Fuller | fa89b03 | 2017-06-14 15:58:26 +0100 | [diff] [blame] | 29 | import icuutil |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 30 | import tzdatautil |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 31 | |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 32 | |
| 33 | # Calculate the paths that are referred to by multiple functions. |
| 34 | android_build_top = i18nutil.GetAndroidRootOrDie() |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 35 | timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) |
| 36 | i18nutil.CheckDirExists(timezone_dir, 'system/timezone') |
| 37 | |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 38 | android_host_out = i18nutil.GetAndroidHostOutOrDie() |
| 39 | |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 40 | zone_compactor_dir = os.path.realpath('%s/system/timezone/zone_compactor' % android_build_top) |
| 41 | i18nutil.CheckDirExists(timezone_dir, 'system/timezone/zone_zompactor') |
| 42 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 43 | timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 44 | timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) |
| 45 | |
Neil Fuller | 35467e1 | 2017-06-12 16:55:44 +0100 | [diff] [blame] | 46 | timezone_output_data_dir = '%s/output_data' % timezone_dir |
| 47 | i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 48 | |
| 49 | tmp_dir = tempfile.mkdtemp('-tzdata') |
| 50 | |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 51 | def GenerateZicInputFile(extracted_iana_data_dir): |
| 52 | # Android APIs assume DST means "summer time" so we follow the rearguard format |
| 53 | # introduced in 2018e. |
| 54 | zic_input_file_name = 'rearguard.zi' |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 55 | |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 56 | # 'NDATA=' is used to remove unnecessary rules files. |
| 57 | subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) |
| 58 | |
| 59 | zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) |
| 60 | if not os.path.exists(zic_input_file): |
| 61 | print 'Could not find %s' % zic_input_file |
| 62 | sys.exit(1) |
| 63 | return zic_input_file |
| 64 | |
| 65 | |
| 66 | def WriteSetupFile(zic_input_file): |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 67 | """Writes the list of zones that ZoneCompactor should process.""" |
| 68 | links = [] |
| 69 | zones = [] |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 70 | for line in open(zic_input_file): |
| 71 | fields = line.split() |
| 72 | if fields: |
| 73 | if fields[0] == 'Link': |
| 74 | links.append('%s %s %s' % (fields[0], fields[1], fields[2])) |
| 75 | zones.append(fields[2]) |
| 76 | elif fields[0] == 'Zone': |
| 77 | zones.append(fields[1]) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 78 | zones.sort() |
| 79 | |
| 80 | zone_compactor_setup_file = '%s/setup' % tmp_dir |
| 81 | setup = open(zone_compactor_setup_file, 'w') |
| 82 | for link in sorted(set(links)): |
| 83 | setup.write('%s\n' % link) |
| 84 | for zone in sorted(set(zones)): |
| 85 | setup.write('%s\n' % zone) |
| 86 | setup.close() |
| 87 | return zone_compactor_setup_file |
| 88 | |
| 89 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 90 | def BuildIcuData(iana_data_tar_file): |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 91 | icu_build_dir = '%s/icu' % tmp_dir |
| 92 | |
Neil Fuller | fa89b03 | 2017-06-14 15:58:26 +0100 | [diff] [blame] | 93 | icuutil.PrepareIcuBuild(icu_build_dir) |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 94 | icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) |
Neil Fuller | fa89b03 | 2017-06-14 15:58:26 +0100 | [diff] [blame] | 95 | |
| 96 | # Create ICU system image files. |
| 97 | icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) |
| 98 | |
| 99 | # Create the ICU overlay time zone file. |
| 100 | icu_overlay_dat_file = '%s/icu_overlay/icu_tzdata.dat' % timezone_output_data_dir |
| 101 | icuutil.MakeAndCopyOverlayTzIcuData(icu_build_dir, icu_overlay_dat_file) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 102 | |
| 103 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 104 | def GetIanaVersion(iana_tar_file): |
| 105 | iana_tar_filename = os.path.basename(iana_tar_file) |
| 106 | iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 107 | return iana_version |
| 108 | |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 109 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 110 | def ExtractTarFile(tar_file, dir): |
| 111 | print 'Extracting %s...' % tar_file |
| 112 | if not os.path.exists(dir): |
| 113 | os.mkdir(dir) |
| 114 | tar = tarfile.open(tar_file, 'r') |
| 115 | tar.extractall(dir) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 116 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 117 | |
| 118 | def BuildZic(iana_tools_dir): |
Neil Fuller | ad3a1e6 | 2018-05-14 16:36:40 +0100 | [diff] [blame] | 119 | iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode') |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 120 | iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) |
Neil Fuller | ad3a1e6 | 2018-05-14 16:36:40 +0100 | [diff] [blame] | 121 | iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata') |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 122 | iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) |
| 123 | |
| 124 | print 'Found IANA zic release %s/%s in %s/%s ...' \ |
| 125 | % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, iana_zic_data_tar_file) |
| 126 | |
| 127 | zic_build_dir = '%s/zic' % tmp_dir |
| 128 | ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) |
| 129 | ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) |
| 130 | |
| 131 | # zic |
| 132 | print 'Building zic...' |
| 133 | # VERSION_DEPS= is to stop the build process looking for files that might not |
| 134 | # be present across different versions. |
| 135 | subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) |
| 136 | |
| 137 | zic_binary_file = '%s/zic' % zic_build_dir |
| 138 | if not os.path.exists(zic_binary_file): |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 139 | print 'Could not find %s' % zic_binary_file |
| 140 | sys.exit(1) |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 141 | return zic_binary_file |
| 142 | |
| 143 | |
| 144 | def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 145 | print 'Generating zic input file...' |
| 146 | zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) |
| 147 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 148 | print 'Calling zic...' |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 149 | zic_output_dir = '%s/data' % tmp_dir |
| 150 | os.mkdir(zic_output_dir) |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 151 | zic_cmd = [zic_binary_file, '-d', zic_output_dir, zic_input_file] |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 152 | subprocess.check_call(zic_cmd) |
| 153 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 154 | # ZoneCompactor |
Neil Fuller | a591fb0 | 2018-05-08 17:29:11 +0100 | [diff] [blame] | 155 | zone_compactor_setup_file = WriteSetupFile(zic_input_file) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 156 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 157 | print 'Calling ZoneCompactor to update tzdata to %s...' % iana_data_version |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 158 | subprocess.check_call(['make', '-C', android_build_top, '-j30', 'zone_compactor']) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 159 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 160 | # Create args for ZoneCompactor |
| 161 | zone_tab_file = '%s/zone.tab' % extracted_iana_data_dir |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 162 | jar_file = '%s/framework/zone_compactor.jar' % android_host_out |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 163 | header_string = 'tzdata%s' % iana_data_version |
Neil Fuller | 35467e1 | 2017-06-12 16:55:44 +0100 | [diff] [blame] | 164 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 165 | print 'Executing ZoneCompactor...' |
Neil Fuller | 35467e1 | 2017-06-12 16:55:44 +0100 | [diff] [blame] | 166 | iana_output_data_dir = '%s/iana' % timezone_output_data_dir |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 167 | subprocess.check_call(['java', '-jar', jar_file, |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 168 | zone_compactor_setup_file, zic_output_dir, zone_tab_file, |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 169 | iana_output_data_dir, header_string]) |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 170 | |
| 171 | |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 172 | def BuildTzlookup(iana_data_dir): |
| 173 | countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir |
Neil Fuller | 35467e1 | 2017-06-12 16:55:44 +0100 | [diff] [blame] | 174 | tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 175 | |
| 176 | print 'Calling TzLookupGenerator to create tzlookup.xml...' |
| 177 | subprocess.check_call(['make', '-C', android_build_top, '-j30', 'tzlookup_generator']) |
| 178 | |
| 179 | jar_file = '%s/framework/tzlookup_generator.jar' % android_host_out |
| 180 | zone_tab_file = '%s/zone.tab' % iana_data_dir |
| 181 | subprocess.check_call(['java', '-jar', jar_file, |
| 182 | countryzones_source_file, zone_tab_file, tzlookup_dest_file]) |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 183 | |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 184 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 185 | def CreateDistroFiles(iana_data_version, output_dir): |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 186 | create_distro_script = '%s/distro/tools/create-distro.py' % timezone_dir |
| 187 | |
| 188 | tzdata_file = '%s/iana/tzdata' % timezone_output_data_dir |
| 189 | icu_file = '%s/icu_overlay/icu_tzdata.dat' % timezone_output_data_dir |
| 190 | tzlookup_file = '%s/android/tzlookup.xml' % timezone_output_data_dir |
| 191 | |
| 192 | distro_file_pattern = '%s/*.zip' % output_dir |
| 193 | existing_distro_files = glob.glob(distro_file_pattern) |
| 194 | |
| 195 | distro_file_metadata_pattern = '%s/*.txt' % output_dir |
| 196 | existing_distro_metadata_files = glob.glob(distro_file_metadata_pattern) |
| 197 | existing_files = existing_distro_files + existing_distro_metadata_files |
| 198 | |
| 199 | print 'Removing %s' % existing_files |
| 200 | for existing_file in existing_files: |
| 201 | os.remove(existing_file) |
| 202 | |
| 203 | subprocess.check_call([create_distro_script, |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 204 | '-iana_version', iana_data_version, |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 205 | '-tzdata', tzdata_file, |
| 206 | '-icu', icu_file, |
| 207 | '-tzlookup', tzlookup_file, |
| 208 | '-output', output_dir]) |
| 209 | |
Neil Fuller | ad3c876 | 2017-10-20 15:58:57 +0100 | [diff] [blame] | 210 | def UpdateTestFiles(): |
| 211 | testing_data_dir = '%s/testing/data' % timezone_dir |
| 212 | update_test_files_script = '%s/create-test-data.sh' % testing_data_dir |
| 213 | subprocess.check_call([update_test_files_script], cwd=testing_data_dir) |
| 214 | |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 215 | |
| 216 | # Run with no arguments from any directory, with no special setup required. |
| 217 | # See http://www.iana.org/time-zones/ for more about the source of this data. |
| 218 | def main(): |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 219 | print 'Source data file structure: %s' % timezone_input_data_dir |
| 220 | print 'Source tools file structure: %s' % timezone_input_tools_dir |
| 221 | print 'Output data file structure: %s' % timezone_output_data_dir |
Neil Fuller | 56166d3 | 2017-06-12 12:57:10 +0100 | [diff] [blame] | 222 | |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 223 | iana_input_data_dir = '%s/iana' % timezone_input_data_dir |
Neil Fuller | ad3a1e6 | 2018-05-14 16:36:40 +0100 | [diff] [blame] | 224 | iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata') |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 225 | iana_data_version = GetIanaVersion(iana_data_tar_file) |
| 226 | print 'IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file) |
Neil Fuller | fa89b03 | 2017-06-14 15:58:26 +0100 | [diff] [blame] | 227 | |
| 228 | icu_dir = icuutil.icuDir() |
| 229 | print 'Found icu in %s ...' % icu_dir |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 230 | |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 231 | BuildIcuData(iana_data_tar_file) |
| 232 | |
| 233 | iana_tools_dir = '%s/iana' % timezone_input_tools_dir |
| 234 | zic_binary_file = BuildZic(iana_tools_dir) |
| 235 | |
| 236 | iana_data_dir = '%s/iana_data' % tmp_dir |
| 237 | ExtractTarFile(iana_data_tar_file, iana_data_dir) |
| 238 | BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) |
Neil Fuller | 8cec561 | 2017-10-11 10:38:17 +0100 | [diff] [blame] | 239 | BuildTzlookup(iana_data_dir) |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 240 | |
| 241 | # Create a distro file from the output from prior stages. |
| 242 | distro_output_dir = '%s/distro' % timezone_output_data_dir |
Neil Fuller | cdf5695 | 2018-02-20 17:12:37 +0000 | [diff] [blame] | 243 | CreateDistroFiles(iana_data_version, distro_output_dir) |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 244 | |
Neil Fuller | ad3c876 | 2017-10-20 15:58:57 +0100 | [diff] [blame] | 245 | # Update test versions of distro files too. |
| 246 | UpdateTestFiles() |
| 247 | |
Neil Fuller | ad7e81c | 2017-06-15 16:45:56 +0100 | [diff] [blame] | 248 | print 'Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir) |
Neil Fuller | 86e72c5 | 2017-06-12 15:36:06 +0100 | [diff] [blame] | 249 | sys.exit(0) |
| 250 | |
| 251 | |
| 252 | if __name__ == '__main__': |
| 253 | main() |