Update the zic version used when generating tzdata
Historically, Android has used zone info compiler (zic) 2014b
(or before) to generate the tzdata file used by libcore's ZoneInfo
and bionic's mktime / localtime, etc.
Originally the version used was implicit and was based
on the zic version available on the developer's machine.
The version to be used was made explicit for consistency
in commit cdf5695c2b83e2781c287d8d315eb903554940da.
This change upgrades the tools version to the most
recent available at the time of writing and also modifies
/ renames the download-iana-data.py script so it can also
download the files needed for building zic.
NOTE: This change likely cannot be applied on older
versions of Android as there are bugs in libcore.util.ZoneInfo
and android.text.format.Time. See bug 118835133.
Bug: 118835133
Bug: 129926912
Bug: 73719425
Test: Manual execution of download-iana-files.py with missing and
old files
Test: run cts-dev -m CtsLibcoreTestCases
Test: run cts-dev -m CtsBionicTestCases
Test: run cts-dev -m CtsTextTestCases
Test: run cts-dev -m CtsLibcoreOjTestCases
Change-Id: Ie443b6baec6f4ce62a65cd430663fffb96543c02
diff --git a/download-iana-files.py b/download-iana-files.py
new file mode 100755
index 0000000..bcf551f
--- /dev/null
+++ b/download-iana-files.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python -B
+
+# Copyright 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Downloads the latest IANA time zone files."""
+
+import argparse
+import ftplib
+import os
+import shutil
+import subprocess
+import sys
+
+sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP'))
+import i18nutil
+import tzdatautil
+
+# Calculate the paths that are referred to by multiple functions.
+android_build_top = i18nutil.GetAndroidRootOrDie()
+iana_data_dir = os.path.realpath('%s/system/timezone/input_data/iana' % android_build_top)
+iana_tools_dir = os.path.realpath('%s/system/timezone/input_tools/iana' % android_build_top)
+
+def FtpRetrieveFile(ftp, filename):
+ ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
+
+
+def CheckSignature(data_filename, signature_filename):
+ """Checks the signature of a file."""
+ print 'Verifying signature of %s using %s...' % (data_filename, signature_filename)
+ try:
+ subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
+ signature_filename, data_filename])
+ except subprocess.CalledProcessError as err:
+ print 'Unable to verify signature'
+ print '\n\n******'
+ print 'If this fails for you, you probably need to import Paul Eggert''s public key:'
+ print ' gpg --receive-keys ED97E90E62AA7E34'
+ print '******\n\n'
+ raise
+
+
+def FindLatestRemoteTar(ftp, file_prefix):
+ iana_tar_filenames = []
+
+ for filename in ftp.nlst():
+ if "/" in filename:
+ print "FTP server returned bogus file name"
+ sys.exit(1)
+
+ if filename.startswith(file_prefix) and filename.endswith('.tar.gz'):
+ iana_tar_filenames.append(filename)
+
+ iana_tar_filenames.sort(reverse=True)
+
+ if len(iana_tar_filenames) == 0:
+ print 'No files found'
+ sys.exit(1)
+
+ return iana_tar_filenames[0]
+
+
+def DownloadAndReplaceLocalFiles(file_prefixes, ftp, local_dir):
+ output_files = []
+
+ for file_prefix in file_prefixes:
+ latest_iana_tar_filename = FindLatestRemoteTar(ftp, file_prefix)
+ local_iana_tar_file = tzdatautil.GetIanaTarFile(local_dir, file_prefix)
+ if local_iana_tar_file:
+ local_iana_tar_filename = os.path.basename(local_iana_tar_file)
+ if latest_iana_tar_filename <= local_iana_tar_filename:
+ print('Latest remote file for %s is called %s and is older or the same as'
+ ' current local file %s'
+ % (local_dir, latest_iana_tar_filename, local_iana_tar_filename))
+ continue
+
+ print 'Found new %s* file for %s: %s' % (file_prefix, local_dir, latest_iana_tar_filename)
+ i18nutil.SwitchToNewTemporaryDirectory()
+
+ print 'Downloading file %s...' % latest_iana_tar_filename
+ FtpRetrieveFile(ftp, latest_iana_tar_filename)
+
+ signature_filename = '%s.asc' % latest_iana_tar_filename
+ print 'Downloading signature %s...' % signature_filename
+ FtpRetrieveFile(ftp, signature_filename)
+
+ CheckSignature(latest_iana_tar_filename, signature_filename)
+
+ new_local_iana_tar_file = '%s/%s' % (local_dir, latest_iana_tar_filename)
+ shutil.copyfile(latest_iana_tar_filename, new_local_iana_tar_file)
+ new_local_signature_file = '%s/%s' % (local_dir, signature_filename)
+ shutil.copyfile(signature_filename, new_local_signature_file)
+
+ output_files.append(new_local_iana_tar_file)
+ output_files.append(new_local_signature_file)
+
+ # Delete the existing local IANA tar file, if there is one.
+ if local_iana_tar_file:
+ os.remove(local_iana_tar_file)
+
+ local_signature_file = '%s.asc' % local_iana_tar_file
+ if os.path.exists(local_signature_file):
+ os.remove(local_signature_file)
+
+ return output_files
+
+# Run from any directory after having run source/envsetup.sh / lunch
+# See http://www.iana.org/time-zones/ for more about the source of this data.
+def main():
+ parser = argparse.ArgumentParser(description=('Update IANA files from upstream'))
+ parser.add_argument('--notools', help='Skip tools files', action='store_true')
+ parser.add_argument('--nodata', help='Skip data files', action='store_true')
+ args = parser.parse_args()
+ if args.notools and args.nodata:
+ parser.error("Nothing to do")
+ sys.exit(1)
+
+ print 'Looking for new IANA files...'
+
+ ftp = ftplib.FTP('ftp.iana.org')
+ ftp.login()
+ ftp.cwd('tz/releases')
+
+ output_files = []
+ if not args.notools:
+ # The tools and data files are kept separate to make the updates independent.
+ # This means we duplicate the tzdata20xx file (once in the tools dir, once
+ # in the data dir) but the contents of the data tar appear to be needed for
+ # the zic build.
+ new_files = DownloadAndReplaceLocalFiles(['tzdata20', 'tzcode20'], ftp, iana_tools_dir)
+ output_files += new_files
+ if not args.nodata:
+ new_files = DownloadAndReplaceLocalFiles(['tzdata20'], ftp, iana_data_dir)
+ output_files += new_files
+
+ if len(output_files) == 0:
+ print 'No files updated'
+ else:
+ print 'New files:'
+ for output_file in output_files:
+ print ' %s' % output_file
+
+
+if __name__ == '__main__':
+ main()