blob: d0d57a9949a0d99b623c78e2c7e758601aca2c3a [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Doug Zongkerea5d7a92010-09-12 15:26:16 -070015import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070016import errno
Doug Zongkereef39442009-04-02 12:14:19 -070017import getopt
18import getpass
Doug Zongker05d3dea2009-06-22 11:32:31 -070019import imp
Doug Zongkereef39442009-04-02 12:14:19 -070020import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080021import platform
Doug Zongkereef39442009-04-02 12:14:19 -070022import re
T.R. Fullhart37e10522013-03-18 10:31:26 -070023import shlex
Doug Zongkereef39442009-04-02 12:14:19 -070024import shutil
25import subprocess
26import sys
27import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070028import threading
29import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070030import zipfile
Doug Zongkereef39442009-04-02 12:14:19 -070031
Doug Zongker55d93282011-01-25 17:03:34 -080032try:
davidcad0bb92011-03-15 14:21:38 +000033 from hashlib import sha1 as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080034except ImportError:
davidcad0bb92011-03-15 14:21:38 +000035 from sha import sha as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080036
Doug Zongkereef39442009-04-02 12:14:19 -070037# missing in Python 2.4 and before
38if not hasattr(os, "SEEK_SET"):
39 os.SEEK_SET = 0
40
41class Options(object): pass
42OPTIONS = Options()
Doug Zongker602a84e2009-06-18 08:35:12 -070043OPTIONS.search_path = "out/host/linux-x86"
T.R. Fullhart37e10522013-03-18 10:31:26 -070044OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path
45OPTIONS.extra_signapk_args = []
46OPTIONS.java_path = "java" # Use the one on the path by default.
47OPTIONS.public_key_suffix = ".x509.pem"
48OPTIONS.private_key_suffix = ".pk8"
Doug Zongkereef39442009-04-02 12:14:19 -070049OPTIONS.verbose = False
50OPTIONS.tempfiles = []
Doug Zongker05d3dea2009-06-22 11:32:31 -070051OPTIONS.device_specific = None
Doug Zongker8bec09e2009-11-30 15:37:14 -080052OPTIONS.extras = {}
Doug Zongkerc77a9ad2010-09-16 11:28:43 -070053OPTIONS.info_dict = None
Doug Zongkereef39442009-04-02 12:14:19 -070054
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080055
56# Values for "certificate" in apkcerts that mean special things.
57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
58
59
Doug Zongkereef39442009-04-02 12:14:19 -070060class ExternalError(RuntimeError): pass
61
62
63def Run(args, **kwargs):
64 """Create and return a subprocess.Popen object, printing the command
65 line on the terminal if -v was specified."""
66 if OPTIONS.verbose:
67 print " running: ", " ".join(args)
68 return subprocess.Popen(args, **kwargs)
69
70
Ying Wang7e6d4e42010-12-13 16:25:36 -080071def CloseInheritedPipes():
72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
73 before doing other work."""
74 if platform.system() != "Darwin":
75 return
76 for d in range(3, 1025):
77 try:
78 stat = os.fstat(d)
79 if stat is not None:
80 pipebit = stat[0] & 0x1000
81 if pipebit != 0:
82 os.close(d)
83 except OSError:
84 pass
85
86
Doug Zongker37974732010-09-16 17:44:38 -070087def LoadInfoDict(zip):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070088 """Read and parse the META/misc_info.txt key/value pairs from the
89 input target files and return a dict."""
90
91 d = {}
92 try:
Doug Zongker37974732010-09-16 17:44:38 -070093 for line in zip.read("META/misc_info.txt").split("\n"):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070094 line = line.strip()
95 if not line or line.startswith("#"): continue
96 k, v = line.split("=", 1)
97 d[k] = v
Doug Zongker37974732010-09-16 17:44:38 -070098 except KeyError:
99 # ok if misc_info.txt doesn't exist
100 pass
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700101
Doug Zongker37974732010-09-16 17:44:38 -0700102 # backwards compatibility: These values used to be in their own
103 # files. Look for them, in case we're processing an old
104 # target_files zip.
105
106 if "mkyaffs2_extra_flags" not in d:
107 try:
108 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
109 except KeyError:
110 # ok if flags don't exist
111 pass
112
113 if "recovery_api_version" not in d:
114 try:
115 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
116 except KeyError:
117 raise ValueError("can't find recovery API version in input target-files")
118
119 if "tool_extensions" not in d:
120 try:
121 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
122 except KeyError:
123 # ok if extensions don't exist
124 pass
125
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800126 if "fstab_version" not in d:
127 d["fstab_version"] = "1"
128
Doug Zongker37974732010-09-16 17:44:38 -0700129 try:
130 data = zip.read("META/imagesizes.txt")
131 for line in data.split("\n"):
132 if not line: continue
Doug Zongker1684d9c2010-09-17 07:44:38 -0700133 name, value = line.split(" ", 1)
134 if not value: continue
Doug Zongker37974732010-09-16 17:44:38 -0700135 if name == "blocksize":
136 d[name] = value
137 else:
138 d[name + "_size"] = value
139 except KeyError:
140 pass
141
142 def makeint(key):
143 if key in d:
144 d[key] = int(d[key], 0)
145
146 makeint("recovery_api_version")
147 makeint("blocksize")
148 makeint("system_size")
149 makeint("userdata_size")
Ying Wang9f8e8db2011-11-04 11:37:01 -0700150 makeint("cache_size")
Doug Zongker37974732010-09-16 17:44:38 -0700151 makeint("recovery_size")
152 makeint("boot_size")
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800153 makeint("fstab_version")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700154
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800155 d["fstab"] = LoadRecoveryFSTab(zip, d["fstab_version"])
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700156 d["build.prop"] = LoadBuildProp(zip)
157 return d
158
159def LoadBuildProp(zip):
160 try:
161 data = zip.read("SYSTEM/build.prop")
162 except KeyError:
163 print "Warning: could not find SYSTEM/build.prop in %s" % zip
164 data = ""
165
166 d = {}
167 for line in data.split("\n"):
168 line = line.strip()
169 if not line or line.startswith("#"): continue
170 name, value = line.split("=", 1)
171 d[name] = value
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700172 return d
173
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800174def LoadRecoveryFSTab(zip, fstab_version):
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700175 class Partition(object):
176 pass
177
178 try:
179 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
180 except KeyError:
Jeff Davidson033fbe22011-10-26 18:08:09 -0700181 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip
182 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700183
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800184 if fstab_version == 1:
185 d = {}
186 for line in data.split("\n"):
187 line = line.strip()
188 if not line or line.startswith("#"): continue
189 pieces = line.split()
190 if not (3 <= len(pieces) <= 4):
191 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700192
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800193 p = Partition()
194 p.mount_point = pieces[0]
195 p.fs_type = pieces[1]
196 p.device = pieces[2]
197 p.length = 0
198 options = None
199 if len(pieces) >= 4:
200 if pieces[3].startswith("/"):
201 p.device2 = pieces[3]
202 if len(pieces) >= 5:
203 options = pieces[4]
204 else:
205 p.device2 = None
206 options = pieces[3]
Doug Zongker086cbb02011-02-17 15:54:20 -0800207 else:
208 p.device2 = None
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700209
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800210 if options:
211 options = options.split(",")
212 for i in options:
213 if i.startswith("length="):
214 p.length = int(i[7:])
215 else:
216 print "%s: unknown option \"%s\"" % (p.mount_point, i)
217
218 d[p.mount_point] = p
219
220 elif fstab_version == 2:
221 d = {}
222 for line in data.split("\n"):
223 line = line.strip()
224 if not line or line.startswith("#"): continue
225 pieces = line.split()
226 if len(pieces) != 5:
227 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
228
229 # Ignore entries that are managed by vold
230 options = pieces[4]
231 if "voldmanaged=" in options: continue
232
233 # It's a good line, parse it
234 p = Partition()
235 p.device = pieces[0]
236 p.mount_point = pieces[1]
237 p.fs_type = pieces[2]
238 p.device2 = None
239 p.length = 0
240
Doug Zongker086cbb02011-02-17 15:54:20 -0800241 options = options.split(",")
242 for i in options:
243 if i.startswith("length="):
244 p.length = int(i[7:])
245 else:
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800246 # Ignore all unknown options in the unified fstab
247 continue
Doug Zongker086cbb02011-02-17 15:54:20 -0800248
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800249 d[p.mount_point] = p
250
251 else:
252 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,))
253
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700254 return d
255
256
Doug Zongker37974732010-09-16 17:44:38 -0700257def DumpInfoDict(d):
258 for k, v in sorted(d.items()):
259 print "%-25s = (%s) %s" % (k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700260
Doug Zongkerd5131602012-08-02 14:46:42 -0700261def BuildBootableImage(sourcedir, fs_config_file, info_dict=None):
Doug Zongkereef39442009-04-02 12:14:19 -0700262 """Take a kernel, cmdline, and ramdisk directory from the input (in
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700263 'sourcedir'), and turn them into a boot image. Return the image
264 data, or None if sourcedir does not appear to contains files for
265 building the requested image."""
266
267 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
268 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
269 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700270
Doug Zongkerd5131602012-08-02 14:46:42 -0700271 if info_dict is None:
272 info_dict = OPTIONS.info_dict
273
Doug Zongkereef39442009-04-02 12:14:19 -0700274 ramdisk_img = tempfile.NamedTemporaryFile()
275 img = tempfile.NamedTemporaryFile()
276
Doug Zongkerfffe1d52012-05-03 16:15:29 -0700277 if os.access(fs_config_file, os.F_OK):
278 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")]
279 else:
280 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")]
281 p1 = Run(cmd, stdout=subprocess.PIPE)
Doug Zongker32da27a2009-05-29 09:35:56 -0700282 p2 = Run(["minigzip"],
283 stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
Doug Zongkereef39442009-04-02 12:14:19 -0700284
285 p2.wait()
286 p1.wait()
287 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
Doug Zongker32da27a2009-05-29 09:35:56 -0700288 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
Doug Zongkereef39442009-04-02 12:14:19 -0700289
Bjorn Andersson612e2cd2012-11-25 16:53:44 -0800290 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set
291 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg"
292
293 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")]
Doug Zongker38a649f2009-06-17 09:07:09 -0700294
Doug Zongker171f1cd2009-06-15 22:36:37 -0700295 fn = os.path.join(sourcedir, "cmdline")
296 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700297 cmd.append("--cmdline")
298 cmd.append(open(fn).read().rstrip("\n"))
299
300 fn = os.path.join(sourcedir, "base")
301 if os.access(fn, os.F_OK):
302 cmd.append("--base")
303 cmd.append(open(fn).read().rstrip("\n"))
304
Ying Wang4de6b5b2010-08-25 14:29:34 -0700305 fn = os.path.join(sourcedir, "pagesize")
306 if os.access(fn, os.F_OK):
307 cmd.append("--pagesize")
308 cmd.append(open(fn).read().rstrip("\n"))
309
Doug Zongkerd5131602012-08-02 14:46:42 -0700310 args = info_dict.get("mkbootimg_args", None)
311 if args and args.strip():
Jianxun Zhang09849492013-04-17 15:19:19 -0700312 cmd.extend(shlex.split(args))
Doug Zongkerd5131602012-08-02 14:46:42 -0700313
Doug Zongker38a649f2009-06-17 09:07:09 -0700314 cmd.extend(["--ramdisk", ramdisk_img.name,
315 "--output", img.name])
316
317 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700318 p.communicate()
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700319 assert p.returncode == 0, "mkbootimg of %s image failed" % (
320 os.path.basename(sourcedir),)
Doug Zongkereef39442009-04-02 12:14:19 -0700321
322 img.seek(os.SEEK_SET, 0)
323 data = img.read()
324
325 ramdisk_img.close()
326 img.close()
327
328 return data
329
330
Doug Zongkerd5131602012-08-02 14:46:42 -0700331def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,
332 info_dict=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800333 """Return a File object (with name 'name') with the desired bootable
334 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
335 'prebuilt_name', otherwise construct it from the source files in
336 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700337
Doug Zongker55d93282011-01-25 17:03:34 -0800338 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
339 if os.path.exists(prebuilt_path):
340 print "using prebuilt %s..." % (prebuilt_name,)
341 return File.FromLocalFile(name, prebuilt_path)
342 else:
343 print "building image from target_files %s..." % (tree_subdir,)
Doug Zongkerfffe1d52012-05-03 16:15:29 -0700344 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt"
345 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir),
Doug Zongkerd5131602012-08-02 14:46:42 -0700346 os.path.join(unpack_dir, fs_config),
347 info_dict))
Doug Zongker55d93282011-01-25 17:03:34 -0800348
Doug Zongkereef39442009-04-02 12:14:19 -0700349
Doug Zongker75f17362009-12-08 13:46:44 -0800350def UnzipTemp(filename, pattern=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800351 """Unzip the given archive into a temporary directory and return the name.
352
353 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
354 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
355
356 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
357 main file), open for reading.
358 """
Doug Zongkereef39442009-04-02 12:14:19 -0700359
360 tmp = tempfile.mkdtemp(prefix="targetfiles-")
361 OPTIONS.tempfiles.append(tmp)
Doug Zongker55d93282011-01-25 17:03:34 -0800362
363 def unzip_to_dir(filename, dirname):
364 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
365 if pattern is not None:
366 cmd.append(pattern)
367 p = Run(cmd, stdout=subprocess.PIPE)
368 p.communicate()
369 if p.returncode != 0:
370 raise ExternalError("failed to unzip input target-files \"%s\"" %
371 (filename,))
372
373 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
374 if m:
375 unzip_to_dir(m.group(1), tmp)
376 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
377 filename = m.group(1)
378 else:
379 unzip_to_dir(filename, tmp)
380
381 return tmp, zipfile.ZipFile(filename, "r")
Doug Zongkereef39442009-04-02 12:14:19 -0700382
383
384def GetKeyPasswords(keylist):
385 """Given a list of keys, prompt the user to enter passwords for
386 those which require them. Return a {key: password} dict. password
387 will be None if the key has no password."""
388
Doug Zongker8ce7c252009-05-22 13:34:54 -0700389 no_passwords = []
390 need_passwords = []
T.R. Fullhart37e10522013-03-18 10:31:26 -0700391 key_passwords = {}
Doug Zongkereef39442009-04-02 12:14:19 -0700392 devnull = open("/dev/null", "w+b")
393 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800394 # We don't need a password for things that aren't really keys.
395 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700396 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700397 continue
398
T.R. Fullhart37e10522013-03-18 10:31:26 -0700399 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
Doug Zongker602a84e2009-06-18 08:35:12 -0700400 "-inform", "DER", "-nocrypt"],
401 stdin=devnull.fileno(),
402 stdout=devnull.fileno(),
403 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700404 p.communicate()
405 if p.returncode == 0:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700406 # Definitely an unencrypted key.
Doug Zongker8ce7c252009-05-22 13:34:54 -0700407 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700408 else:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700409 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
410 "-inform", "DER", "-passin", "pass:"],
411 stdin=devnull.fileno(),
412 stdout=devnull.fileno(),
413 stderr=subprocess.PIPE)
414 stdout, stderr = p.communicate()
415 if p.returncode == 0:
416 # Encrypted key with empty string as password.
417 key_passwords[k] = ''
418 elif stderr.startswith('Error decrypting key'):
419 # Definitely encrypted key.
420 # It would have said "Error reading key" if it didn't parse correctly.
421 need_passwords.append(k)
422 else:
423 # Potentially, a type of key that openssl doesn't understand.
424 # We'll let the routines in signapk.jar handle it.
425 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700426 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700427
T.R. Fullhart37e10522013-03-18 10:31:26 -0700428 key_passwords.update(PasswordManager().GetPasswords(need_passwords))
Doug Zongker8ce7c252009-05-22 13:34:54 -0700429 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700430 return key_passwords
431
432
Doug Zongker951495f2009-08-14 12:44:19 -0700433def SignFile(input_name, output_name, key, password, align=None,
434 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700435 """Sign the input_name zip/jar/apk, producing output_name. Use the
436 given key and password (the latter may be None if the key does not
437 have a password.
438
439 If align is an integer > 1, zipalign is run to align stored files in
440 the output zip on 'align'-byte boundaries.
Doug Zongker951495f2009-08-14 12:44:19 -0700441
442 If whole_file is true, use the "-w" option to SignApk to embed a
443 signature that covers the whole file in the archive comment of the
444 zip file.
Doug Zongkereef39442009-04-02 12:14:19 -0700445 """
Doug Zongker951495f2009-08-14 12:44:19 -0700446
Doug Zongkereef39442009-04-02 12:14:19 -0700447 if align == 0 or align == 1:
448 align = None
449
450 if align:
451 temp = tempfile.NamedTemporaryFile()
452 sign_name = temp.name
453 else:
454 sign_name = output_name
455
T.R. Fullhart37e10522013-03-18 10:31:26 -0700456 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar",
457 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)]
458 cmd.extend(OPTIONS.extra_signapk_args)
Doug Zongker951495f2009-08-14 12:44:19 -0700459 if whole_file:
460 cmd.append("-w")
T.R. Fullhart37e10522013-03-18 10:31:26 -0700461 cmd.extend([key + OPTIONS.public_key_suffix,
462 key + OPTIONS.private_key_suffix,
Doug Zongker951495f2009-08-14 12:44:19 -0700463 input_name, sign_name])
464
465 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700466 if password is not None:
467 password += "\n"
468 p.communicate(password)
469 if p.returncode != 0:
470 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
471
472 if align:
Doug Zongker602a84e2009-06-18 08:35:12 -0700473 p = Run(["zipalign", "-f", str(align), sign_name, output_name])
Doug Zongkereef39442009-04-02 12:14:19 -0700474 p.communicate()
475 if p.returncode != 0:
476 raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
477 temp.close()
478
479
Doug Zongker37974732010-09-16 17:44:38 -0700480def CheckSize(data, target, info_dict):
Doug Zongkereef39442009-04-02 12:14:19 -0700481 """Check the data string passed against the max size limit, if
482 any, for the given target. Raise exception if the data is too big.
483 Print a warning if the data is nearing the maximum size."""
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700484
Doug Zongker1684d9c2010-09-17 07:44:38 -0700485 if target.endswith(".img"): target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700486 mount_point = "/" + target
487
488 if info_dict["fstab"]:
489 if mount_point == "/userdata": mount_point = "/data"
490 p = info_dict["fstab"][mount_point]
491 fs_type = p.fs_type
Andrew Boie0f9aec82012-02-14 09:32:52 -0800492 device = p.device
493 if "/" in device:
494 device = device[device.rfind("/")+1:]
495 limit = info_dict.get(device + "_size", None)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700496 if not fs_type or not limit: return
Doug Zongkereef39442009-04-02 12:14:19 -0700497
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700498 if fs_type == "yaffs2":
499 # image size should be increased by 1/64th to account for the
500 # spare area (64 bytes per 2k page)
501 limit = limit / 2048 * (2048+64)
Andrew Boie0f9aec82012-02-14 09:32:52 -0800502 size = len(data)
503 pct = float(size) * 100.0 / limit
504 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
505 if pct >= 99.0:
506 raise ExternalError(msg)
507 elif pct >= 95.0:
508 print
509 print " WARNING: ", msg
510 print
511 elif OPTIONS.verbose:
512 print " ", msg
Doug Zongkereef39442009-04-02 12:14:19 -0700513
514
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800515def ReadApkCerts(tf_zip):
516 """Given a target_files ZipFile, parse the META/apkcerts.txt file
517 and return a {package: cert} dict."""
518 certmap = {}
519 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
520 line = line.strip()
521 if not line: continue
522 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
523 r'private_key="(.*)"$', line)
524 if m:
525 name, cert, privkey = m.groups()
T.R. Fullhart37e10522013-03-18 10:31:26 -0700526 public_key_suffix_len = len(OPTIONS.public_key_suffix)
527 private_key_suffix_len = len(OPTIONS.private_key_suffix)
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800528 if cert in SPECIAL_CERT_STRINGS and not privkey:
529 certmap[name] = cert
T.R. Fullhart37e10522013-03-18 10:31:26 -0700530 elif (cert.endswith(OPTIONS.public_key_suffix) and
531 privkey.endswith(OPTIONS.private_key_suffix) and
532 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]):
533 certmap[name] = cert[:-public_key_suffix_len]
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800534 else:
535 raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
536 return certmap
537
538
Doug Zongkereef39442009-04-02 12:14:19 -0700539COMMON_DOCSTRING = """
540 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700541 Prepend <dir>/bin to the list of places to search for binaries
542 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700543
Doug Zongker05d3dea2009-06-22 11:32:31 -0700544 -s (--device_specific) <file>
545 Path to the python module containing device-specific
546 releasetools code.
547
Doug Zongker8bec09e2009-11-30 15:37:14 -0800548 -x (--extra) <key=value>
549 Add a key/value pair to the 'extras' dict, which device-specific
550 extension code may look at.
551
Doug Zongkereef39442009-04-02 12:14:19 -0700552 -v (--verbose)
553 Show command lines being executed.
554
555 -h (--help)
556 Display this usage message and exit.
557"""
558
559def Usage(docstring):
560 print docstring.rstrip("\n")
561 print COMMON_DOCSTRING
562
563
564def ParseOptions(argv,
565 docstring,
566 extra_opts="", extra_long_opts=(),
567 extra_option_handler=None):
568 """Parse the options in argv and return any arguments that aren't
569 flags. docstring is the calling module's docstring, to be displayed
570 for errors and -h. extra_opts and extra_long_opts are for flags
571 defined by the caller, which are processed by passing them to
572 extra_option_handler."""
573
574 try:
575 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800576 argv, "hvp:s:x:" + extra_opts,
T.R. Fullhart37e10522013-03-18 10:31:26 -0700577 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=",
578 "java_path=", "public_key_suffix=", "private_key_suffix=",
579 "device_specific=", "extra="] +
580 list(extra_long_opts))
Doug Zongkereef39442009-04-02 12:14:19 -0700581 except getopt.GetoptError, err:
582 Usage(docstring)
583 print "**", str(err), "**"
584 sys.exit(2)
585
586 path_specified = False
587
588 for o, a in opts:
589 if o in ("-h", "--help"):
590 Usage(docstring)
591 sys.exit()
592 elif o in ("-v", "--verbose"):
593 OPTIONS.verbose = True
594 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -0700595 OPTIONS.search_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -0700596 elif o in ("--signapk_path",):
597 OPTIONS.signapk_path = a
598 elif o in ("--extra_signapk_args",):
599 OPTIONS.extra_signapk_args = shlex.split(a)
600 elif o in ("--java_path",):
601 OPTIONS.java_path = a
602 elif o in ("--public_key_suffix",):
603 OPTIONS.public_key_suffix = a
604 elif o in ("--private_key_suffix",):
605 OPTIONS.private_key_suffix = a
Doug Zongker05d3dea2009-06-22 11:32:31 -0700606 elif o in ("-s", "--device_specific"):
607 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -0800608 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -0800609 key, value = a.split("=", 1)
610 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -0700611 else:
612 if extra_option_handler is None or not extra_option_handler(o, a):
613 assert False, "unknown option \"%s\"" % (o,)
614
Doug Zongker602a84e2009-06-18 08:35:12 -0700615 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
616 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -0700617
618 return args
619
620
621def Cleanup():
622 for i in OPTIONS.tempfiles:
623 if os.path.isdir(i):
624 shutil.rmtree(i)
625 else:
626 os.remove(i)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700627
628
629class PasswordManager(object):
630 def __init__(self):
631 self.editor = os.getenv("EDITOR", None)
632 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
633
634 def GetPasswords(self, items):
635 """Get passwords corresponding to each string in 'items',
636 returning a dict. (The dict may have keys in addition to the
637 values in 'items'.)
638
639 Uses the passwords in $ANDROID_PW_FILE if available, letting the
640 user edit that file to add more needed passwords. If no editor is
641 available, or $ANDROID_PW_FILE isn't define, prompts the user
642 interactively in the ordinary way.
643 """
644
645 current = self.ReadFile()
646
647 first = True
648 while True:
649 missing = []
650 for i in items:
651 if i not in current or not current[i]:
652 missing.append(i)
653 # Are all the passwords already in the file?
654 if not missing: return current
655
656 for i in missing:
657 current[i] = ""
658
659 if not first:
660 print "key file %s still missing some passwords." % (self.pwfile,)
661 answer = raw_input("try to edit again? [y]> ").strip()
662 if answer and answer[0] not in 'yY':
663 raise RuntimeError("key passwords unavailable")
664 first = False
665
666 current = self.UpdateAndReadFile(current)
667
668 def PromptResult(self, current):
669 """Prompt the user to enter a value (password) for each key in
670 'current' whose value is fales. Returns a new dict with all the
671 values.
672 """
673 result = {}
674 for k, v in sorted(current.iteritems()):
675 if v:
676 result[k] = v
677 else:
678 while True:
679 result[k] = getpass.getpass("Enter password for %s key> "
680 % (k,)).strip()
681 if result[k]: break
682 return result
683
684 def UpdateAndReadFile(self, current):
685 if not self.editor or not self.pwfile:
686 return self.PromptResult(current)
687
688 f = open(self.pwfile, "w")
689 os.chmod(self.pwfile, 0600)
690 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
691 f.write("# (Additional spaces are harmless.)\n\n")
692
693 first_line = None
694 sorted = [(not v, k, v) for (k, v) in current.iteritems()]
695 sorted.sort()
696 for i, (_, k, v) in enumerate(sorted):
697 f.write("[[[ %s ]]] %s\n" % (v, k))
698 if not v and first_line is None:
699 # position cursor on first line with no password.
700 first_line = i + 4
701 f.close()
702
703 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
704 _, _ = p.communicate()
705
706 return self.ReadFile()
707
708 def ReadFile(self):
709 result = {}
710 if self.pwfile is None: return result
711 try:
712 f = open(self.pwfile, "r")
713 for line in f:
714 line = line.strip()
715 if not line or line[0] == '#': continue
716 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
717 if not m:
718 print "failed to parse password file: ", line
719 else:
720 result[m.group(2)] = m.group(1)
721 f.close()
722 except IOError, e:
723 if e.errno != errno.ENOENT:
724 print "error reading password file: ", str(e)
725 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -0700726
727
728def ZipWriteStr(zip, filename, data, perms=0644):
729 # use a fixed timestamp so the output is repeatable.
730 zinfo = zipfile.ZipInfo(filename=filename,
731 date_time=(2009, 1, 1, 0, 0, 0))
732 zinfo.compress_type = zip.compression
733 zinfo.external_attr = perms << 16
734 zip.writestr(zinfo, data)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700735
736
737class DeviceSpecificParams(object):
738 module = None
739 def __init__(self, **kwargs):
740 """Keyword arguments to the constructor become attributes of this
741 object, which is passed to all functions in the device-specific
742 module."""
743 for k, v in kwargs.iteritems():
744 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -0800745 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -0700746
747 if self.module is None:
748 path = OPTIONS.device_specific
Doug Zongkerc18736b2009-09-30 09:20:32 -0700749 if not path: return
Doug Zongker8e2f2b92009-06-24 14:34:57 -0700750 try:
751 if os.path.isdir(path):
752 info = imp.find_module("releasetools", [path])
753 else:
754 d, f = os.path.split(path)
755 b, x = os.path.splitext(f)
756 if x == ".py":
757 f = b
758 info = imp.find_module(f, [d])
Doug Zongkereb0a78a2014-01-27 10:01:06 -0800759 print "loaded device-specific extensions from", path
Doug Zongker8e2f2b92009-06-24 14:34:57 -0700760 self.module = imp.load_module("device_specific", *info)
761 except ImportError:
762 print "unable to load device-specific module; assuming none"
Doug Zongker05d3dea2009-06-22 11:32:31 -0700763
764 def _DoCall(self, function_name, *args, **kwargs):
765 """Call the named function in the device-specific module, passing
766 the given args and kwargs. The first argument to the call will be
767 the DeviceSpecific object itself. If there is no module, or the
768 module does not define the function, return the value of the
769 'default' kwarg (which itself defaults to None)."""
770 if self.module is None or not hasattr(self.module, function_name):
771 return kwargs.get("default", None)
772 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
773
774 def FullOTA_Assertions(self):
775 """Called after emitting the block of assertions at the top of a
776 full OTA package. Implementations can add whatever additional
777 assertions they like."""
778 return self._DoCall("FullOTA_Assertions")
779
Doug Zongkere5ff5902012-01-17 10:55:37 -0800780 def FullOTA_InstallBegin(self):
781 """Called at the start of full OTA installation."""
782 return self._DoCall("FullOTA_InstallBegin")
783
Doug Zongker05d3dea2009-06-22 11:32:31 -0700784 def FullOTA_InstallEnd(self):
785 """Called at the end of full OTA installation; typically this is
786 used to install the image for the device's baseband processor."""
787 return self._DoCall("FullOTA_InstallEnd")
788
789 def IncrementalOTA_Assertions(self):
790 """Called after emitting the block of assertions at the top of an
791 incremental OTA package. Implementations can add whatever
792 additional assertions they like."""
793 return self._DoCall("IncrementalOTA_Assertions")
794
Doug Zongkere5ff5902012-01-17 10:55:37 -0800795 def IncrementalOTA_VerifyBegin(self):
796 """Called at the start of the verification phase of incremental
797 OTA installation; additional checks can be placed here to abort
798 the script before any changes are made."""
799 return self._DoCall("IncrementalOTA_VerifyBegin")
800
Doug Zongker05d3dea2009-06-22 11:32:31 -0700801 def IncrementalOTA_VerifyEnd(self):
802 """Called at the end of the verification phase of incremental OTA
803 installation; additional checks can be placed here to abort the
804 script before any changes are made."""
805 return self._DoCall("IncrementalOTA_VerifyEnd")
806
Doug Zongkere5ff5902012-01-17 10:55:37 -0800807 def IncrementalOTA_InstallBegin(self):
808 """Called at the start of incremental OTA installation (after
809 verification is complete)."""
810 return self._DoCall("IncrementalOTA_InstallBegin")
811
Doug Zongker05d3dea2009-06-22 11:32:31 -0700812 def IncrementalOTA_InstallEnd(self):
813 """Called at the end of incremental OTA installation; typically
814 this is used to install the image for the device's baseband
815 processor."""
816 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700817
818class File(object):
819 def __init__(self, name, data):
820 self.name = name
821 self.data = data
822 self.size = len(data)
Doug Zongker55d93282011-01-25 17:03:34 -0800823 self.sha1 = sha1(data).hexdigest()
824
825 @classmethod
826 def FromLocalFile(cls, name, diskname):
827 f = open(diskname, "rb")
828 data = f.read()
829 f.close()
830 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700831
832 def WriteToTemp(self):
833 t = tempfile.NamedTemporaryFile()
834 t.write(self.data)
835 t.flush()
836 return t
837
838 def AddToZip(self, z):
839 ZipWriteStr(z, self.name, self.data)
840
841DIFF_PROGRAM_BY_EXT = {
842 ".gz" : "imgdiff",
843 ".zip" : ["imgdiff", "-z"],
844 ".jar" : ["imgdiff", "-z"],
845 ".apk" : ["imgdiff", "-z"],
846 ".img" : "imgdiff",
847 }
848
849class Difference(object):
Doug Zongker24cd2802012-08-14 16:36:15 -0700850 def __init__(self, tf, sf, diff_program=None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700851 self.tf = tf
852 self.sf = sf
853 self.patch = None
Doug Zongker24cd2802012-08-14 16:36:15 -0700854 self.diff_program = diff_program
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700855
856 def ComputePatch(self):
857 """Compute the patch (as a string of data) needed to turn sf into
858 tf. Returns the same tuple as GetPatch()."""
859
860 tf = self.tf
861 sf = self.sf
862
Doug Zongker24cd2802012-08-14 16:36:15 -0700863 if self.diff_program:
864 diff_program = self.diff_program
865 else:
866 ext = os.path.splitext(tf.name)[1]
867 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700868
869 ttemp = tf.WriteToTemp()
870 stemp = sf.WriteToTemp()
871
872 ext = os.path.splitext(tf.name)[1]
873
874 try:
875 ptemp = tempfile.NamedTemporaryFile()
876 if isinstance(diff_program, list):
877 cmd = copy.copy(diff_program)
878 else:
879 cmd = [diff_program]
880 cmd.append(stemp.name)
881 cmd.append(ttemp.name)
882 cmd.append(ptemp.name)
883 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
884 _, err = p.communicate()
885 if err or p.returncode != 0:
886 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
887 return None
888 diff = ptemp.read()
889 finally:
890 ptemp.close()
891 stemp.close()
892 ttemp.close()
893
894 self.patch = diff
895 return self.tf, self.sf, self.patch
896
897
898 def GetPatch(self):
899 """Return a tuple (target_file, source_file, patch_data).
900 patch_data may be None if ComputePatch hasn't been called, or if
901 computing the patch failed."""
902 return self.tf, self.sf, self.patch
903
904
905def ComputeDifferences(diffs):
906 """Call ComputePatch on all the Difference objects in 'diffs'."""
907 print len(diffs), "diffs to compute"
908
909 # Do the largest files first, to try and reduce the long-pole effect.
910 by_size = [(i.tf.size, i) for i in diffs]
911 by_size.sort(reverse=True)
912 by_size = [i[1] for i in by_size]
913
914 lock = threading.Lock()
915 diff_iter = iter(by_size) # accessed under lock
916
917 def worker():
918 try:
919 lock.acquire()
920 for d in diff_iter:
921 lock.release()
922 start = time.time()
923 d.ComputePatch()
924 dur = time.time() - start
925 lock.acquire()
926
927 tf, sf, patch = d.GetPatch()
928 if sf.name == tf.name:
929 name = tf.name
930 else:
931 name = "%s (%s)" % (tf.name, sf.name)
932 if patch is None:
933 print "patching failed! %s" % (name,)
934 else:
935 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
936 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
937 lock.release()
938 except Exception, e:
939 print e
940 raise
941
942 # start worker threads; wait for them all to finish.
943 threads = [threading.Thread(target=worker)
944 for i in range(OPTIONS.worker_threads)]
945 for th in threads:
946 th.start()
947 while threads:
948 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -0700949
950
951# map recovery.fstab's fs_types to mount/format "partition types"
952PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
953 "ext4": "EMMC", "emmc": "EMMC" }
954
955def GetTypeAndDevice(mount_point, info):
956 fstab = info["fstab"]
957 if fstab:
958 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
959 else:
Ying Wanga73b6562011-03-03 21:52:08 -0800960 return None
Baligh Uddinbeb6afd2013-11-13 00:22:34 +0000961
962
963def ParseCertificate(data):
964 """Parse a PEM-format certificate."""
965 cert = []
966 save = False
967 for line in data.split("\n"):
968 if "--END CERTIFICATE--" in line:
969 break
970 if save:
971 cert.append(line)
972 if "--BEGIN CERTIFICATE--" in line:
973 save = True
974 cert = "".join(cert).decode('base64')
975 return cert