blob: 1049591c8d5b21e6e2d98549d9dc1d2ccce0f26b [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Doug Zongkerea5d7a92010-09-12 15:26:16 -070015import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070016import errno
Doug Zongkereef39442009-04-02 12:14:19 -070017import getopt
18import getpass
Doug Zongker05d3dea2009-06-22 11:32:31 -070019import imp
Doug Zongkereef39442009-04-02 12:14:19 -070020import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080021import platform
Doug Zongkereef39442009-04-02 12:14:19 -070022import re
23import shutil
24import subprocess
25import sys
26import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070027import threading
28import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070029import zipfile
Doug Zongkereef39442009-04-02 12:14:19 -070030
Doug Zongker55d93282011-01-25 17:03:34 -080031try:
davidcad0bb92011-03-15 14:21:38 +000032 from hashlib import sha1 as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080033except ImportError:
davidcad0bb92011-03-15 14:21:38 +000034 from sha import sha as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080035
Doug Zongkereef39442009-04-02 12:14:19 -070036# missing in Python 2.4 and before
37if not hasattr(os, "SEEK_SET"):
38 os.SEEK_SET = 0
39
40class Options(object): pass
41OPTIONS = Options()
Doug Zongker602a84e2009-06-18 08:35:12 -070042OPTIONS.search_path = "out/host/linux-x86"
Doug Zongkereef39442009-04-02 12:14:19 -070043OPTIONS.verbose = False
44OPTIONS.tempfiles = []
Doug Zongker05d3dea2009-06-22 11:32:31 -070045OPTIONS.device_specific = None
Doug Zongker8bec09e2009-11-30 15:37:14 -080046OPTIONS.extras = {}
Doug Zongkerc77a9ad2010-09-16 11:28:43 -070047OPTIONS.info_dict = None
Doug Zongkereef39442009-04-02 12:14:19 -070048
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080049
50# Values for "certificate" in apkcerts that mean special things.
51SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
52
53
Doug Zongkereef39442009-04-02 12:14:19 -070054class ExternalError(RuntimeError): pass
55
56
57def Run(args, **kwargs):
58 """Create and return a subprocess.Popen object, printing the command
59 line on the terminal if -v was specified."""
60 if OPTIONS.verbose:
61 print " running: ", " ".join(args)
62 return subprocess.Popen(args, **kwargs)
63
64
Ying Wang7e6d4e42010-12-13 16:25:36 -080065def CloseInheritedPipes():
66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
67 before doing other work."""
68 if platform.system() != "Darwin":
69 return
70 for d in range(3, 1025):
71 try:
72 stat = os.fstat(d)
73 if stat is not None:
74 pipebit = stat[0] & 0x1000
75 if pipebit != 0:
76 os.close(d)
77 except OSError:
78 pass
79
80
Doug Zongker37974732010-09-16 17:44:38 -070081def LoadInfoDict(zip):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070082 """Read and parse the META/misc_info.txt key/value pairs from the
83 input target files and return a dict."""
84
85 d = {}
86 try:
Doug Zongker37974732010-09-16 17:44:38 -070087 for line in zip.read("META/misc_info.txt").split("\n"):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070088 line = line.strip()
89 if not line or line.startswith("#"): continue
90 k, v = line.split("=", 1)
91 d[k] = v
Doug Zongker37974732010-09-16 17:44:38 -070092 except KeyError:
93 # ok if misc_info.txt doesn't exist
94 pass
Doug Zongkerc19a8d52010-07-01 15:30:11 -070095
Doug Zongker37974732010-09-16 17:44:38 -070096 # backwards compatibility: These values used to be in their own
97 # files. Look for them, in case we're processing an old
98 # target_files zip.
99
100 if "mkyaffs2_extra_flags" not in d:
101 try:
102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
103 except KeyError:
104 # ok if flags don't exist
105 pass
106
107 if "recovery_api_version" not in d:
108 try:
109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
110 except KeyError:
111 raise ValueError("can't find recovery API version in input target-files")
112
113 if "tool_extensions" not in d:
114 try:
115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
116 except KeyError:
117 # ok if extensions don't exist
118 pass
119
120 try:
121 data = zip.read("META/imagesizes.txt")
122 for line in data.split("\n"):
123 if not line: continue
Doug Zongker1684d9c2010-09-17 07:44:38 -0700124 name, value = line.split(" ", 1)
125 if not value: continue
Doug Zongker37974732010-09-16 17:44:38 -0700126 if name == "blocksize":
127 d[name] = value
128 else:
129 d[name + "_size"] = value
130 except KeyError:
131 pass
132
133 def makeint(key):
134 if key in d:
135 d[key] = int(d[key], 0)
136
137 makeint("recovery_api_version")
138 makeint("blocksize")
139 makeint("system_size")
140 makeint("userdata_size")
Ying Wang9f8e8db2011-11-04 11:37:01 -0700141 makeint("cache_size")
Doug Zongker37974732010-09-16 17:44:38 -0700142 makeint("recovery_size")
143 makeint("boot_size")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700144
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700145 d["fstab"] = LoadRecoveryFSTab(zip)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700146 return d
147
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700148def LoadRecoveryFSTab(zip):
149 class Partition(object):
150 pass
151
152 try:
153 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
154 except KeyError:
Jeff Davidson033fbe22011-10-26 18:08:09 -0700155 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip
156 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700157
158 d = {}
159 for line in data.split("\n"):
160 line = line.strip()
161 if not line or line.startswith("#"): continue
162 pieces = line.split()
163 if not (3 <= len(pieces) <= 4):
164 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
165
166 p = Partition()
167 p.mount_point = pieces[0]
168 p.fs_type = pieces[1]
169 p.device = pieces[2]
Doug Zongker086cbb02011-02-17 15:54:20 -0800170 p.length = 0
171 options = None
172 if len(pieces) >= 4:
173 if pieces[3].startswith("/"):
174 p.device2 = pieces[3]
175 if len(pieces) >= 5:
176 options = pieces[4]
177 else:
178 p.device2 = None
179 options = pieces[3]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700180 else:
181 p.device2 = None
182
Doug Zongker086cbb02011-02-17 15:54:20 -0800183 if options:
184 options = options.split(",")
185 for i in options:
186 if i.startswith("length="):
187 p.length = int(i[7:])
188 else:
189 print "%s: unknown option \"%s\"" % (p.mount_point, i)
190
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700191 d[p.mount_point] = p
192 return d
193
194
Doug Zongker37974732010-09-16 17:44:38 -0700195def DumpInfoDict(d):
196 for k, v in sorted(d.items()):
197 print "%-25s = (%s) %s" % (k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700198
Doug Zongkereef39442009-04-02 12:14:19 -0700199def BuildBootableImage(sourcedir):
200 """Take a kernel, cmdline, and ramdisk directory from the input (in
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700201 'sourcedir'), and turn them into a boot image. Return the image
202 data, or None if sourcedir does not appear to contains files for
203 building the requested image."""
204
205 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
206 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
207 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700208
209 ramdisk_img = tempfile.NamedTemporaryFile()
210 img = tempfile.NamedTemporaryFile()
211
212 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
213 stdout=subprocess.PIPE)
Doug Zongker32da27a2009-05-29 09:35:56 -0700214 p2 = Run(["minigzip"],
215 stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
Doug Zongkereef39442009-04-02 12:14:19 -0700216
217 p2.wait()
218 p1.wait()
219 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
Doug Zongker32da27a2009-05-29 09:35:56 -0700220 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
Doug Zongkereef39442009-04-02 12:14:19 -0700221
Doug Zongker38a649f2009-06-17 09:07:09 -0700222 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
223
Doug Zongker171f1cd2009-06-15 22:36:37 -0700224 fn = os.path.join(sourcedir, "cmdline")
225 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700226 cmd.append("--cmdline")
227 cmd.append(open(fn).read().rstrip("\n"))
228
229 fn = os.path.join(sourcedir, "base")
230 if os.access(fn, os.F_OK):
231 cmd.append("--base")
232 cmd.append(open(fn).read().rstrip("\n"))
233
Ying Wang4de6b5b2010-08-25 14:29:34 -0700234 fn = os.path.join(sourcedir, "pagesize")
235 if os.access(fn, os.F_OK):
236 cmd.append("--pagesize")
237 cmd.append(open(fn).read().rstrip("\n"))
238
Doug Zongker38a649f2009-06-17 09:07:09 -0700239 cmd.extend(["--ramdisk", ramdisk_img.name,
240 "--output", img.name])
241
242 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700243 p.communicate()
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700244 assert p.returncode == 0, "mkbootimg of %s image failed" % (
245 os.path.basename(sourcedir),)
Doug Zongkereef39442009-04-02 12:14:19 -0700246
247 img.seek(os.SEEK_SET, 0)
248 data = img.read()
249
250 ramdisk_img.close()
251 img.close()
252
253 return data
254
255
Doug Zongker55d93282011-01-25 17:03:34 -0800256def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
257 """Return a File object (with name 'name') with the desired bootable
258 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
259 'prebuilt_name', otherwise construct it from the source files in
260 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700261
Doug Zongker55d93282011-01-25 17:03:34 -0800262 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
263 if os.path.exists(prebuilt_path):
264 print "using prebuilt %s..." % (prebuilt_name,)
265 return File.FromLocalFile(name, prebuilt_path)
266 else:
267 print "building image from target_files %s..." % (tree_subdir,)
268 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
269
Doug Zongkereef39442009-04-02 12:14:19 -0700270
Doug Zongker75f17362009-12-08 13:46:44 -0800271def UnzipTemp(filename, pattern=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800272 """Unzip the given archive into a temporary directory and return the name.
273
274 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
275 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
276
277 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
278 main file), open for reading.
279 """
Doug Zongkereef39442009-04-02 12:14:19 -0700280
281 tmp = tempfile.mkdtemp(prefix="targetfiles-")
282 OPTIONS.tempfiles.append(tmp)
Doug Zongker55d93282011-01-25 17:03:34 -0800283
284 def unzip_to_dir(filename, dirname):
285 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
286 if pattern is not None:
287 cmd.append(pattern)
288 p = Run(cmd, stdout=subprocess.PIPE)
289 p.communicate()
290 if p.returncode != 0:
291 raise ExternalError("failed to unzip input target-files \"%s\"" %
292 (filename,))
293
294 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
295 if m:
296 unzip_to_dir(m.group(1), tmp)
297 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
298 filename = m.group(1)
299 else:
300 unzip_to_dir(filename, tmp)
301
302 return tmp, zipfile.ZipFile(filename, "r")
Doug Zongkereef39442009-04-02 12:14:19 -0700303
304
305def GetKeyPasswords(keylist):
306 """Given a list of keys, prompt the user to enter passwords for
307 those which require them. Return a {key: password} dict. password
308 will be None if the key has no password."""
309
Doug Zongker8ce7c252009-05-22 13:34:54 -0700310 no_passwords = []
311 need_passwords = []
Doug Zongkereef39442009-04-02 12:14:19 -0700312 devnull = open("/dev/null", "w+b")
313 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800314 # We don't need a password for things that aren't really keys.
315 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700316 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700317 continue
318
Doug Zongker602a84e2009-06-18 08:35:12 -0700319 p = Run(["openssl", "pkcs8", "-in", k+".pk8",
320 "-inform", "DER", "-nocrypt"],
321 stdin=devnull.fileno(),
322 stdout=devnull.fileno(),
323 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700324 p.communicate()
325 if p.returncode == 0:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700326 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700327 else:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700328 need_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700329 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700330
331 key_passwords = PasswordManager().GetPasswords(need_passwords)
332 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700333 return key_passwords
334
335
Doug Zongker951495f2009-08-14 12:44:19 -0700336def SignFile(input_name, output_name, key, password, align=None,
337 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700338 """Sign the input_name zip/jar/apk, producing output_name. Use the
339 given key and password (the latter may be None if the key does not
340 have a password.
341
342 If align is an integer > 1, zipalign is run to align stored files in
343 the output zip on 'align'-byte boundaries.
Doug Zongker951495f2009-08-14 12:44:19 -0700344
345 If whole_file is true, use the "-w" option to SignApk to embed a
346 signature that covers the whole file in the archive comment of the
347 zip file.
Doug Zongkereef39442009-04-02 12:14:19 -0700348 """
Doug Zongker951495f2009-08-14 12:44:19 -0700349
Doug Zongkereef39442009-04-02 12:14:19 -0700350 if align == 0 or align == 1:
351 align = None
352
353 if align:
354 temp = tempfile.NamedTemporaryFile()
355 sign_name = temp.name
356 else:
357 sign_name = output_name
358
Doug Zongkerca855e92011-02-23 09:25:01 -0800359 cmd = ["java", "-Xmx2048m", "-jar",
Doug Zongker951495f2009-08-14 12:44:19 -0700360 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
361 if whole_file:
362 cmd.append("-w")
363 cmd.extend([key + ".x509.pem", key + ".pk8",
364 input_name, sign_name])
365
366 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700367 if password is not None:
368 password += "\n"
369 p.communicate(password)
370 if p.returncode != 0:
371 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
372
373 if align:
Doug Zongker602a84e2009-06-18 08:35:12 -0700374 p = Run(["zipalign", "-f", str(align), sign_name, output_name])
Doug Zongkereef39442009-04-02 12:14:19 -0700375 p.communicate()
376 if p.returncode != 0:
377 raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
378 temp.close()
379
380
Doug Zongker37974732010-09-16 17:44:38 -0700381def CheckSize(data, target, info_dict):
Doug Zongkereef39442009-04-02 12:14:19 -0700382 """Check the data string passed against the max size limit, if
383 any, for the given target. Raise exception if the data is too big.
384 Print a warning if the data is nearing the maximum size."""
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700385
Doug Zongker1684d9c2010-09-17 07:44:38 -0700386 if target.endswith(".img"): target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700387 mount_point = "/" + target
388
389 if info_dict["fstab"]:
390 if mount_point == "/userdata": mount_point = "/data"
391 p = info_dict["fstab"][mount_point]
392 fs_type = p.fs_type
393 limit = info_dict.get(p.device + "_size", None)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700394 if not fs_type or not limit: return
Doug Zongkereef39442009-04-02 12:14:19 -0700395
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700396 if fs_type == "yaffs2":
397 # image size should be increased by 1/64th to account for the
398 # spare area (64 bytes per 2k page)
399 limit = limit / 2048 * (2048+64)
Doug Zongker486de122010-09-16 14:01:56 -0700400 size = len(data)
401 pct = float(size) * 100.0 / limit
402 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
403 if pct >= 99.0:
404 raise ExternalError(msg)
405 elif pct >= 95.0:
406 print
407 print " WARNING: ", msg
408 print
409 elif OPTIONS.verbose:
410 print " ", msg
Doug Zongkereef39442009-04-02 12:14:19 -0700411
412
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800413def ReadApkCerts(tf_zip):
414 """Given a target_files ZipFile, parse the META/apkcerts.txt file
415 and return a {package: cert} dict."""
416 certmap = {}
417 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
418 line = line.strip()
419 if not line: continue
420 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
421 r'private_key="(.*)"$', line)
422 if m:
423 name, cert, privkey = m.groups()
424 if cert in SPECIAL_CERT_STRINGS and not privkey:
425 certmap[name] = cert
426 elif (cert.endswith(".x509.pem") and
427 privkey.endswith(".pk8") and
428 cert[:-9] == privkey[:-4]):
429 certmap[name] = cert[:-9]
430 else:
431 raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
432 return certmap
433
434
Doug Zongkereef39442009-04-02 12:14:19 -0700435COMMON_DOCSTRING = """
436 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700437 Prepend <dir>/bin to the list of places to search for binaries
438 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700439
Doug Zongker05d3dea2009-06-22 11:32:31 -0700440 -s (--device_specific) <file>
441 Path to the python module containing device-specific
442 releasetools code.
443
Doug Zongker8bec09e2009-11-30 15:37:14 -0800444 -x (--extra) <key=value>
445 Add a key/value pair to the 'extras' dict, which device-specific
446 extension code may look at.
447
Doug Zongkereef39442009-04-02 12:14:19 -0700448 -v (--verbose)
449 Show command lines being executed.
450
451 -h (--help)
452 Display this usage message and exit.
453"""
454
455def Usage(docstring):
456 print docstring.rstrip("\n")
457 print COMMON_DOCSTRING
458
459
460def ParseOptions(argv,
461 docstring,
462 extra_opts="", extra_long_opts=(),
463 extra_option_handler=None):
464 """Parse the options in argv and return any arguments that aren't
465 flags. docstring is the calling module's docstring, to be displayed
466 for errors and -h. extra_opts and extra_long_opts are for flags
467 defined by the caller, which are processed by passing them to
468 extra_option_handler."""
469
470 try:
471 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800472 argv, "hvp:s:x:" + extra_opts,
473 ["help", "verbose", "path=", "device_specific=", "extra="] +
Doug Zongker05d3dea2009-06-22 11:32:31 -0700474 list(extra_long_opts))
Doug Zongkereef39442009-04-02 12:14:19 -0700475 except getopt.GetoptError, err:
476 Usage(docstring)
477 print "**", str(err), "**"
478 sys.exit(2)
479
480 path_specified = False
481
482 for o, a in opts:
483 if o in ("-h", "--help"):
484 Usage(docstring)
485 sys.exit()
486 elif o in ("-v", "--verbose"):
487 OPTIONS.verbose = True
488 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -0700489 OPTIONS.search_path = a
Doug Zongker05d3dea2009-06-22 11:32:31 -0700490 elif o in ("-s", "--device_specific"):
491 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -0800492 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -0800493 key, value = a.split("=", 1)
494 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -0700495 else:
496 if extra_option_handler is None or not extra_option_handler(o, a):
497 assert False, "unknown option \"%s\"" % (o,)
498
Doug Zongker602a84e2009-06-18 08:35:12 -0700499 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
500 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -0700501
502 return args
503
504
505def Cleanup():
506 for i in OPTIONS.tempfiles:
507 if os.path.isdir(i):
508 shutil.rmtree(i)
509 else:
510 os.remove(i)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700511
512
513class PasswordManager(object):
514 def __init__(self):
515 self.editor = os.getenv("EDITOR", None)
516 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
517
518 def GetPasswords(self, items):
519 """Get passwords corresponding to each string in 'items',
520 returning a dict. (The dict may have keys in addition to the
521 values in 'items'.)
522
523 Uses the passwords in $ANDROID_PW_FILE if available, letting the
524 user edit that file to add more needed passwords. If no editor is
525 available, or $ANDROID_PW_FILE isn't define, prompts the user
526 interactively in the ordinary way.
527 """
528
529 current = self.ReadFile()
530
531 first = True
532 while True:
533 missing = []
534 for i in items:
535 if i not in current or not current[i]:
536 missing.append(i)
537 # Are all the passwords already in the file?
538 if not missing: return current
539
540 for i in missing:
541 current[i] = ""
542
543 if not first:
544 print "key file %s still missing some passwords." % (self.pwfile,)
545 answer = raw_input("try to edit again? [y]> ").strip()
546 if answer and answer[0] not in 'yY':
547 raise RuntimeError("key passwords unavailable")
548 first = False
549
550 current = self.UpdateAndReadFile(current)
551
552 def PromptResult(self, current):
553 """Prompt the user to enter a value (password) for each key in
554 'current' whose value is fales. Returns a new dict with all the
555 values.
556 """
557 result = {}
558 for k, v in sorted(current.iteritems()):
559 if v:
560 result[k] = v
561 else:
562 while True:
563 result[k] = getpass.getpass("Enter password for %s key> "
564 % (k,)).strip()
565 if result[k]: break
566 return result
567
568 def UpdateAndReadFile(self, current):
569 if not self.editor or not self.pwfile:
570 return self.PromptResult(current)
571
572 f = open(self.pwfile, "w")
573 os.chmod(self.pwfile, 0600)
574 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
575 f.write("# (Additional spaces are harmless.)\n\n")
576
577 first_line = None
578 sorted = [(not v, k, v) for (k, v) in current.iteritems()]
579 sorted.sort()
580 for i, (_, k, v) in enumerate(sorted):
581 f.write("[[[ %s ]]] %s\n" % (v, k))
582 if not v and first_line is None:
583 # position cursor on first line with no password.
584 first_line = i + 4
585 f.close()
586
587 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
588 _, _ = p.communicate()
589
590 return self.ReadFile()
591
592 def ReadFile(self):
593 result = {}
594 if self.pwfile is None: return result
595 try:
596 f = open(self.pwfile, "r")
597 for line in f:
598 line = line.strip()
599 if not line or line[0] == '#': continue
600 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
601 if not m:
602 print "failed to parse password file: ", line
603 else:
604 result[m.group(2)] = m.group(1)
605 f.close()
606 except IOError, e:
607 if e.errno != errno.ENOENT:
608 print "error reading password file: ", str(e)
609 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -0700610
611
612def ZipWriteStr(zip, filename, data, perms=0644):
613 # use a fixed timestamp so the output is repeatable.
614 zinfo = zipfile.ZipInfo(filename=filename,
615 date_time=(2009, 1, 1, 0, 0, 0))
616 zinfo.compress_type = zip.compression
617 zinfo.external_attr = perms << 16
618 zip.writestr(zinfo, data)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700619
620
621class DeviceSpecificParams(object):
622 module = None
623 def __init__(self, **kwargs):
624 """Keyword arguments to the constructor become attributes of this
625 object, which is passed to all functions in the device-specific
626 module."""
627 for k, v in kwargs.iteritems():
628 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -0800629 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -0700630
631 if self.module is None:
632 path = OPTIONS.device_specific
Doug Zongkerc18736b2009-09-30 09:20:32 -0700633 if not path: return
Doug Zongker8e2f2b92009-06-24 14:34:57 -0700634 try:
635 if os.path.isdir(path):
636 info = imp.find_module("releasetools", [path])
637 else:
638 d, f = os.path.split(path)
639 b, x = os.path.splitext(f)
640 if x == ".py":
641 f = b
642 info = imp.find_module(f, [d])
643 self.module = imp.load_module("device_specific", *info)
644 except ImportError:
645 print "unable to load device-specific module; assuming none"
Doug Zongker05d3dea2009-06-22 11:32:31 -0700646
647 def _DoCall(self, function_name, *args, **kwargs):
648 """Call the named function in the device-specific module, passing
649 the given args and kwargs. The first argument to the call will be
650 the DeviceSpecific object itself. If there is no module, or the
651 module does not define the function, return the value of the
652 'default' kwarg (which itself defaults to None)."""
653 if self.module is None or not hasattr(self.module, function_name):
654 return kwargs.get("default", None)
655 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
656
657 def FullOTA_Assertions(self):
658 """Called after emitting the block of assertions at the top of a
659 full OTA package. Implementations can add whatever additional
660 assertions they like."""
661 return self._DoCall("FullOTA_Assertions")
662
663 def FullOTA_InstallEnd(self):
664 """Called at the end of full OTA installation; typically this is
665 used to install the image for the device's baseband processor."""
666 return self._DoCall("FullOTA_InstallEnd")
667
668 def IncrementalOTA_Assertions(self):
669 """Called after emitting the block of assertions at the top of an
670 incremental OTA package. Implementations can add whatever
671 additional assertions they like."""
672 return self._DoCall("IncrementalOTA_Assertions")
673
674 def IncrementalOTA_VerifyEnd(self):
675 """Called at the end of the verification phase of incremental OTA
676 installation; additional checks can be placed here to abort the
677 script before any changes are made."""
678 return self._DoCall("IncrementalOTA_VerifyEnd")
679
680 def IncrementalOTA_InstallEnd(self):
681 """Called at the end of incremental OTA installation; typically
682 this is used to install the image for the device's baseband
683 processor."""
684 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700685
686class File(object):
687 def __init__(self, name, data):
688 self.name = name
689 self.data = data
690 self.size = len(data)
Doug Zongker55d93282011-01-25 17:03:34 -0800691 self.sha1 = sha1(data).hexdigest()
692
693 @classmethod
694 def FromLocalFile(cls, name, diskname):
695 f = open(diskname, "rb")
696 data = f.read()
697 f.close()
698 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700699
700 def WriteToTemp(self):
701 t = tempfile.NamedTemporaryFile()
702 t.write(self.data)
703 t.flush()
704 return t
705
706 def AddToZip(self, z):
707 ZipWriteStr(z, self.name, self.data)
708
709DIFF_PROGRAM_BY_EXT = {
710 ".gz" : "imgdiff",
711 ".zip" : ["imgdiff", "-z"],
712 ".jar" : ["imgdiff", "-z"],
713 ".apk" : ["imgdiff", "-z"],
714 ".img" : "imgdiff",
715 }
716
717class Difference(object):
718 def __init__(self, tf, sf):
719 self.tf = tf
720 self.sf = sf
721 self.patch = None
722
723 def ComputePatch(self):
724 """Compute the patch (as a string of data) needed to turn sf into
725 tf. Returns the same tuple as GetPatch()."""
726
727 tf = self.tf
728 sf = self.sf
729
730 ext = os.path.splitext(tf.name)[1]
731 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
732
733 ttemp = tf.WriteToTemp()
734 stemp = sf.WriteToTemp()
735
736 ext = os.path.splitext(tf.name)[1]
737
738 try:
739 ptemp = tempfile.NamedTemporaryFile()
740 if isinstance(diff_program, list):
741 cmd = copy.copy(diff_program)
742 else:
743 cmd = [diff_program]
744 cmd.append(stemp.name)
745 cmd.append(ttemp.name)
746 cmd.append(ptemp.name)
747 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
748 _, err = p.communicate()
749 if err or p.returncode != 0:
750 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
751 return None
752 diff = ptemp.read()
753 finally:
754 ptemp.close()
755 stemp.close()
756 ttemp.close()
757
758 self.patch = diff
759 return self.tf, self.sf, self.patch
760
761
762 def GetPatch(self):
763 """Return a tuple (target_file, source_file, patch_data).
764 patch_data may be None if ComputePatch hasn't been called, or if
765 computing the patch failed."""
766 return self.tf, self.sf, self.patch
767
768
769def ComputeDifferences(diffs):
770 """Call ComputePatch on all the Difference objects in 'diffs'."""
771 print len(diffs), "diffs to compute"
772
773 # Do the largest files first, to try and reduce the long-pole effect.
774 by_size = [(i.tf.size, i) for i in diffs]
775 by_size.sort(reverse=True)
776 by_size = [i[1] for i in by_size]
777
778 lock = threading.Lock()
779 diff_iter = iter(by_size) # accessed under lock
780
781 def worker():
782 try:
783 lock.acquire()
784 for d in diff_iter:
785 lock.release()
786 start = time.time()
787 d.ComputePatch()
788 dur = time.time() - start
789 lock.acquire()
790
791 tf, sf, patch = d.GetPatch()
792 if sf.name == tf.name:
793 name = tf.name
794 else:
795 name = "%s (%s)" % (tf.name, sf.name)
796 if patch is None:
797 print "patching failed! %s" % (name,)
798 else:
799 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
800 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
801 lock.release()
802 except Exception, e:
803 print e
804 raise
805
806 # start worker threads; wait for them all to finish.
807 threads = [threading.Thread(target=worker)
808 for i in range(OPTIONS.worker_threads)]
809 for th in threads:
810 th.start()
811 while threads:
812 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -0700813
814
815# map recovery.fstab's fs_types to mount/format "partition types"
816PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
817 "ext4": "EMMC", "emmc": "EMMC" }
818
819def GetTypeAndDevice(mount_point, info):
820 fstab = info["fstab"]
821 if fstab:
822 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
823 else:
Ying Wanga73b6562011-03-03 21:52:08 -0800824 return None