blob: 4957354af6c037e80a6986e289cf1df66c81a5f9 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Doug Zongkerea5d7a92010-09-12 15:26:16 -070015import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070016import errno
Doug Zongkereef39442009-04-02 12:14:19 -070017import getopt
18import getpass
Doug Zongker05d3dea2009-06-22 11:32:31 -070019import imp
Doug Zongkereef39442009-04-02 12:14:19 -070020import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080021import platform
Doug Zongkereef39442009-04-02 12:14:19 -070022import re
23import shutil
24import subprocess
25import sys
26import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070027import threading
28import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070029import zipfile
Doug Zongkereef39442009-04-02 12:14:19 -070030
Doug Zongker55d93282011-01-25 17:03:34 -080031try:
davidcad0bb92011-03-15 14:21:38 +000032 from hashlib import sha1 as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080033except ImportError:
davidcad0bb92011-03-15 14:21:38 +000034 from sha import sha as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080035
Doug Zongkereef39442009-04-02 12:14:19 -070036# missing in Python 2.4 and before
37if not hasattr(os, "SEEK_SET"):
38 os.SEEK_SET = 0
39
40class Options(object): pass
41OPTIONS = Options()
Doug Zongker602a84e2009-06-18 08:35:12 -070042OPTIONS.search_path = "out/host/linux-x86"
Doug Zongkereef39442009-04-02 12:14:19 -070043OPTIONS.verbose = False
44OPTIONS.tempfiles = []
Doug Zongker05d3dea2009-06-22 11:32:31 -070045OPTIONS.device_specific = None
Doug Zongker8bec09e2009-11-30 15:37:14 -080046OPTIONS.extras = {}
Doug Zongkerc77a9ad2010-09-16 11:28:43 -070047OPTIONS.info_dict = None
Doug Zongkereef39442009-04-02 12:14:19 -070048
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080049
50# Values for "certificate" in apkcerts that mean special things.
51SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
52
53
Doug Zongkereef39442009-04-02 12:14:19 -070054class ExternalError(RuntimeError): pass
55
56
57def Run(args, **kwargs):
58 """Create and return a subprocess.Popen object, printing the command
59 line on the terminal if -v was specified."""
60 if OPTIONS.verbose:
61 print " running: ", " ".join(args)
62 return subprocess.Popen(args, **kwargs)
63
64
Ying Wang7e6d4e42010-12-13 16:25:36 -080065def CloseInheritedPipes():
66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
67 before doing other work."""
68 if platform.system() != "Darwin":
69 return
70 for d in range(3, 1025):
71 try:
72 stat = os.fstat(d)
73 if stat is not None:
74 pipebit = stat[0] & 0x1000
75 if pipebit != 0:
76 os.close(d)
77 except OSError:
78 pass
79
80
Doug Zongker37974732010-09-16 17:44:38 -070081def LoadInfoDict(zip):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070082 """Read and parse the META/misc_info.txt key/value pairs from the
83 input target files and return a dict."""
84
85 d = {}
86 try:
Doug Zongker37974732010-09-16 17:44:38 -070087 for line in zip.read("META/misc_info.txt").split("\n"):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070088 line = line.strip()
89 if not line or line.startswith("#"): continue
90 k, v = line.split("=", 1)
91 d[k] = v
Doug Zongker37974732010-09-16 17:44:38 -070092 except KeyError:
93 # ok if misc_info.txt doesn't exist
94 pass
Doug Zongkerc19a8d52010-07-01 15:30:11 -070095
Doug Zongker37974732010-09-16 17:44:38 -070096 # backwards compatibility: These values used to be in their own
97 # files. Look for them, in case we're processing an old
98 # target_files zip.
99
100 if "mkyaffs2_extra_flags" not in d:
101 try:
102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
103 except KeyError:
104 # ok if flags don't exist
105 pass
106
107 if "recovery_api_version" not in d:
108 try:
109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
110 except KeyError:
111 raise ValueError("can't find recovery API version in input target-files")
112
113 if "tool_extensions" not in d:
114 try:
115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
116 except KeyError:
117 # ok if extensions don't exist
118 pass
119
120 try:
121 data = zip.read("META/imagesizes.txt")
122 for line in data.split("\n"):
123 if not line: continue
Doug Zongker1684d9c2010-09-17 07:44:38 -0700124 name, value = line.split(" ", 1)
125 if not value: continue
Doug Zongker37974732010-09-16 17:44:38 -0700126 if name == "blocksize":
127 d[name] = value
128 else:
129 d[name + "_size"] = value
130 except KeyError:
131 pass
132
133 def makeint(key):
134 if key in d:
135 d[key] = int(d[key], 0)
136
137 makeint("recovery_api_version")
138 makeint("blocksize")
139 makeint("system_size")
140 makeint("userdata_size")
141 makeint("recovery_size")
142 makeint("boot_size")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700143
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700144 d["fstab"] = LoadRecoveryFSTab(zip)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700145 return d
146
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700147def LoadRecoveryFSTab(zip):
148 class Partition(object):
149 pass
150
151 try:
152 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
153 except KeyError:
Jeff Davidson033fbe22011-10-26 18:08:09 -0700154 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip
155 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700156
157 d = {}
158 for line in data.split("\n"):
159 line = line.strip()
160 if not line or line.startswith("#"): continue
161 pieces = line.split()
162 if not (3 <= len(pieces) <= 4):
163 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
164
165 p = Partition()
166 p.mount_point = pieces[0]
167 p.fs_type = pieces[1]
168 p.device = pieces[2]
Doug Zongker086cbb02011-02-17 15:54:20 -0800169 p.length = 0
170 options = None
171 if len(pieces) >= 4:
172 if pieces[3].startswith("/"):
173 p.device2 = pieces[3]
174 if len(pieces) >= 5:
175 options = pieces[4]
176 else:
177 p.device2 = None
178 options = pieces[3]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700179 else:
180 p.device2 = None
181
Doug Zongker086cbb02011-02-17 15:54:20 -0800182 if options:
183 options = options.split(",")
184 for i in options:
185 if i.startswith("length="):
186 p.length = int(i[7:])
187 else:
188 print "%s: unknown option \"%s\"" % (p.mount_point, i)
189
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700190 d[p.mount_point] = p
191 return d
192
193
Doug Zongker37974732010-09-16 17:44:38 -0700194def DumpInfoDict(d):
195 for k, v in sorted(d.items()):
196 print "%-25s = (%s) %s" % (k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700197
Doug Zongkereef39442009-04-02 12:14:19 -0700198def BuildBootableImage(sourcedir):
199 """Take a kernel, cmdline, and ramdisk directory from the input (in
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700200 'sourcedir'), and turn them into a boot image. Return the image
201 data, or None if sourcedir does not appear to contains files for
202 building the requested image."""
203
204 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
205 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
206 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700207
208 ramdisk_img = tempfile.NamedTemporaryFile()
209 img = tempfile.NamedTemporaryFile()
210
211 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
212 stdout=subprocess.PIPE)
Doug Zongker32da27a2009-05-29 09:35:56 -0700213 p2 = Run(["minigzip"],
214 stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
Doug Zongkereef39442009-04-02 12:14:19 -0700215
216 p2.wait()
217 p1.wait()
218 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
Doug Zongker32da27a2009-05-29 09:35:56 -0700219 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
Doug Zongkereef39442009-04-02 12:14:19 -0700220
Doug Zongker38a649f2009-06-17 09:07:09 -0700221 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
222
Doug Zongker171f1cd2009-06-15 22:36:37 -0700223 fn = os.path.join(sourcedir, "cmdline")
224 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700225 cmd.append("--cmdline")
226 cmd.append(open(fn).read().rstrip("\n"))
227
228 fn = os.path.join(sourcedir, "base")
229 if os.access(fn, os.F_OK):
230 cmd.append("--base")
231 cmd.append(open(fn).read().rstrip("\n"))
232
Ying Wang4de6b5b2010-08-25 14:29:34 -0700233 fn = os.path.join(sourcedir, "pagesize")
234 if os.access(fn, os.F_OK):
235 cmd.append("--pagesize")
236 cmd.append(open(fn).read().rstrip("\n"))
237
Doug Zongker38a649f2009-06-17 09:07:09 -0700238 cmd.extend(["--ramdisk", ramdisk_img.name,
239 "--output", img.name])
240
241 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700242 p.communicate()
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700243 assert p.returncode == 0, "mkbootimg of %s image failed" % (
244 os.path.basename(sourcedir),)
Doug Zongkereef39442009-04-02 12:14:19 -0700245
246 img.seek(os.SEEK_SET, 0)
247 data = img.read()
248
249 ramdisk_img.close()
250 img.close()
251
252 return data
253
254
Doug Zongker55d93282011-01-25 17:03:34 -0800255def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
256 """Return a File object (with name 'name') with the desired bootable
257 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
258 'prebuilt_name', otherwise construct it from the source files in
259 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700260
Doug Zongker55d93282011-01-25 17:03:34 -0800261 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
262 if os.path.exists(prebuilt_path):
263 print "using prebuilt %s..." % (prebuilt_name,)
264 return File.FromLocalFile(name, prebuilt_path)
265 else:
266 print "building image from target_files %s..." % (tree_subdir,)
267 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
268
Doug Zongkereef39442009-04-02 12:14:19 -0700269
Doug Zongker75f17362009-12-08 13:46:44 -0800270def UnzipTemp(filename, pattern=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800271 """Unzip the given archive into a temporary directory and return the name.
272
273 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
274 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
275
276 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
277 main file), open for reading.
278 """
Doug Zongkereef39442009-04-02 12:14:19 -0700279
280 tmp = tempfile.mkdtemp(prefix="targetfiles-")
281 OPTIONS.tempfiles.append(tmp)
Doug Zongker55d93282011-01-25 17:03:34 -0800282
283 def unzip_to_dir(filename, dirname):
284 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
285 if pattern is not None:
286 cmd.append(pattern)
287 p = Run(cmd, stdout=subprocess.PIPE)
288 p.communicate()
289 if p.returncode != 0:
290 raise ExternalError("failed to unzip input target-files \"%s\"" %
291 (filename,))
292
293 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
294 if m:
295 unzip_to_dir(m.group(1), tmp)
296 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
297 filename = m.group(1)
298 else:
299 unzip_to_dir(filename, tmp)
300
301 return tmp, zipfile.ZipFile(filename, "r")
Doug Zongkereef39442009-04-02 12:14:19 -0700302
303
304def GetKeyPasswords(keylist):
305 """Given a list of keys, prompt the user to enter passwords for
306 those which require them. Return a {key: password} dict. password
307 will be None if the key has no password."""
308
Doug Zongker8ce7c252009-05-22 13:34:54 -0700309 no_passwords = []
310 need_passwords = []
Doug Zongkereef39442009-04-02 12:14:19 -0700311 devnull = open("/dev/null", "w+b")
312 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800313 # We don't need a password for things that aren't really keys.
314 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700315 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700316 continue
317
Doug Zongker602a84e2009-06-18 08:35:12 -0700318 p = Run(["openssl", "pkcs8", "-in", k+".pk8",
319 "-inform", "DER", "-nocrypt"],
320 stdin=devnull.fileno(),
321 stdout=devnull.fileno(),
322 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700323 p.communicate()
324 if p.returncode == 0:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700325 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700326 else:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700327 need_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700328 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700329
330 key_passwords = PasswordManager().GetPasswords(need_passwords)
331 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700332 return key_passwords
333
334
Doug Zongker951495f2009-08-14 12:44:19 -0700335def SignFile(input_name, output_name, key, password, align=None,
336 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700337 """Sign the input_name zip/jar/apk, producing output_name. Use the
338 given key and password (the latter may be None if the key does not
339 have a password.
340
341 If align is an integer > 1, zipalign is run to align stored files in
342 the output zip on 'align'-byte boundaries.
Doug Zongker951495f2009-08-14 12:44:19 -0700343
344 If whole_file is true, use the "-w" option to SignApk to embed a
345 signature that covers the whole file in the archive comment of the
346 zip file.
Doug Zongkereef39442009-04-02 12:14:19 -0700347 """
Doug Zongker951495f2009-08-14 12:44:19 -0700348
Doug Zongkereef39442009-04-02 12:14:19 -0700349 if align == 0 or align == 1:
350 align = None
351
352 if align:
353 temp = tempfile.NamedTemporaryFile()
354 sign_name = temp.name
355 else:
356 sign_name = output_name
357
Doug Zongkerca855e92011-02-23 09:25:01 -0800358 cmd = ["java", "-Xmx2048m", "-jar",
Doug Zongker951495f2009-08-14 12:44:19 -0700359 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
360 if whole_file:
361 cmd.append("-w")
362 cmd.extend([key + ".x509.pem", key + ".pk8",
363 input_name, sign_name])
364
365 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700366 if password is not None:
367 password += "\n"
368 p.communicate(password)
369 if p.returncode != 0:
370 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
371
372 if align:
Doug Zongker602a84e2009-06-18 08:35:12 -0700373 p = Run(["zipalign", "-f", str(align), sign_name, output_name])
Doug Zongkereef39442009-04-02 12:14:19 -0700374 p.communicate()
375 if p.returncode != 0:
376 raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
377 temp.close()
378
379
Doug Zongker37974732010-09-16 17:44:38 -0700380def CheckSize(data, target, info_dict):
Doug Zongkereef39442009-04-02 12:14:19 -0700381 """Check the data string passed against the max size limit, if
382 any, for the given target. Raise exception if the data is too big.
383 Print a warning if the data is nearing the maximum size."""
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700384
Doug Zongker1684d9c2010-09-17 07:44:38 -0700385 if target.endswith(".img"): target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700386 mount_point = "/" + target
387
388 if info_dict["fstab"]:
389 if mount_point == "/userdata": mount_point = "/data"
390 p = info_dict["fstab"][mount_point]
391 fs_type = p.fs_type
392 limit = info_dict.get(p.device + "_size", None)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700393 if not fs_type or not limit: return
Doug Zongkereef39442009-04-02 12:14:19 -0700394
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700395 if fs_type == "yaffs2":
396 # image size should be increased by 1/64th to account for the
397 # spare area (64 bytes per 2k page)
398 limit = limit / 2048 * (2048+64)
Doug Zongker486de122010-09-16 14:01:56 -0700399 size = len(data)
400 pct = float(size) * 100.0 / limit
401 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
402 if pct >= 99.0:
403 raise ExternalError(msg)
404 elif pct >= 95.0:
405 print
406 print " WARNING: ", msg
407 print
408 elif OPTIONS.verbose:
409 print " ", msg
Doug Zongkereef39442009-04-02 12:14:19 -0700410
411
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800412def ReadApkCerts(tf_zip):
413 """Given a target_files ZipFile, parse the META/apkcerts.txt file
414 and return a {package: cert} dict."""
415 certmap = {}
416 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
417 line = line.strip()
418 if not line: continue
419 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
420 r'private_key="(.*)"$', line)
421 if m:
422 name, cert, privkey = m.groups()
423 if cert in SPECIAL_CERT_STRINGS and not privkey:
424 certmap[name] = cert
425 elif (cert.endswith(".x509.pem") and
426 privkey.endswith(".pk8") and
427 cert[:-9] == privkey[:-4]):
428 certmap[name] = cert[:-9]
429 else:
430 raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
431 return certmap
432
433
Doug Zongkereef39442009-04-02 12:14:19 -0700434COMMON_DOCSTRING = """
435 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700436 Prepend <dir>/bin to the list of places to search for binaries
437 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700438
Doug Zongker05d3dea2009-06-22 11:32:31 -0700439 -s (--device_specific) <file>
440 Path to the python module containing device-specific
441 releasetools code.
442
Doug Zongker8bec09e2009-11-30 15:37:14 -0800443 -x (--extra) <key=value>
444 Add a key/value pair to the 'extras' dict, which device-specific
445 extension code may look at.
446
Doug Zongkereef39442009-04-02 12:14:19 -0700447 -v (--verbose)
448 Show command lines being executed.
449
450 -h (--help)
451 Display this usage message and exit.
452"""
453
454def Usage(docstring):
455 print docstring.rstrip("\n")
456 print COMMON_DOCSTRING
457
458
459def ParseOptions(argv,
460 docstring,
461 extra_opts="", extra_long_opts=(),
462 extra_option_handler=None):
463 """Parse the options in argv and return any arguments that aren't
464 flags. docstring is the calling module's docstring, to be displayed
465 for errors and -h. extra_opts and extra_long_opts are for flags
466 defined by the caller, which are processed by passing them to
467 extra_option_handler."""
468
469 try:
470 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800471 argv, "hvp:s:x:" + extra_opts,
472 ["help", "verbose", "path=", "device_specific=", "extra="] +
Doug Zongker05d3dea2009-06-22 11:32:31 -0700473 list(extra_long_opts))
Doug Zongkereef39442009-04-02 12:14:19 -0700474 except getopt.GetoptError, err:
475 Usage(docstring)
476 print "**", str(err), "**"
477 sys.exit(2)
478
479 path_specified = False
480
481 for o, a in opts:
482 if o in ("-h", "--help"):
483 Usage(docstring)
484 sys.exit()
485 elif o in ("-v", "--verbose"):
486 OPTIONS.verbose = True
487 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -0700488 OPTIONS.search_path = a
Doug Zongker05d3dea2009-06-22 11:32:31 -0700489 elif o in ("-s", "--device_specific"):
490 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -0800491 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -0800492 key, value = a.split("=", 1)
493 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -0700494 else:
495 if extra_option_handler is None or not extra_option_handler(o, a):
496 assert False, "unknown option \"%s\"" % (o,)
497
Doug Zongker602a84e2009-06-18 08:35:12 -0700498 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
499 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -0700500
501 return args
502
503
504def Cleanup():
505 for i in OPTIONS.tempfiles:
506 if os.path.isdir(i):
507 shutil.rmtree(i)
508 else:
509 os.remove(i)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700510
511
512class PasswordManager(object):
513 def __init__(self):
514 self.editor = os.getenv("EDITOR", None)
515 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
516
517 def GetPasswords(self, items):
518 """Get passwords corresponding to each string in 'items',
519 returning a dict. (The dict may have keys in addition to the
520 values in 'items'.)
521
522 Uses the passwords in $ANDROID_PW_FILE if available, letting the
523 user edit that file to add more needed passwords. If no editor is
524 available, or $ANDROID_PW_FILE isn't define, prompts the user
525 interactively in the ordinary way.
526 """
527
528 current = self.ReadFile()
529
530 first = True
531 while True:
532 missing = []
533 for i in items:
534 if i not in current or not current[i]:
535 missing.append(i)
536 # Are all the passwords already in the file?
537 if not missing: return current
538
539 for i in missing:
540 current[i] = ""
541
542 if not first:
543 print "key file %s still missing some passwords." % (self.pwfile,)
544 answer = raw_input("try to edit again? [y]> ").strip()
545 if answer and answer[0] not in 'yY':
546 raise RuntimeError("key passwords unavailable")
547 first = False
548
549 current = self.UpdateAndReadFile(current)
550
551 def PromptResult(self, current):
552 """Prompt the user to enter a value (password) for each key in
553 'current' whose value is fales. Returns a new dict with all the
554 values.
555 """
556 result = {}
557 for k, v in sorted(current.iteritems()):
558 if v:
559 result[k] = v
560 else:
561 while True:
562 result[k] = getpass.getpass("Enter password for %s key> "
563 % (k,)).strip()
564 if result[k]: break
565 return result
566
567 def UpdateAndReadFile(self, current):
568 if not self.editor or not self.pwfile:
569 return self.PromptResult(current)
570
571 f = open(self.pwfile, "w")
572 os.chmod(self.pwfile, 0600)
573 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
574 f.write("# (Additional spaces are harmless.)\n\n")
575
576 first_line = None
577 sorted = [(not v, k, v) for (k, v) in current.iteritems()]
578 sorted.sort()
579 for i, (_, k, v) in enumerate(sorted):
580 f.write("[[[ %s ]]] %s\n" % (v, k))
581 if not v and first_line is None:
582 # position cursor on first line with no password.
583 first_line = i + 4
584 f.close()
585
586 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
587 _, _ = p.communicate()
588
589 return self.ReadFile()
590
591 def ReadFile(self):
592 result = {}
593 if self.pwfile is None: return result
594 try:
595 f = open(self.pwfile, "r")
596 for line in f:
597 line = line.strip()
598 if not line or line[0] == '#': continue
599 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
600 if not m:
601 print "failed to parse password file: ", line
602 else:
603 result[m.group(2)] = m.group(1)
604 f.close()
605 except IOError, e:
606 if e.errno != errno.ENOENT:
607 print "error reading password file: ", str(e)
608 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -0700609
610
611def ZipWriteStr(zip, filename, data, perms=0644):
612 # use a fixed timestamp so the output is repeatable.
613 zinfo = zipfile.ZipInfo(filename=filename,
614 date_time=(2009, 1, 1, 0, 0, 0))
615 zinfo.compress_type = zip.compression
616 zinfo.external_attr = perms << 16
617 zip.writestr(zinfo, data)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700618
619
620class DeviceSpecificParams(object):
621 module = None
622 def __init__(self, **kwargs):
623 """Keyword arguments to the constructor become attributes of this
624 object, which is passed to all functions in the device-specific
625 module."""
626 for k, v in kwargs.iteritems():
627 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -0800628 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -0700629
630 if self.module is None:
631 path = OPTIONS.device_specific
Doug Zongkerc18736b2009-09-30 09:20:32 -0700632 if not path: return
Doug Zongker8e2f2b92009-06-24 14:34:57 -0700633 try:
634 if os.path.isdir(path):
635 info = imp.find_module("releasetools", [path])
636 else:
637 d, f = os.path.split(path)
638 b, x = os.path.splitext(f)
639 if x == ".py":
640 f = b
641 info = imp.find_module(f, [d])
642 self.module = imp.load_module("device_specific", *info)
643 except ImportError:
644 print "unable to load device-specific module; assuming none"
Doug Zongker05d3dea2009-06-22 11:32:31 -0700645
646 def _DoCall(self, function_name, *args, **kwargs):
647 """Call the named function in the device-specific module, passing
648 the given args and kwargs. The first argument to the call will be
649 the DeviceSpecific object itself. If there is no module, or the
650 module does not define the function, return the value of the
651 'default' kwarg (which itself defaults to None)."""
652 if self.module is None or not hasattr(self.module, function_name):
653 return kwargs.get("default", None)
654 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
655
656 def FullOTA_Assertions(self):
657 """Called after emitting the block of assertions at the top of a
658 full OTA package. Implementations can add whatever additional
659 assertions they like."""
660 return self._DoCall("FullOTA_Assertions")
661
662 def FullOTA_InstallEnd(self):
663 """Called at the end of full OTA installation; typically this is
664 used to install the image for the device's baseband processor."""
665 return self._DoCall("FullOTA_InstallEnd")
666
667 def IncrementalOTA_Assertions(self):
668 """Called after emitting the block of assertions at the top of an
669 incremental OTA package. Implementations can add whatever
670 additional assertions they like."""
671 return self._DoCall("IncrementalOTA_Assertions")
672
673 def IncrementalOTA_VerifyEnd(self):
674 """Called at the end of the verification phase of incremental OTA
675 installation; additional checks can be placed here to abort the
676 script before any changes are made."""
677 return self._DoCall("IncrementalOTA_VerifyEnd")
678
679 def IncrementalOTA_InstallEnd(self):
680 """Called at the end of incremental OTA installation; typically
681 this is used to install the image for the device's baseband
682 processor."""
683 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700684
685class File(object):
686 def __init__(self, name, data):
687 self.name = name
688 self.data = data
689 self.size = len(data)
Doug Zongker55d93282011-01-25 17:03:34 -0800690 self.sha1 = sha1(data).hexdigest()
691
692 @classmethod
693 def FromLocalFile(cls, name, diskname):
694 f = open(diskname, "rb")
695 data = f.read()
696 f.close()
697 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700698
699 def WriteToTemp(self):
700 t = tempfile.NamedTemporaryFile()
701 t.write(self.data)
702 t.flush()
703 return t
704
705 def AddToZip(self, z):
706 ZipWriteStr(z, self.name, self.data)
707
708DIFF_PROGRAM_BY_EXT = {
709 ".gz" : "imgdiff",
710 ".zip" : ["imgdiff", "-z"],
711 ".jar" : ["imgdiff", "-z"],
712 ".apk" : ["imgdiff", "-z"],
713 ".img" : "imgdiff",
714 }
715
716class Difference(object):
717 def __init__(self, tf, sf):
718 self.tf = tf
719 self.sf = sf
720 self.patch = None
721
722 def ComputePatch(self):
723 """Compute the patch (as a string of data) needed to turn sf into
724 tf. Returns the same tuple as GetPatch()."""
725
726 tf = self.tf
727 sf = self.sf
728
729 ext = os.path.splitext(tf.name)[1]
730 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
731
732 ttemp = tf.WriteToTemp()
733 stemp = sf.WriteToTemp()
734
735 ext = os.path.splitext(tf.name)[1]
736
737 try:
738 ptemp = tempfile.NamedTemporaryFile()
739 if isinstance(diff_program, list):
740 cmd = copy.copy(diff_program)
741 else:
742 cmd = [diff_program]
743 cmd.append(stemp.name)
744 cmd.append(ttemp.name)
745 cmd.append(ptemp.name)
746 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
747 _, err = p.communicate()
748 if err or p.returncode != 0:
749 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
750 return None
751 diff = ptemp.read()
752 finally:
753 ptemp.close()
754 stemp.close()
755 ttemp.close()
756
757 self.patch = diff
758 return self.tf, self.sf, self.patch
759
760
761 def GetPatch(self):
762 """Return a tuple (target_file, source_file, patch_data).
763 patch_data may be None if ComputePatch hasn't been called, or if
764 computing the patch failed."""
765 return self.tf, self.sf, self.patch
766
767
768def ComputeDifferences(diffs):
769 """Call ComputePatch on all the Difference objects in 'diffs'."""
770 print len(diffs), "diffs to compute"
771
772 # Do the largest files first, to try and reduce the long-pole effect.
773 by_size = [(i.tf.size, i) for i in diffs]
774 by_size.sort(reverse=True)
775 by_size = [i[1] for i in by_size]
776
777 lock = threading.Lock()
778 diff_iter = iter(by_size) # accessed under lock
779
780 def worker():
781 try:
782 lock.acquire()
783 for d in diff_iter:
784 lock.release()
785 start = time.time()
786 d.ComputePatch()
787 dur = time.time() - start
788 lock.acquire()
789
790 tf, sf, patch = d.GetPatch()
791 if sf.name == tf.name:
792 name = tf.name
793 else:
794 name = "%s (%s)" % (tf.name, sf.name)
795 if patch is None:
796 print "patching failed! %s" % (name,)
797 else:
798 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
799 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
800 lock.release()
801 except Exception, e:
802 print e
803 raise
804
805 # start worker threads; wait for them all to finish.
806 threads = [threading.Thread(target=worker)
807 for i in range(OPTIONS.worker_threads)]
808 for th in threads:
809 th.start()
810 while threads:
811 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -0700812
813
814# map recovery.fstab's fs_types to mount/format "partition types"
815PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
816 "ext4": "EMMC", "emmc": "EMMC" }
817
818def GetTypeAndDevice(mount_point, info):
819 fstab = info["fstab"]
820 if fstab:
821 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
822 else:
Ying Wanga73b6562011-03-03 21:52:08 -0800823 return None