Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright (C) 2008 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """ |
| 18 | Given a target-files zipfile, produces an OTA package that installs |
| 19 | that build. An incremental OTA is produced if -i is given, otherwise |
| 20 | a full OTA is produced. |
| 21 | |
| 22 | Usage: ota_from_target_files [flags] input_target_files output_ota_package |
| 23 | |
| 24 | -b (--board_config) <file> |
| 25 | Specifies a BoardConfig.mk file containing image max sizes |
| 26 | against which the generated image files are checked. |
| 27 | |
| 28 | -k (--package_key) <key> |
| 29 | Key to use to sign the package (default is |
| 30 | "build/target/product/security/testkey"). |
| 31 | |
| 32 | -i (--incremental_from) <file> |
| 33 | Generate an incremental OTA using the given target-files zip as |
| 34 | the starting build. |
| 35 | |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 36 | -w (--wipe_user_data) |
| 37 | Generate an OTA package that will wipe the user data partition |
| 38 | when installed. |
| 39 | |
Doug Zongker | 962069c | 2009-04-23 11:41:58 -0700 | [diff] [blame] | 40 | -n (--no_prereq) |
| 41 | Omit the timestamp prereq check normally included at the top of |
| 42 | the build scripts (used for developer OTA packages which |
| 43 | legitimately need to go back and forth). |
| 44 | |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 45 | -e (--extra_script) <file> |
| 46 | Insert the contents of file at the end of the update script. |
| 47 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 48 | """ |
| 49 | |
| 50 | import sys |
| 51 | |
| 52 | if sys.hexversion < 0x02040000: |
| 53 | print >> sys.stderr, "Python 2.4 or newer is required." |
| 54 | sys.exit(1) |
| 55 | |
| 56 | import copy |
| 57 | import os |
| 58 | import re |
| 59 | import sha |
| 60 | import subprocess |
| 61 | import tempfile |
| 62 | import time |
| 63 | import zipfile |
| 64 | |
| 65 | import common |
| 66 | |
| 67 | OPTIONS = common.OPTIONS |
| 68 | OPTIONS.package_key = "build/target/product/security/testkey" |
| 69 | OPTIONS.incremental_source = None |
| 70 | OPTIONS.require_verbatim = set() |
| 71 | OPTIONS.prohibit_verbatim = set(("system/build.prop",)) |
| 72 | OPTIONS.patch_threshold = 0.95 |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 73 | OPTIONS.wipe_user_data = False |
Doug Zongker | 962069c | 2009-04-23 11:41:58 -0700 | [diff] [blame] | 74 | OPTIONS.omit_prereq = False |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 75 | OPTIONS.extra_script = None |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 76 | |
| 77 | def MostPopularKey(d, default): |
| 78 | """Given a dict, return the key corresponding to the largest |
| 79 | value. Returns 'default' if the dict is empty.""" |
| 80 | x = [(v, k) for (k, v) in d.iteritems()] |
| 81 | if not x: return default |
| 82 | x.sort() |
| 83 | return x[-1][1] |
| 84 | |
| 85 | |
| 86 | def IsSymlink(info): |
| 87 | """Return true if the zipfile.ZipInfo object passed in represents a |
| 88 | symlink.""" |
| 89 | return (info.external_attr >> 16) == 0120777 |
| 90 | |
| 91 | |
| 92 | |
| 93 | class Item: |
| 94 | """Items represent the metadata (user, group, mode) of files and |
| 95 | directories in the system image.""" |
| 96 | ITEMS = {} |
| 97 | def __init__(self, name, dir=False): |
| 98 | self.name = name |
| 99 | self.uid = None |
| 100 | self.gid = None |
| 101 | self.mode = None |
| 102 | self.dir = dir |
| 103 | |
| 104 | if name: |
| 105 | self.parent = Item.Get(os.path.dirname(name), dir=True) |
| 106 | self.parent.children.append(self) |
| 107 | else: |
| 108 | self.parent = None |
| 109 | if dir: |
| 110 | self.children = [] |
| 111 | |
| 112 | def Dump(self, indent=0): |
| 113 | if self.uid is not None: |
| 114 | print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode) |
| 115 | else: |
| 116 | print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode) |
| 117 | if self.dir: |
| 118 | print "%s%s" % (" "*indent, self.descendants) |
| 119 | print "%s%s" % (" "*indent, self.best_subtree) |
| 120 | for i in self.children: |
| 121 | i.Dump(indent=indent+1) |
| 122 | |
| 123 | @classmethod |
| 124 | def Get(cls, name, dir=False): |
| 125 | if name not in cls.ITEMS: |
| 126 | cls.ITEMS[name] = Item(name, dir=dir) |
| 127 | return cls.ITEMS[name] |
| 128 | |
| 129 | @classmethod |
| 130 | def GetMetadata(cls): |
| 131 | """Run the external 'fs_config' program to determine the desired |
| 132 | uid, gid, and mode for every Item object.""" |
| 133 | p = common.Run(["fs_config"], stdin=subprocess.PIPE, |
| 134 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 135 | suffix = { False: "", True: "/" } |
| 136 | input = "".join(["%s%s\n" % (i.name, suffix[i.dir]) |
| 137 | for i in cls.ITEMS.itervalues() if i.name]) |
| 138 | output, error = p.communicate(input) |
| 139 | assert not error |
| 140 | |
| 141 | for line in output.split("\n"): |
| 142 | if not line: continue |
| 143 | name, uid, gid, mode = line.split() |
| 144 | i = cls.ITEMS[name] |
| 145 | i.uid = int(uid) |
| 146 | i.gid = int(gid) |
| 147 | i.mode = int(mode, 8) |
| 148 | if i.dir: |
| 149 | i.children.sort(key=lambda i: i.name) |
| 150 | |
| 151 | def CountChildMetadata(self): |
| 152 | """Count up the (uid, gid, mode) tuples for all children and |
| 153 | determine the best strategy for using set_perm_recursive and |
| 154 | set_perm to correctly chown/chmod all the files to their desired |
| 155 | values. Recursively calls itself for all descendants. |
| 156 | |
| 157 | Returns a dict of {(uid, gid, dmode, fmode): count} counting up |
| 158 | all descendants of this node. (dmode or fmode may be None.) Also |
| 159 | sets the best_subtree of each directory Item to the (uid, gid, |
| 160 | dmode, fmode) tuple that will match the most descendants of that |
| 161 | Item. |
| 162 | """ |
| 163 | |
| 164 | assert self.dir |
| 165 | d = self.descendants = {(self.uid, self.gid, self.mode, None): 1} |
| 166 | for i in self.children: |
| 167 | if i.dir: |
| 168 | for k, v in i.CountChildMetadata().iteritems(): |
| 169 | d[k] = d.get(k, 0) + v |
| 170 | else: |
| 171 | k = (i.uid, i.gid, None, i.mode) |
| 172 | d[k] = d.get(k, 0) + 1 |
| 173 | |
| 174 | # Find the (uid, gid, dmode, fmode) tuple that matches the most |
| 175 | # descendants. |
| 176 | |
| 177 | # First, find the (uid, gid) pair that matches the most |
| 178 | # descendants. |
| 179 | ug = {} |
| 180 | for (uid, gid, _, _), count in d.iteritems(): |
| 181 | ug[(uid, gid)] = ug.get((uid, gid), 0) + count |
| 182 | ug = MostPopularKey(ug, (0, 0)) |
| 183 | |
| 184 | # Now find the dmode and fmode that match the most descendants |
| 185 | # with that (uid, gid), and choose those. |
| 186 | best_dmode = (0, 0755) |
| 187 | best_fmode = (0, 0644) |
| 188 | for k, count in d.iteritems(): |
| 189 | if k[:2] != ug: continue |
| 190 | if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2]) |
| 191 | if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3]) |
| 192 | self.best_subtree = ug + (best_dmode[1], best_fmode[1]) |
| 193 | |
| 194 | return d |
| 195 | |
| 196 | def SetPermissions(self, script, renamer=lambda x: x): |
| 197 | """Append set_perm/set_perm_recursive commands to 'script' to |
| 198 | set all permissions, users, and groups for the tree of files |
| 199 | rooted at 'self'. 'renamer' turns the filenames stored in the |
| 200 | tree of Items into the strings used in the script.""" |
| 201 | |
| 202 | self.CountChildMetadata() |
| 203 | |
| 204 | def recurse(item, current): |
| 205 | # current is the (uid, gid, dmode, fmode) tuple that the current |
| 206 | # item (and all its children) have already been set to. We only |
| 207 | # need to issue set_perm/set_perm_recursive commands if we're |
| 208 | # supposed to be something different. |
| 209 | if item.dir: |
| 210 | if current != item.best_subtree: |
| 211 | script.append("set_perm_recursive %d %d 0%o 0%o %s" % |
| 212 | (item.best_subtree + (renamer(item.name),))) |
| 213 | current = item.best_subtree |
| 214 | |
| 215 | if item.uid != current[0] or item.gid != current[1] or \ |
| 216 | item.mode != current[2]: |
| 217 | script.append("set_perm %d %d 0%o %s" % |
| 218 | (item.uid, item.gid, item.mode, renamer(item.name))) |
| 219 | |
| 220 | for i in item.children: |
| 221 | recurse(i, current) |
| 222 | else: |
| 223 | if item.uid != current[0] or item.gid != current[1] or \ |
| 224 | item.mode != current[3]: |
| 225 | script.append("set_perm %d %d 0%o %s" % |
| 226 | (item.uid, item.gid, item.mode, renamer(item.name))) |
| 227 | |
| 228 | recurse(self, (-1, -1, -1, -1)) |
| 229 | |
| 230 | |
| 231 | def CopySystemFiles(input_zip, output_zip=None, |
| 232 | substitute=None): |
| 233 | """Copies files underneath system/ in the input zip to the output |
| 234 | zip. Populates the Item class with their metadata, and returns a |
| 235 | list of symlinks. output_zip may be None, in which case the copy is |
| 236 | skipped (but the other side effects still happen). substitute is an |
| 237 | optional dict of {output filename: contents} to be output instead of |
| 238 | certain input files. |
| 239 | """ |
| 240 | |
| 241 | symlinks = [] |
| 242 | |
| 243 | for info in input_zip.infolist(): |
| 244 | if info.filename.startswith("SYSTEM/"): |
| 245 | basefilename = info.filename[7:] |
| 246 | if IsSymlink(info): |
| 247 | symlinks.append((input_zip.read(info.filename), |
| 248 | "SYSTEM:" + basefilename)) |
| 249 | else: |
| 250 | info2 = copy.copy(info) |
| 251 | fn = info2.filename = "system/" + basefilename |
| 252 | if substitute and fn in substitute and substitute[fn] is None: |
| 253 | continue |
| 254 | if output_zip is not None: |
| 255 | if substitute and fn in substitute: |
| 256 | data = substitute[fn] |
| 257 | else: |
| 258 | data = input_zip.read(info.filename) |
| 259 | output_zip.writestr(info2, data) |
| 260 | if fn.endswith("/"): |
| 261 | Item.Get(fn[:-1], dir=True) |
| 262 | else: |
| 263 | Item.Get(fn, dir=False) |
| 264 | |
| 265 | symlinks.sort() |
| 266 | return symlinks |
| 267 | |
| 268 | |
| 269 | def AddScript(script, output_zip): |
| 270 | now = time.localtime() |
| 271 | i = zipfile.ZipInfo("META-INF/com/google/android/update-script", |
| 272 | (now.tm_year, now.tm_mon, now.tm_mday, |
| 273 | now.tm_hour, now.tm_min, now.tm_sec)) |
| 274 | output_zip.writestr(i, "\n".join(script) + "\n") |
| 275 | |
| 276 | |
| 277 | def SignOutput(temp_zip_name, output_zip_name): |
| 278 | key_passwords = common.GetKeyPasswords([OPTIONS.package_key]) |
| 279 | pw = key_passwords[OPTIONS.package_key] |
| 280 | |
| 281 | common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw) |
| 282 | |
| 283 | |
| 284 | def SubstituteRoot(s): |
| 285 | if s == "system": return "SYSTEM:" |
| 286 | assert s.startswith("system/") |
| 287 | return "SYSTEM:" + s[7:] |
| 288 | |
| 289 | def FixPermissions(script): |
| 290 | Item.GetMetadata() |
| 291 | root = Item.Get("system") |
| 292 | root.SetPermissions(script, renamer=SubstituteRoot) |
| 293 | |
| 294 | def DeleteFiles(script, to_delete): |
| 295 | line = [] |
| 296 | t = 0 |
| 297 | for i in to_delete: |
| 298 | line.append(i) |
| 299 | t += len(i) + 1 |
| 300 | if t > 80: |
| 301 | script.append("delete " + " ".join(line)) |
| 302 | line = [] |
| 303 | t = 0 |
| 304 | if line: |
| 305 | script.append("delete " + " ".join(line)) |
| 306 | |
| 307 | def AppendAssertions(script, input_zip): |
| 308 | script.append('assert compatible_with("0.2") == "true"') |
| 309 | |
| 310 | device = GetBuildProp("ro.product.device", input_zip) |
| 311 | script.append('assert getprop("ro.product.device") == "%s" || ' |
| 312 | 'getprop("ro.build.product") == "%s"' % (device, device)) |
| 313 | |
| 314 | info = input_zip.read("OTA/android-info.txt") |
| 315 | m = re.search(r"require\s+version-bootloader\s*=\s*(\S+)", info) |
| 316 | if not m: |
| 317 | raise ExternalError("failed to find required bootloaders in " |
| 318 | "android-info.txt") |
| 319 | bootloaders = m.group(1).split("|") |
| 320 | script.append("assert " + |
| 321 | " || ".join(['getprop("ro.bootloader") == "%s"' % (b,) |
| 322 | for b in bootloaders])) |
| 323 | |
| 324 | |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 325 | def IncludeBinary(name, input_zip, output_zip, input_path=None): |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 326 | try: |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 327 | if input_path is not None: |
| 328 | data = open(input_path).read() |
| 329 | else: |
| 330 | data = input_zip.read(os.path.join("OTA/bin", name)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 331 | output_zip.writestr(name, data) |
| 332 | except IOError: |
| 333 | raise ExternalError('unable to include device binary "%s"' % (name,)) |
| 334 | |
| 335 | |
| 336 | def WriteFullOTAPackage(input_zip, output_zip): |
| 337 | script = [] |
| 338 | |
Doug Zongker | 962069c | 2009-04-23 11:41:58 -0700 | [diff] [blame] | 339 | if not OPTIONS.omit_prereq: |
| 340 | ts = GetBuildProp("ro.build.date.utc", input_zip) |
| 341 | script.append("run_program PACKAGE:check_prereq %s" % (ts,)) |
| 342 | IncludeBinary("check_prereq", input_zip, output_zip) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 343 | |
| 344 | AppendAssertions(script, input_zip) |
| 345 | |
| 346 | script.append("format BOOT:") |
| 347 | script.append("show_progress 0.1 0") |
| 348 | |
| 349 | output_zip.writestr("radio.img", input_zip.read("RADIO/image")) |
| 350 | script.append("write_radio_image PACKAGE:radio.img") |
| 351 | script.append("show_progress 0.5 0") |
| 352 | |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 353 | if OPTIONS.wipe_user_data: |
| 354 | script.append("format DATA:") |
| 355 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 356 | script.append("format SYSTEM:") |
| 357 | script.append("copy_dir PACKAGE:system SYSTEM:") |
| 358 | |
| 359 | symlinks = CopySystemFiles(input_zip, output_zip) |
| 360 | script.extend(["symlink %s %s" % s for s in symlinks]) |
| 361 | |
| 362 | common.BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), |
| 363 | "system/recovery.img", output_zip) |
| 364 | Item.Get("system/recovery.img", dir=False) |
| 365 | |
| 366 | FixPermissions(script) |
| 367 | |
| 368 | common.AddBoot(output_zip) |
| 369 | script.append("show_progress 0.2 0") |
| 370 | script.append("write_raw_image PACKAGE:boot.img BOOT:") |
| 371 | script.append("show_progress 0.2 10") |
| 372 | |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 373 | if OPTIONS.extra_script is not None: |
| 374 | script.append(OPTIONS.extra_script) |
| 375 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 376 | AddScript(script, output_zip) |
| 377 | |
| 378 | |
| 379 | class File(object): |
| 380 | def __init__(self, name, data): |
| 381 | self.name = name |
| 382 | self.data = data |
| 383 | self.size = len(data) |
| 384 | self.sha1 = sha.sha(data).hexdigest() |
| 385 | |
| 386 | def WriteToTemp(self): |
| 387 | t = tempfile.NamedTemporaryFile() |
| 388 | t.write(self.data) |
| 389 | t.flush() |
| 390 | return t |
| 391 | |
| 392 | def AddToZip(self, z): |
| 393 | z.writestr(self.name, self.data) |
| 394 | |
| 395 | |
| 396 | def LoadSystemFiles(z): |
| 397 | """Load all the files from SYSTEM/... in a given target-files |
| 398 | ZipFile, and return a dict of {filename: File object}.""" |
| 399 | out = {} |
| 400 | for info in z.infolist(): |
| 401 | if info.filename.startswith("SYSTEM/") and not IsSymlink(info): |
| 402 | fn = "system/" + info.filename[7:] |
| 403 | data = z.read(info.filename) |
| 404 | out[fn] = File(fn, data) |
| 405 | return out |
| 406 | |
| 407 | |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 408 | def Difference(tf, sf, diff_program): |
| 409 | """Return the patch (as a string of data) needed to turn sf into tf. |
| 410 | diff_program is the name of an external program (or list, if |
| 411 | additional arguments are desired) to run to generate the diff. |
| 412 | """ |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 413 | |
| 414 | ttemp = tf.WriteToTemp() |
| 415 | stemp = sf.WriteToTemp() |
| 416 | |
| 417 | ext = os.path.splitext(tf.name)[1] |
| 418 | |
| 419 | try: |
| 420 | ptemp = tempfile.NamedTemporaryFile() |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 421 | if isinstance(diff_program, list): |
| 422 | cmd = copy.copy(diff_program) |
| 423 | else: |
| 424 | cmd = [diff_program] |
| 425 | cmd.append(stemp.name) |
| 426 | cmd.append(ttemp.name) |
| 427 | cmd.append(ptemp.name) |
| 428 | p = common.Run(cmd) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 429 | _, err = p.communicate() |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 430 | if err or p.returncode != 0: |
| 431 | print "WARNING: failure running %s:\n%s\n" % (diff_program, err) |
| 432 | return None |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 433 | diff = ptemp.read() |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 434 | finally: |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 435 | ptemp.close() |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 436 | stemp.close() |
| 437 | ttemp.close() |
| 438 | |
| 439 | return diff |
| 440 | |
| 441 | |
| 442 | def GetBuildProp(property, z): |
| 443 | """Return the fingerprint of the build of a given target-files |
| 444 | ZipFile object.""" |
| 445 | bp = z.read("SYSTEM/build.prop") |
| 446 | if not property: |
| 447 | return bp |
| 448 | m = re.search(re.escape(property) + r"=(.*)\n", bp) |
| 449 | if not m: |
| 450 | raise ExternalException("couldn't find %s in build.prop" % (property,)) |
| 451 | return m.group(1).strip() |
| 452 | |
| 453 | |
| 454 | def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): |
| 455 | script = [] |
| 456 | |
| 457 | print "Loading target..." |
| 458 | target_data = LoadSystemFiles(target_zip) |
| 459 | print "Loading source..." |
| 460 | source_data = LoadSystemFiles(source_zip) |
| 461 | |
| 462 | verbatim_targets = [] |
| 463 | patch_list = [] |
| 464 | largest_source_size = 0 |
| 465 | for fn in sorted(target_data.keys()): |
| 466 | tf = target_data[fn] |
| 467 | sf = source_data.get(fn, None) |
| 468 | |
| 469 | if sf is None or fn in OPTIONS.require_verbatim: |
| 470 | # This file should be included verbatim |
| 471 | if fn in OPTIONS.prohibit_verbatim: |
| 472 | raise ExternalError("\"%s\" must be sent verbatim" % (fn,)) |
| 473 | print "send", fn, "verbatim" |
| 474 | tf.AddToZip(output_zip) |
| 475 | verbatim_targets.append((fn, tf.size)) |
| 476 | elif tf.sha1 != sf.sha1: |
| 477 | # File is different; consider sending as a patch |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 478 | diff_method = "bsdiff" |
| 479 | if tf.name.endswith(".gz"): |
| 480 | diff_method = "imgdiff" |
| 481 | d = Difference(tf, sf, diff_method) |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 482 | if d is not None: |
| 483 | print fn, tf.size, len(d), (float(len(d)) / tf.size) |
| 484 | if d is None or len(d) > tf.size * OPTIONS.patch_threshold: |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 485 | # patch is almost as big as the file; don't bother patching |
| 486 | tf.AddToZip(output_zip) |
| 487 | verbatim_targets.append((fn, tf.size)) |
| 488 | else: |
| 489 | output_zip.writestr("patch/" + fn + ".p", d) |
| 490 | patch_list.append((fn, tf, sf, tf.size)) |
| 491 | largest_source_size = max(largest_source_size, sf.size) |
| 492 | else: |
| 493 | # Target file identical to source. |
| 494 | pass |
| 495 | |
| 496 | total_verbatim_size = sum([i[1] for i in verbatim_targets]) |
| 497 | total_patched_size = sum([i[3] for i in patch_list]) |
| 498 | |
| 499 | source_fp = GetBuildProp("ro.build.fingerprint", source_zip) |
| 500 | target_fp = GetBuildProp("ro.build.fingerprint", target_zip) |
| 501 | |
| 502 | script.append(('assert file_contains("SYSTEM:build.prop", ' |
| 503 | '"ro.build.fingerprint=%s") == "true" || ' |
| 504 | 'file_contains("SYSTEM:build.prop", ' |
| 505 | '"ro.build.fingerprint=%s") == "true"') % |
| 506 | (source_fp, target_fp)) |
| 507 | |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 508 | source_boot = File("/tmp/boot.img", |
| 509 | common.BuildBootableImage( |
| 510 | os.path.join(OPTIONS.source_tmp, "BOOT"))) |
| 511 | target_boot = File("/tmp/boot.img", |
| 512 | common.BuildBootableImage( |
| 513 | os.path.join(OPTIONS.target_tmp, "BOOT"))) |
| 514 | updating_boot = (source_boot.data != target_boot.data) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 515 | |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 516 | source_recovery = File("system/recovery.img", |
| 517 | common.BuildBootableImage( |
| 518 | os.path.join(OPTIONS.source_tmp, "RECOVERY"))) |
| 519 | target_recovery = File("system/recovery.img", |
| 520 | common.BuildBootableImage( |
| 521 | os.path.join(OPTIONS.target_tmp, "RECOVERY"))) |
| 522 | updating_recovery = (source_recovery.data != target_recovery.data) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 523 | |
| 524 | source_radio = source_zip.read("RADIO/image") |
| 525 | target_radio = target_zip.read("RADIO/image") |
| 526 | updating_radio = (source_radio != target_radio) |
| 527 | |
| 528 | # The last 0.1 is reserved for creating symlinks, fixing |
| 529 | # permissions, and writing the boot image (if necessary). |
| 530 | progress_bar_total = 1.0 |
| 531 | if updating_boot: |
| 532 | progress_bar_total -= 0.1 |
| 533 | if updating_radio: |
| 534 | progress_bar_total -= 0.3 |
| 535 | |
| 536 | AppendAssertions(script, target_zip) |
| 537 | |
| 538 | pb_verify = progress_bar_total * 0.3 * \ |
| 539 | (total_patched_size / |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 540 | float(total_patched_size+total_verbatim_size+1)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 541 | |
| 542 | for i, (fn, tf, sf, size) in enumerate(patch_list): |
| 543 | if i % 5 == 0: |
| 544 | next_sizes = sum([i[3] for i in patch_list[i:i+5]]) |
| 545 | script.append("show_progress %f 1" % |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 546 | (next_sizes * pb_verify / (total_patched_size+1),)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 547 | script.append("run_program PACKAGE:applypatch -c /%s %s %s" % |
| 548 | (fn, tf.sha1, sf.sha1)) |
| 549 | |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 550 | if updating_recovery: |
| 551 | d = Difference(target_recovery, source_recovery, "imgdiff") |
| 552 | print "recovery target: %d source: %d diff: %d" % ( |
| 553 | target_recovery.size, source_recovery.size, len(d)) |
| 554 | |
| 555 | output_zip.writestr("patch/recovery.img.p", d) |
| 556 | |
| 557 | script.append(("run_program PACKAGE:applypatch -c " |
| 558 | "MTD:recovery:%d:%s:%d:%s") % |
| 559 | (source_recovery.size, source_recovery.sha1, |
| 560 | target_recovery.size, target_recovery.sha1)) |
| 561 | |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 562 | if updating_boot: |
| 563 | d = Difference(target_boot, source_boot, "imgdiff") |
| 564 | print "boot target: %d source: %d diff: %d" % ( |
| 565 | target_boot.size, source_boot.size, len(d)) |
| 566 | |
| 567 | output_zip.writestr("patch/boot.img.p", d) |
| 568 | |
| 569 | script.append(("run_program PACKAGE:applypatch -c " |
| 570 | "MTD:boot:%d:%s:%d:%s") % |
| 571 | (source_boot.size, source_boot.sha1, |
| 572 | target_boot.size, target_boot.sha1)) |
| 573 | |
| 574 | if patch_list or updating_recovery or updating_boot: |
| 575 | script.append("run_program PACKAGE:applypatch -s %d" % |
| 576 | (largest_source_size,)) |
| 577 | script.append("copy_dir PACKAGE:patch CACHE:../tmp/patchtmp") |
| 578 | IncludeBinary("applypatch", target_zip, output_zip) |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 579 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 580 | script.append("\n# ---- start making changes here\n") |
| 581 | |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 582 | if OPTIONS.wipe_user_data: |
| 583 | script.append("format DATA:") |
| 584 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 585 | DeleteFiles(script, [SubstituteRoot(i[0]) for i in verbatim_targets]) |
| 586 | |
| 587 | if updating_boot: |
Doug Zongker | 5da317e | 2009-06-02 13:38:17 -0700 | [diff] [blame^] | 588 | # Produce the boot image by applying a patch to the current |
| 589 | # contents of the boot partition, and write it back to the |
| 590 | # partition. |
| 591 | script.append(("run_program PACKAGE:applypatch " |
| 592 | "MTD:boot:%d:%s:%d:%s - " |
| 593 | "%s %d %s:/tmp/patchtmp/boot.img.p") |
| 594 | % (source_boot.size, source_boot.sha1, |
| 595 | target_boot.size, target_boot.sha1, |
| 596 | target_boot.sha1, |
| 597 | target_boot.size, |
| 598 | source_boot.sha1)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 599 | print "boot image changed; including." |
| 600 | else: |
| 601 | print "boot image unchanged; skipping." |
| 602 | |
| 603 | if updating_recovery: |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 604 | # Produce /system/recovery.img by applying a patch to the current |
| 605 | # contents of the recovery partition. |
| 606 | script.append(("run_program PACKAGE:applypatch MTD:recovery:%d:%s:%d:%s " |
| 607 | "/system/recovery.img %s %d %s:/tmp/patchtmp/recovery.img.p") |
| 608 | % (source_recovery.size, source_recovery.sha1, |
| 609 | target_recovery.size, target_recovery.sha1, |
| 610 | target_recovery.sha1, |
| 611 | target_recovery.size, |
| 612 | source_recovery.sha1)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 613 | print "recovery image changed; including." |
| 614 | else: |
| 615 | print "recovery image unchanged; skipping." |
| 616 | |
| 617 | if updating_radio: |
| 618 | script.append("show_progress 0.3 10") |
| 619 | script.append("write_radio_image PACKAGE:radio.img") |
| 620 | output_zip.writestr("radio.img", target_radio) |
| 621 | print "radio image changed; including." |
| 622 | else: |
| 623 | print "radio image unchanged; skipping." |
| 624 | |
| 625 | pb_apply = progress_bar_total * 0.7 * \ |
| 626 | (total_patched_size / |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 627 | float(total_patched_size+total_verbatim_size+1)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 628 | for i, (fn, tf, sf, size) in enumerate(patch_list): |
| 629 | if i % 5 == 0: |
| 630 | next_sizes = sum([i[3] for i in patch_list[i:i+5]]) |
| 631 | script.append("show_progress %f 1" % |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 632 | (next_sizes * pb_apply / (total_patched_size+1),)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 633 | script.append(("run_program PACKAGE:applypatch " |
Doug Zongker | ef85ea6 | 2009-05-08 13:46:25 -0700 | [diff] [blame] | 634 | "/%s - %s %d %s:/tmp/patchtmp/%s.p") % |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 635 | (fn, tf.sha1, tf.size, sf.sha1, fn)) |
| 636 | |
| 637 | target_symlinks = CopySystemFiles(target_zip, None) |
| 638 | |
| 639 | target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks]) |
| 640 | temp_script = [] |
| 641 | FixPermissions(temp_script) |
| 642 | |
| 643 | # Note that this call will mess up the tree of Items, so make sure |
| 644 | # we're done with it. |
| 645 | source_symlinks = CopySystemFiles(source_zip, None) |
| 646 | source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks]) |
| 647 | |
| 648 | # Delete all the symlinks in source that aren't in target. This |
| 649 | # needs to happen before verbatim files are unpacked, in case a |
| 650 | # symlink in the source is replaced by a real file in the target. |
| 651 | to_delete = [] |
| 652 | for dest, link in source_symlinks: |
| 653 | if link not in target_symlinks_d: |
| 654 | to_delete.append(link) |
| 655 | DeleteFiles(script, to_delete) |
| 656 | |
| 657 | if verbatim_targets: |
| 658 | pb_verbatim = progress_bar_total * \ |
| 659 | (total_verbatim_size / |
Doug Zongker | f6a8bad | 2009-05-29 11:41:21 -0700 | [diff] [blame] | 660 | float(total_patched_size+total_verbatim_size+1)) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 661 | script.append("show_progress %f 5" % (pb_verbatim,)) |
| 662 | script.append("copy_dir PACKAGE:system SYSTEM:") |
| 663 | |
| 664 | # Create all the symlinks that don't already exist, or point to |
| 665 | # somewhere different than what we want. Delete each symlink before |
| 666 | # creating it, since the 'symlink' command won't overwrite. |
| 667 | to_create = [] |
| 668 | for dest, link in target_symlinks: |
| 669 | if link in source_symlinks_d: |
| 670 | if dest != source_symlinks_d[link]: |
| 671 | to_create.append((dest, link)) |
| 672 | else: |
| 673 | to_create.append((dest, link)) |
| 674 | DeleteFiles(script, [i[1] for i in to_create]) |
| 675 | script.extend(["symlink %s %s" % s for s in to_create]) |
| 676 | |
| 677 | # Now that the symlinks are created, we can set all the |
| 678 | # permissions. |
| 679 | script.extend(temp_script) |
| 680 | |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 681 | if OPTIONS.extra_script is not None: |
| 682 | script.append(OPTIONS.extra_script) |
| 683 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 684 | AddScript(script, output_zip) |
| 685 | |
| 686 | |
| 687 | def main(argv): |
| 688 | |
| 689 | def option_handler(o, a): |
| 690 | if o in ("-b", "--board_config"): |
| 691 | common.LoadBoardConfig(a) |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 692 | elif o in ("-k", "--package_key"): |
| 693 | OPTIONS.package_key = a |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 694 | elif o in ("-i", "--incremental_from"): |
| 695 | OPTIONS.incremental_source = a |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 696 | elif o in ("-w", "--wipe_user_data"): |
| 697 | OPTIONS.wipe_user_data = True |
Doug Zongker | 962069c | 2009-04-23 11:41:58 -0700 | [diff] [blame] | 698 | elif o in ("-n", "--no_prereq"): |
| 699 | OPTIONS.omit_prereq = True |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 700 | elif o in ("-e", "--extra_script"): |
| 701 | OPTIONS.extra_script = a |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 702 | else: |
| 703 | return False |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 704 | return True |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 705 | |
| 706 | args = common.ParseOptions(argv, __doc__, |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 707 | extra_opts="b:k:i:d:wne:", |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 708 | extra_long_opts=["board_config=", |
| 709 | "package_key=", |
Doug Zongker | dbfaae5 | 2009-04-21 17:12:54 -0700 | [diff] [blame] | 710 | "incremental_from=", |
Doug Zongker | 962069c | 2009-04-23 11:41:58 -0700 | [diff] [blame] | 711 | "wipe_user_data", |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 712 | "no_prereq", |
| 713 | "extra_script="], |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 714 | extra_option_handler=option_handler) |
| 715 | |
| 716 | if len(args) != 2: |
| 717 | common.Usage(__doc__) |
| 718 | sys.exit(1) |
| 719 | |
| 720 | if not OPTIONS.max_image_size: |
| 721 | print |
| 722 | print " WARNING: No board config specified; will not check image" |
| 723 | print " sizes against limits. Use -b to make sure the generated" |
| 724 | print " images don't exceed partition sizes." |
| 725 | print |
| 726 | |
Doug Zongker | 1c390a2 | 2009-05-14 19:06:36 -0700 | [diff] [blame] | 727 | if OPTIONS.extra_script is not None: |
| 728 | OPTIONS.extra_script = open(OPTIONS.extra_script).read() |
| 729 | |
Doug Zongker | eef3944 | 2009-04-02 12:14:19 -0700 | [diff] [blame] | 730 | print "unzipping target target-files..." |
| 731 | OPTIONS.input_tmp = common.UnzipTemp(args[0]) |
| 732 | OPTIONS.target_tmp = OPTIONS.input_tmp |
| 733 | input_zip = zipfile.ZipFile(args[0], "r") |
| 734 | if OPTIONS.package_key: |
| 735 | temp_zip_file = tempfile.NamedTemporaryFile() |
| 736 | output_zip = zipfile.ZipFile(temp_zip_file, "w", |
| 737 | compression=zipfile.ZIP_DEFLATED) |
| 738 | else: |
| 739 | output_zip = zipfile.ZipFile(args[1], "w", |
| 740 | compression=zipfile.ZIP_DEFLATED) |
| 741 | |
| 742 | if OPTIONS.incremental_source is None: |
| 743 | WriteFullOTAPackage(input_zip, output_zip) |
| 744 | else: |
| 745 | print "unzipping source target-files..." |
| 746 | OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source) |
| 747 | source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r") |
| 748 | WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) |
| 749 | |
| 750 | output_zip.close() |
| 751 | if OPTIONS.package_key: |
| 752 | SignOutput(temp_zip_file.name, args[1]) |
| 753 | temp_zip_file.close() |
| 754 | |
| 755 | common.Cleanup() |
| 756 | |
| 757 | print "done." |
| 758 | |
| 759 | |
| 760 | if __name__ == '__main__': |
| 761 | try: |
| 762 | main(sys.argv[1:]) |
| 763 | except common.ExternalError, e: |
| 764 | print |
| 765 | print " ERROR: %s" % (e,) |
| 766 | print |
| 767 | sys.exit(1) |