blob: fc6a4c66a296a0c796f7f384cc0ea047e2f7bf38 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build. An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage: ota_from_target_files [flags] input_target_files output_ota_package
23
24 -b (--board_config) <file>
Doug Zongkerfdd8e692009-08-03 17:27:48 -070025 Deprecated.
Doug Zongkereef39442009-04-02 12:14:19 -070026
27 -k (--package_key) <key>
28 Key to use to sign the package (default is
29 "build/target/product/security/testkey").
30
31 -i (--incremental_from) <file>
32 Generate an incremental OTA using the given target-files zip as
33 the starting build.
34
Doug Zongkerdbfaae52009-04-21 17:12:54 -070035 -w (--wipe_user_data)
36 Generate an OTA package that will wipe the user data partition
37 when installed.
38
Doug Zongker962069c2009-04-23 11:41:58 -070039 -n (--no_prereq)
40 Omit the timestamp prereq check normally included at the top of
41 the build scripts (used for developer OTA packages which
42 legitimately need to go back and forth).
43
Doug Zongker1c390a22009-05-14 19:06:36 -070044 -e (--extra_script) <file>
45 Insert the contents of file at the end of the update script.
46
Doug Zongkerc494d7c2009-06-18 08:43:44 -070047 -m (--script_mode) <mode>
48 Specify 'amend' or 'edify' scripts, or 'auto' to pick
49 automatically (this is the default).
50
Doug Zongkereef39442009-04-02 12:14:19 -070051"""
52
53import sys
54
55if sys.hexversion < 0x02040000:
56 print >> sys.stderr, "Python 2.4 or newer is required."
57 sys.exit(1)
58
59import copy
60import os
61import re
62import sha
63import subprocess
64import tempfile
65import time
66import zipfile
67
68import common
Doug Zongkerc494d7c2009-06-18 08:43:44 -070069import amend_generator
70import edify_generator
Doug Zongker03061472009-07-13 18:36:37 -070071import both_generator
Doug Zongkereef39442009-04-02 12:14:19 -070072
73OPTIONS = common.OPTIONS
74OPTIONS.package_key = "build/target/product/security/testkey"
75OPTIONS.incremental_source = None
76OPTIONS.require_verbatim = set()
77OPTIONS.prohibit_verbatim = set(("system/build.prop",))
78OPTIONS.patch_threshold = 0.95
Doug Zongkerdbfaae52009-04-21 17:12:54 -070079OPTIONS.wipe_user_data = False
Doug Zongker962069c2009-04-23 11:41:58 -070080OPTIONS.omit_prereq = False
Doug Zongker1c390a22009-05-14 19:06:36 -070081OPTIONS.extra_script = None
Doug Zongkerc494d7c2009-06-18 08:43:44 -070082OPTIONS.script_mode = 'auto'
Doug Zongkereef39442009-04-02 12:14:19 -070083
84def MostPopularKey(d, default):
85 """Given a dict, return the key corresponding to the largest
86 value. Returns 'default' if the dict is empty."""
87 x = [(v, k) for (k, v) in d.iteritems()]
88 if not x: return default
89 x.sort()
90 return x[-1][1]
91
92
93def IsSymlink(info):
94 """Return true if the zipfile.ZipInfo object passed in represents a
95 symlink."""
96 return (info.external_attr >> 16) == 0120777
97
98
99
100class Item:
101 """Items represent the metadata (user, group, mode) of files and
102 directories in the system image."""
103 ITEMS = {}
104 def __init__(self, name, dir=False):
105 self.name = name
106 self.uid = None
107 self.gid = None
108 self.mode = None
109 self.dir = dir
110
111 if name:
112 self.parent = Item.Get(os.path.dirname(name), dir=True)
113 self.parent.children.append(self)
114 else:
115 self.parent = None
116 if dir:
117 self.children = []
118
119 def Dump(self, indent=0):
120 if self.uid is not None:
121 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode)
122 else:
123 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode)
124 if self.dir:
125 print "%s%s" % (" "*indent, self.descendants)
126 print "%s%s" % (" "*indent, self.best_subtree)
127 for i in self.children:
128 i.Dump(indent=indent+1)
129
130 @classmethod
131 def Get(cls, name, dir=False):
132 if name not in cls.ITEMS:
133 cls.ITEMS[name] = Item(name, dir=dir)
134 return cls.ITEMS[name]
135
136 @classmethod
137 def GetMetadata(cls):
138 """Run the external 'fs_config' program to determine the desired
139 uid, gid, and mode for every Item object."""
140 p = common.Run(["fs_config"], stdin=subprocess.PIPE,
141 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
142 suffix = { False: "", True: "/" }
143 input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
144 for i in cls.ITEMS.itervalues() if i.name])
145 output, error = p.communicate(input)
146 assert not error
147
148 for line in output.split("\n"):
149 if not line: continue
150 name, uid, gid, mode = line.split()
151 i = cls.ITEMS[name]
152 i.uid = int(uid)
153 i.gid = int(gid)
154 i.mode = int(mode, 8)
155 if i.dir:
156 i.children.sort(key=lambda i: i.name)
157
158 def CountChildMetadata(self):
159 """Count up the (uid, gid, mode) tuples for all children and
160 determine the best strategy for using set_perm_recursive and
161 set_perm to correctly chown/chmod all the files to their desired
162 values. Recursively calls itself for all descendants.
163
164 Returns a dict of {(uid, gid, dmode, fmode): count} counting up
165 all descendants of this node. (dmode or fmode may be None.) Also
166 sets the best_subtree of each directory Item to the (uid, gid,
167 dmode, fmode) tuple that will match the most descendants of that
168 Item.
169 """
170
171 assert self.dir
172 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
173 for i in self.children:
174 if i.dir:
175 for k, v in i.CountChildMetadata().iteritems():
176 d[k] = d.get(k, 0) + v
177 else:
178 k = (i.uid, i.gid, None, i.mode)
179 d[k] = d.get(k, 0) + 1
180
181 # Find the (uid, gid, dmode, fmode) tuple that matches the most
182 # descendants.
183
184 # First, find the (uid, gid) pair that matches the most
185 # descendants.
186 ug = {}
187 for (uid, gid, _, _), count in d.iteritems():
188 ug[(uid, gid)] = ug.get((uid, gid), 0) + count
189 ug = MostPopularKey(ug, (0, 0))
190
191 # Now find the dmode and fmode that match the most descendants
192 # with that (uid, gid), and choose those.
193 best_dmode = (0, 0755)
194 best_fmode = (0, 0644)
195 for k, count in d.iteritems():
196 if k[:2] != ug: continue
197 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
198 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
199 self.best_subtree = ug + (best_dmode[1], best_fmode[1])
200
201 return d
202
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700203 def SetPermissions(self, script):
Doug Zongkereef39442009-04-02 12:14:19 -0700204 """Append set_perm/set_perm_recursive commands to 'script' to
205 set all permissions, users, and groups for the tree of files
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700206 rooted at 'self'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700207
208 self.CountChildMetadata()
209
210 def recurse(item, current):
211 # current is the (uid, gid, dmode, fmode) tuple that the current
212 # item (and all its children) have already been set to. We only
213 # need to issue set_perm/set_perm_recursive commands if we're
214 # supposed to be something different.
215 if item.dir:
216 if current != item.best_subtree:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700217 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
Doug Zongkereef39442009-04-02 12:14:19 -0700218 current = item.best_subtree
219
220 if item.uid != current[0] or item.gid != current[1] or \
221 item.mode != current[2]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700222 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700223
224 for i in item.children:
225 recurse(i, current)
226 else:
227 if item.uid != current[0] or item.gid != current[1] or \
228 item.mode != current[3]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700229 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700230
231 recurse(self, (-1, -1, -1, -1))
232
233
234def CopySystemFiles(input_zip, output_zip=None,
235 substitute=None):
236 """Copies files underneath system/ in the input zip to the output
237 zip. Populates the Item class with their metadata, and returns a
238 list of symlinks. output_zip may be None, in which case the copy is
239 skipped (but the other side effects still happen). substitute is an
240 optional dict of {output filename: contents} to be output instead of
241 certain input files.
242 """
243
244 symlinks = []
245
246 for info in input_zip.infolist():
247 if info.filename.startswith("SYSTEM/"):
248 basefilename = info.filename[7:]
249 if IsSymlink(info):
250 symlinks.append((input_zip.read(info.filename),
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700251 "/system/" + basefilename))
Doug Zongkereef39442009-04-02 12:14:19 -0700252 else:
253 info2 = copy.copy(info)
254 fn = info2.filename = "system/" + basefilename
255 if substitute and fn in substitute and substitute[fn] is None:
256 continue
257 if output_zip is not None:
258 if substitute and fn in substitute:
259 data = substitute[fn]
260 else:
261 data = input_zip.read(info.filename)
262 output_zip.writestr(info2, data)
263 if fn.endswith("/"):
264 Item.Get(fn[:-1], dir=True)
265 else:
266 Item.Get(fn, dir=False)
267
268 symlinks.sort()
269 return symlinks
270
271
Doug Zongkereef39442009-04-02 12:14:19 -0700272def SignOutput(temp_zip_name, output_zip_name):
273 key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
274 pw = key_passwords[OPTIONS.package_key]
275
Doug Zongker951495f2009-08-14 12:44:19 -0700276 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
277 whole_file=True)
Doug Zongkereef39442009-04-02 12:14:19 -0700278
279
Doug Zongkereef39442009-04-02 12:14:19 -0700280def AppendAssertions(script, input_zip):
Doug Zongkereef39442009-04-02 12:14:19 -0700281 device = GetBuildProp("ro.product.device", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700282 script.AssertDevice(device)
Doug Zongkereef39442009-04-02 12:14:19 -0700283
Doug Zongkereef39442009-04-02 12:14:19 -0700284
Doug Zongker73ef8252009-07-23 15:12:53 -0700285def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
286 """Generate a binary patch that creates the recovery image starting
287 with the boot image. (Most of the space in these images is just the
288 kernel, which is identical for the two, so the resulting patch
289 should be efficient.) Add it to the output zip, along with a shell
290 script that is run from init.rc on first boot to actually do the
291 patching and install the new recovery image.
292
293 recovery_img and boot_img should be File objects for the
294 corresponding images.
295
296 Returns an Item for the shell script, which must be made
297 executable.
298 """
299
300 patch = Difference(recovery_img, boot_img, "imgdiff")
301 common.ZipWriteStr(output_zip, "system/recovery-from-boot.p", patch)
302 Item.Get("system/recovery-from-boot.p", dir=False)
303
304 # Images with different content will have a different first page, so
305 # we check to see if this recovery has already been installed by
306 # testing just the first 2k.
307 HEADER_SIZE = 2048
308 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
309 sh = """#!/system/bin/sh
310if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
311 log -t recovery "Installing new recovery image"
312 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
313else
314 log -t recovery "Recovery image already installed"
315fi
316""" % { 'boot_size': boot_img.size,
317 'boot_sha1': boot_img.sha1,
318 'header_size': HEADER_SIZE,
319 'header_sha1': header_sha1,
320 'recovery_size': recovery_img.size,
321 'recovery_sha1': recovery_img.sha1 }
322 common.ZipWriteStr(output_zip, "system/etc/install-recovery.sh", sh)
323 return Item.Get("system/etc/install-recovery.sh", dir=False)
324
325
Doug Zongkereef39442009-04-02 12:14:19 -0700326def WriteFullOTAPackage(input_zip, output_zip):
Doug Zongker03061472009-07-13 18:36:37 -0700327 if OPTIONS.script_mode == "auto":
328 script = both_generator.BothGenerator(2)
329 elif OPTIONS.script_mode == "amend":
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700330 script = amend_generator.AmendGenerator()
331 else:
332 # TODO: how to determine this? We don't know what version it will
333 # be installed on top of. For now, we expect the API just won't
334 # change very often.
Doug Zongker03061472009-07-13 18:36:37 -0700335 script = edify_generator.EdifyGenerator(2)
Doug Zongkereef39442009-04-02 12:14:19 -0700336
Doug Zongker05d3dea2009-06-22 11:32:31 -0700337 device_specific = common.DeviceSpecificParams(
338 input_zip=input_zip,
339 output_zip=output_zip,
340 script=script,
341 input_tmp=OPTIONS.input_tmp)
342
Doug Zongker962069c2009-04-23 11:41:58 -0700343 if not OPTIONS.omit_prereq:
344 ts = GetBuildProp("ro.build.date.utc", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700345 script.AssertOlderBuild(ts)
Doug Zongkereef39442009-04-02 12:14:19 -0700346
347 AppendAssertions(script, input_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700348 device_specific.FullOTA_Assertions()
Doug Zongker171f1cd2009-06-15 22:36:37 -0700349
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700350 script.ShowProgress(0.5, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700351
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700352 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700353 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700354
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700355 script.FormatPartition("system")
356 script.Mount("MTD", "system", "/system")
357 script.UnpackPackageDir("system", "/system")
Doug Zongkereef39442009-04-02 12:14:19 -0700358
359 symlinks = CopySystemFiles(input_zip, output_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700360 script.MakeSymlinks(symlinks)
Doug Zongkereef39442009-04-02 12:14:19 -0700361
Doug Zongker73ef8252009-07-23 15:12:53 -0700362 boot_img = File("boot.img", common.BuildBootableImage(
363 os.path.join(OPTIONS.input_tmp, "BOOT")))
364 recovery_img = File("recovery.img", common.BuildBootableImage(
365 os.path.join(OPTIONS.input_tmp, "RECOVERY")))
366 i = MakeRecoveryPatch(output_zip, recovery_img, boot_img)
Doug Zongkereef39442009-04-02 12:14:19 -0700367
Doug Zongker73ef8252009-07-23 15:12:53 -0700368 Item.GetMetadata()
Doug Zongkereef39442009-04-02 12:14:19 -0700369
Doug Zongker73ef8252009-07-23 15:12:53 -0700370 # GetMetadata uses the data in android_filesystem_config.h to assign
371 # the uid/gid/mode of all files. We want to override that for the
372 # recovery patching shell script to make it executable.
373 i.uid = 0
374 i.gid = 0
375 i.mode = 0544
376 Item.Get("system").SetPermissions(script)
377
378 common.CheckSize(boot_img.data, "boot.img")
379 common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700380 script.ShowProgress(0.2, 0)
381
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700382 script.ShowProgress(0.2, 10)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700383 script.WriteRawImage("boot", "boot.img")
384
385 script.ShowProgress(0.1, 0)
386 device_specific.FullOTA_InstallEnd()
Doug Zongkereef39442009-04-02 12:14:19 -0700387
Doug Zongker1c390a22009-05-14 19:06:36 -0700388 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700389 script.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700390
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700391 script.AddToZip(input_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700392
393
394class File(object):
395 def __init__(self, name, data):
396 self.name = name
397 self.data = data
398 self.size = len(data)
399 self.sha1 = sha.sha(data).hexdigest()
400
401 def WriteToTemp(self):
402 t = tempfile.NamedTemporaryFile()
403 t.write(self.data)
404 t.flush()
405 return t
406
407 def AddToZip(self, z):
Doug Zongker048e7ca2009-06-15 14:31:53 -0700408 common.ZipWriteStr(z, self.name, self.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700409
410
411def LoadSystemFiles(z):
412 """Load all the files from SYSTEM/... in a given target-files
413 ZipFile, and return a dict of {filename: File object}."""
414 out = {}
415 for info in z.infolist():
416 if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
417 fn = "system/" + info.filename[7:]
418 data = z.read(info.filename)
419 out[fn] = File(fn, data)
420 return out
421
422
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700423def Difference(tf, sf, diff_program):
424 """Return the patch (as a string of data) needed to turn sf into tf.
425 diff_program is the name of an external program (or list, if
426 additional arguments are desired) to run to generate the diff.
427 """
Doug Zongkereef39442009-04-02 12:14:19 -0700428
429 ttemp = tf.WriteToTemp()
430 stemp = sf.WriteToTemp()
431
432 ext = os.path.splitext(tf.name)[1]
433
434 try:
435 ptemp = tempfile.NamedTemporaryFile()
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700436 if isinstance(diff_program, list):
437 cmd = copy.copy(diff_program)
438 else:
439 cmd = [diff_program]
440 cmd.append(stemp.name)
441 cmd.append(ttemp.name)
442 cmd.append(ptemp.name)
443 p = common.Run(cmd)
Doug Zongkereef39442009-04-02 12:14:19 -0700444 _, err = p.communicate()
Doug Zongker5da317e2009-06-02 13:38:17 -0700445 if err or p.returncode != 0:
446 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
447 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700448 diff = ptemp.read()
Doug Zongkereef39442009-04-02 12:14:19 -0700449 finally:
Doug Zongker5da317e2009-06-02 13:38:17 -0700450 ptemp.close()
Doug Zongkereef39442009-04-02 12:14:19 -0700451 stemp.close()
452 ttemp.close()
453
454 return diff
455
456
457def GetBuildProp(property, z):
458 """Return the fingerprint of the build of a given target-files
459 ZipFile object."""
460 bp = z.read("SYSTEM/build.prop")
461 if not property:
462 return bp
463 m = re.search(re.escape(property) + r"=(.*)\n", bp)
464 if not m:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700465 raise common.ExternalError("couldn't find %s in build.prop" % (property,))
Doug Zongkereef39442009-04-02 12:14:19 -0700466 return m.group(1).strip()
467
468
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700469def GetRecoveryAPIVersion(zip):
470 """Returns the version of the recovery API. Version 0 is the older
471 amend code (no separate binary)."""
472 try:
473 version = zip.read("META/recovery-api-version.txt")
474 return int(version)
475 except KeyError:
476 try:
477 # version one didn't have the recovery-api-version.txt file, but
478 # it did include an updater binary.
479 zip.getinfo("OTA/bin/updater")
480 return 1
481 except KeyError:
482 return 0
483
Doug Zongker15604b82009-09-01 17:53:34 -0700484
485DIFF_METHOD_BY_EXT = {
486 ".gz" : "imgdiff",
487 ".zip" : ["imgdiff", "-z"],
488 ".jar" : ["imgdiff", "-z"],
489 ".apk" : ["imgdiff", "-z"],
490 }
491
492
Doug Zongkereef39442009-04-02 12:14:19 -0700493def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700494 source_version = GetRecoveryAPIVersion(source_zip)
495
496 if OPTIONS.script_mode == 'amend':
497 script = amend_generator.AmendGenerator()
498 elif OPTIONS.script_mode == 'edify':
499 if source_version == 0:
500 print ("WARNING: generating edify script for a source that "
501 "can't install it.")
502 script = edify_generator.EdifyGenerator(source_version)
503 elif OPTIONS.script_mode == 'auto':
504 if source_version > 0:
505 script = edify_generator.EdifyGenerator(source_version)
506 else:
507 script = amend_generator.AmendGenerator()
508 else:
509 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
Doug Zongkereef39442009-04-02 12:14:19 -0700510
Doug Zongker05d3dea2009-06-22 11:32:31 -0700511 device_specific = common.DeviceSpecificParams(
512 source_zip=source_zip,
513 target_zip=target_zip,
514 output_zip=output_zip,
515 script=script)
516
Doug Zongkereef39442009-04-02 12:14:19 -0700517 print "Loading target..."
518 target_data = LoadSystemFiles(target_zip)
519 print "Loading source..."
520 source_data = LoadSystemFiles(source_zip)
521
522 verbatim_targets = []
523 patch_list = []
524 largest_source_size = 0
525 for fn in sorted(target_data.keys()):
526 tf = target_data[fn]
527 sf = source_data.get(fn, None)
528
529 if sf is None or fn in OPTIONS.require_verbatim:
530 # This file should be included verbatim
531 if fn in OPTIONS.prohibit_verbatim:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700532 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
Doug Zongkereef39442009-04-02 12:14:19 -0700533 print "send", fn, "verbatim"
534 tf.AddToZip(output_zip)
535 verbatim_targets.append((fn, tf.size))
536 elif tf.sha1 != sf.sha1:
537 # File is different; consider sending as a patch
Doug Zongker15604b82009-09-01 17:53:34 -0700538 ext = os.path.splitext(tf.name)[1]
539 diff_method = DIFF_METHOD_BY_EXT.get(ext, "bsdiff")
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700540 d = Difference(tf, sf, diff_method)
Doug Zongker5da317e2009-06-02 13:38:17 -0700541 if d is not None:
542 print fn, tf.size, len(d), (float(len(d)) / tf.size)
543 if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
Doug Zongkereef39442009-04-02 12:14:19 -0700544 # patch is almost as big as the file; don't bother patching
545 tf.AddToZip(output_zip)
546 verbatim_targets.append((fn, tf.size))
547 else:
Doug Zongker048e7ca2009-06-15 14:31:53 -0700548 common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d)
Doug Zongkereef39442009-04-02 12:14:19 -0700549 patch_list.append((fn, tf, sf, tf.size))
550 largest_source_size = max(largest_source_size, sf.size)
551 else:
552 # Target file identical to source.
553 pass
554
555 total_verbatim_size = sum([i[1] for i in verbatim_targets])
556 total_patched_size = sum([i[3] for i in patch_list])
557
558 source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
559 target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
560
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700561 script.Mount("MTD", "system", "/system")
562 script.AssertSomeFingerprint(source_fp, target_fp)
Doug Zongkereef39442009-04-02 12:14:19 -0700563
Doug Zongker5da317e2009-06-02 13:38:17 -0700564 source_boot = File("/tmp/boot.img",
565 common.BuildBootableImage(
566 os.path.join(OPTIONS.source_tmp, "BOOT")))
567 target_boot = File("/tmp/boot.img",
568 common.BuildBootableImage(
569 os.path.join(OPTIONS.target_tmp, "BOOT")))
570 updating_boot = (source_boot.data != target_boot.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700571
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700572 source_recovery = File("system/recovery.img",
573 common.BuildBootableImage(
574 os.path.join(OPTIONS.source_tmp, "RECOVERY")))
575 target_recovery = File("system/recovery.img",
576 common.BuildBootableImage(
577 os.path.join(OPTIONS.target_tmp, "RECOVERY")))
578 updating_recovery = (source_recovery.data != target_recovery.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700579
Doug Zongker05d3dea2009-06-22 11:32:31 -0700580 # We reserve the last 0.3 of the progress bar for the
581 # device-specific IncrementalOTA_InstallEnd() call at the end, which
582 # will typically install a radio image.
583 progress_bar_total = 0.7
Doug Zongkereef39442009-04-02 12:14:19 -0700584 if updating_boot:
585 progress_bar_total -= 0.1
Doug Zongkereef39442009-04-02 12:14:19 -0700586
587 AppendAssertions(script, target_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700588 device_specific.IncrementalOTA_Assertions()
Doug Zongkereef39442009-04-02 12:14:19 -0700589
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700590 script.Print("Verifying current system...")
591
Doug Zongkereef39442009-04-02 12:14:19 -0700592 pb_verify = progress_bar_total * 0.3 * \
593 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700594 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700595
596 for i, (fn, tf, sf, size) in enumerate(patch_list):
597 if i % 5 == 0:
598 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700599 script.ShowProgress(next_sizes * pb_verify / (total_patched_size+1), 1)
600
601 script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
Doug Zongkereef39442009-04-02 12:14:19 -0700602
Doug Zongker5da317e2009-06-02 13:38:17 -0700603 if updating_boot:
604 d = Difference(target_boot, source_boot, "imgdiff")
605 print "boot target: %d source: %d diff: %d" % (
606 target_boot.size, source_boot.size, len(d))
607
Doug Zongker048e7ca2009-06-15 14:31:53 -0700608 common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
Doug Zongker5da317e2009-06-02 13:38:17 -0700609
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700610 script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
611 (source_boot.size, source_boot.sha1,
612 target_boot.size, target_boot.sha1))
Doug Zongker5da317e2009-06-02 13:38:17 -0700613
614 if patch_list or updating_recovery or updating_boot:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700615 script.CacheFreeSpaceCheck(largest_source_size)
616 script.Print("Unpacking patches...")
617 script.UnpackPackageDir("patch", "/tmp/patchtmp")
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700618
Doug Zongker05d3dea2009-06-22 11:32:31 -0700619 device_specific.IncrementalOTA_VerifyEnd()
620
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700621 script.Comment("---- start making changes here ----")
Doug Zongkereef39442009-04-02 12:14:19 -0700622
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700623 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700624 script.Print("Erasing user data...")
625 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700626
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700627 script.Print("Removing unneeded files...")
Doug Zongker0f3298a2009-06-30 08:16:58 -0700628 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
629 ["/"+i for i in sorted(source_data)
Doug Zongker3b949f02009-08-24 10:24:32 -0700630 if i not in target_data] +
631 ["/system/recovery.img"])
Doug Zongkereef39442009-04-02 12:14:19 -0700632
633 if updating_boot:
Doug Zongker5da317e2009-06-02 13:38:17 -0700634 # Produce the boot image by applying a patch to the current
635 # contents of the boot partition, and write it back to the
636 # partition.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700637 script.Print("Patching boot image...")
638 script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
639 % (source_boot.size, source_boot.sha1,
640 target_boot.size, target_boot.sha1),
641 "-",
642 target_boot.size, target_boot.sha1,
643 source_boot.sha1, "/tmp/patchtmp/boot.img.p")
Doug Zongkereef39442009-04-02 12:14:19 -0700644 print "boot image changed; including."
645 else:
646 print "boot image unchanged; skipping."
647
648 if updating_recovery:
Doug Zongker73ef8252009-07-23 15:12:53 -0700649 # Is it better to generate recovery as a patch from the current
650 # boot image, or from the previous recovery image? For large
651 # updates with significant kernel changes, probably the former.
652 # For small updates where the kernel hasn't changed, almost
653 # certainly the latter. We pick the first option. Future
654 # complicated schemes may let us effectively use both.
655 #
656 # A wacky possibility: as long as there is room in the boot
657 # partition, include the binaries and image files from recovery in
658 # the boot image (though not in the ramdisk) so they can be used
659 # as fodder for constructing the recovery image.
660 recovery_sh_item = MakeRecoveryPatch(output_zip,
661 target_recovery, target_boot)
662 print "recovery image changed; including as patch from boot."
Doug Zongkereef39442009-04-02 12:14:19 -0700663 else:
664 print "recovery image unchanged; skipping."
665
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700666 script.Print("Patching system files...")
Doug Zongkereef39442009-04-02 12:14:19 -0700667 pb_apply = progress_bar_total * 0.7 * \
668 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700669 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700670 for i, (fn, tf, sf, size) in enumerate(patch_list):
671 if i % 5 == 0:
672 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700673 script.ShowProgress(next_sizes * pb_apply / (total_patched_size+1), 1)
674 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1,
675 sf.sha1, "/tmp/patchtmp/"+fn+".p")
Doug Zongkereef39442009-04-02 12:14:19 -0700676
677 target_symlinks = CopySystemFiles(target_zip, None)
678
679 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700680 temp_script = script.MakeTemporary()
Doug Zongker73ef8252009-07-23 15:12:53 -0700681 Item.GetMetadata()
682 if updating_recovery:
683 recovery_sh_item.uid = 0
684 recovery_sh_item.gid = 0
685 recovery_sh_item.mode = 0544
686 Item.Get("system").SetPermissions(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700687
688 # Note that this call will mess up the tree of Items, so make sure
689 # we're done with it.
690 source_symlinks = CopySystemFiles(source_zip, None)
691 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
692
693 # Delete all the symlinks in source that aren't in target. This
694 # needs to happen before verbatim files are unpacked, in case a
695 # symlink in the source is replaced by a real file in the target.
696 to_delete = []
697 for dest, link in source_symlinks:
698 if link not in target_symlinks_d:
699 to_delete.append(link)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700700 script.DeleteFiles(to_delete)
Doug Zongkereef39442009-04-02 12:14:19 -0700701
702 if verbatim_targets:
703 pb_verbatim = progress_bar_total * \
704 (total_verbatim_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700705 float(total_patched_size+total_verbatim_size+1))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700706 script.ShowProgress(pb_verbatim, 5)
707 script.Print("Unpacking new files...")
708 script.UnpackPackageDir("system", "/system")
709
Doug Zongker05d3dea2009-06-22 11:32:31 -0700710 script.Print("Symlinks and permissions...")
Doug Zongkereef39442009-04-02 12:14:19 -0700711
712 # Create all the symlinks that don't already exist, or point to
713 # somewhere different than what we want. Delete each symlink before
714 # creating it, since the 'symlink' command won't overwrite.
715 to_create = []
716 for dest, link in target_symlinks:
717 if link in source_symlinks_d:
718 if dest != source_symlinks_d[link]:
719 to_create.append((dest, link))
720 else:
721 to_create.append((dest, link))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700722 script.DeleteFiles([i[1] for i in to_create])
723 script.MakeSymlinks(to_create)
Doug Zongkereef39442009-04-02 12:14:19 -0700724
725 # Now that the symlinks are created, we can set all the
726 # permissions.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700727 script.AppendScript(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700728
Doug Zongker05d3dea2009-06-22 11:32:31 -0700729 # Write the radio image, if necessary.
730 script.ShowProgress(0.3, 10)
731 device_specific.IncrementalOTA_InstallEnd()
732
Doug Zongker1c390a22009-05-14 19:06:36 -0700733 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700734 scirpt.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700735
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700736 script.AddToZip(target_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700737
738
739def main(argv):
740
741 def option_handler(o, a):
742 if o in ("-b", "--board_config"):
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700743 pass # deprecated
Doug Zongkereef39442009-04-02 12:14:19 -0700744 elif o in ("-k", "--package_key"):
745 OPTIONS.package_key = a
Doug Zongkereef39442009-04-02 12:14:19 -0700746 elif o in ("-i", "--incremental_from"):
747 OPTIONS.incremental_source = a
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700748 elif o in ("-w", "--wipe_user_data"):
749 OPTIONS.wipe_user_data = True
Doug Zongker962069c2009-04-23 11:41:58 -0700750 elif o in ("-n", "--no_prereq"):
751 OPTIONS.omit_prereq = True
Doug Zongker1c390a22009-05-14 19:06:36 -0700752 elif o in ("-e", "--extra_script"):
753 OPTIONS.extra_script = a
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700754 elif o in ("-m", "--script_mode"):
755 OPTIONS.script_mode = a
Doug Zongkereef39442009-04-02 12:14:19 -0700756 else:
757 return False
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700758 return True
Doug Zongkereef39442009-04-02 12:14:19 -0700759
760 args = common.ParseOptions(argv, __doc__,
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700761 extra_opts="b:k:i:d:wne:m:",
Doug Zongkereef39442009-04-02 12:14:19 -0700762 extra_long_opts=["board_config=",
763 "package_key=",
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700764 "incremental_from=",
Doug Zongker962069c2009-04-23 11:41:58 -0700765 "wipe_user_data",
Doug Zongker1c390a22009-05-14 19:06:36 -0700766 "no_prereq",
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700767 "extra_script=",
768 "script_mode="],
Doug Zongkereef39442009-04-02 12:14:19 -0700769 extra_option_handler=option_handler)
770
771 if len(args) != 2:
772 common.Usage(__doc__)
773 sys.exit(1)
774
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700775 if OPTIONS.script_mode not in ("amend", "edify", "auto"):
776 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
777
Doug Zongker1c390a22009-05-14 19:06:36 -0700778 if OPTIONS.extra_script is not None:
779 OPTIONS.extra_script = open(OPTIONS.extra_script).read()
780
Doug Zongkereef39442009-04-02 12:14:19 -0700781 print "unzipping target target-files..."
782 OPTIONS.input_tmp = common.UnzipTemp(args[0])
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700783
784 common.LoadMaxSizes()
785 if not OPTIONS.max_image_size:
786 print
787 print " WARNING: Failed to load max image sizes; will not enforce"
788 print " image size limits."
789 print
790
Doug Zongkereef39442009-04-02 12:14:19 -0700791 OPTIONS.target_tmp = OPTIONS.input_tmp
792 input_zip = zipfile.ZipFile(args[0], "r")
793 if OPTIONS.package_key:
794 temp_zip_file = tempfile.NamedTemporaryFile()
795 output_zip = zipfile.ZipFile(temp_zip_file, "w",
796 compression=zipfile.ZIP_DEFLATED)
797 else:
798 output_zip = zipfile.ZipFile(args[1], "w",
799 compression=zipfile.ZIP_DEFLATED)
800
801 if OPTIONS.incremental_source is None:
802 WriteFullOTAPackage(input_zip, output_zip)
803 else:
804 print "unzipping source target-files..."
805 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
806 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
807 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
808
809 output_zip.close()
810 if OPTIONS.package_key:
811 SignOutput(temp_zip_file.name, args[1])
812 temp_zip_file.close()
813
814 common.Cleanup()
815
816 print "done."
817
818
819if __name__ == '__main__':
820 try:
821 main(sys.argv[1:])
822 except common.ExternalError, e:
823 print
824 print " ERROR: %s" % (e,)
825 print
826 sys.exit(1)