The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright (C) 2008 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the 'License'); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an 'AS IS' BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | # |
| 17 | |
| 18 | """Creates optimized versions of APK files. |
| 19 | |
| 20 | A tool and associated functions to communicate with an Android |
| 21 | emulator instance, run commands, and scrape out files. |
| 22 | |
| 23 | Requires at least python2.4. |
| 24 | """ |
| 25 | |
| 26 | import array |
| 27 | import datetime |
| 28 | import optparse |
| 29 | import os |
| 30 | import posix |
| 31 | import select |
| 32 | import signal |
| 33 | import struct |
| 34 | import subprocess |
| 35 | import sys |
| 36 | import tempfile |
| 37 | import time |
| 38 | import zlib |
| 39 | |
| 40 | |
| 41 | _emulator_popen = None |
| 42 | _DEBUG_READ = 1 |
| 43 | |
| 44 | |
| 45 | def EnsureTempDir(path=None): |
| 46 | """Creates a temporary directory and returns its path. |
| 47 | |
| 48 | Creates any necessary parent directories. |
| 49 | |
| 50 | Args: |
| 51 | path: If specified, used as the temporary directory. If not specified, |
| 52 | a safe temporary path is created. The caller is responsible for |
| 53 | deleting the directory. |
| 54 | |
| 55 | Returns: |
| 56 | The path to the new directory, or None if a problem occurred. |
| 57 | """ |
| 58 | if path is None: |
| 59 | path = tempfile.mkdtemp('', 'dexpreopt-') |
| 60 | elif not os.path.exists(path): |
| 61 | os.makedirs(path) |
| 62 | elif not os.path.isdir(path): |
| 63 | return None |
| 64 | return path |
| 65 | |
| 66 | |
| 67 | def CreateZeroedFile(path, length): |
| 68 | """Creates the named file and writes <length> zero bytes to it. |
| 69 | |
| 70 | Unlinks the file first if it already exists. |
| 71 | Creates its containing directory if necessary. |
| 72 | |
| 73 | Args: |
| 74 | path: The path to the file to create. |
| 75 | length: The number of zero bytes to write to the file. |
| 76 | |
| 77 | Returns: |
| 78 | True on success. |
| 79 | """ |
| 80 | subprocess.call(['rm', '-f', path]) |
| 81 | d = os.path.dirname(path) |
| 82 | if d and not os.path.exists(d): os.makedirs(os.path.dirname(d)) |
| 83 | # TODO: redirect child's stdout to /dev/null |
| 84 | ret = subprocess.call(['dd', 'if=/dev/zero', 'of=%s' % path, |
| 85 | 'bs=%d' % length, 'count=1']) |
| 86 | return not ret # i.e., ret == 0; i.e., the child exited successfully. |
| 87 | |
| 88 | |
| 89 | def StartEmulator(exe_name='emulator', kernel=None, |
| 90 | ramdisk=None, image=None, userdata=None, system=None): |
| 91 | """Runs the emulator with the specified arguments. |
| 92 | |
| 93 | Args: |
| 94 | exe_name: The name of the emulator to run. May be absolute, relative, |
| 95 | or unqualified (and left to exec() to find). |
| 96 | kernel: If set, passed to the emulator as "-kernel". |
| 97 | ramdisk: If set, passed to the emulator as "-ramdisk". |
The Android Open Source Project | 6bce205 | 2009-03-13 13:04:19 -0700 | [diff] [blame] | 98 | image: If set, passed to the emulator as "-system". |
The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 99 | userdata: If set, passed to the emulator as "-initdata" and "-data". |
The Android Open Source Project | 6bce205 | 2009-03-13 13:04:19 -0700 | [diff] [blame] | 100 | system: If set, passed to the emulator as "-sysdir". |
The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 101 | |
| 102 | Returns: |
| 103 | A subprocess.Popen that refers to the emulator process, or None if |
| 104 | a problem occurred. |
| 105 | """ |
| 106 | #exe_name = './stuff' |
| 107 | args = [exe_name] |
| 108 | if kernel: args += ['-kernel', kernel] |
| 109 | if ramdisk: args += ['-ramdisk', ramdisk] |
The Android Open Source Project | 6bce205 | 2009-03-13 13:04:19 -0700 | [diff] [blame] | 110 | if image: args += ['-system', image] |
The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 111 | if userdata: args += ['-initdata', userdata, '-data', userdata] |
The Android Open Source Project | 6bce205 | 2009-03-13 13:04:19 -0700 | [diff] [blame] | 112 | if system: args += ['-sysdir', system] |
| 113 | args += ['-partition-size', '128'] |
The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 114 | args += ['-no-window', '-netfast', '-noaudio'] |
| 115 | |
| 116 | _USE_PIPE = True |
| 117 | |
| 118 | if _USE_PIPE: |
| 119 | # Use dedicated fds instead of stdin/out to talk to the |
| 120 | # emulator so that the emulator doesn't try to tty-cook |
| 121 | # the data. |
| 122 | em_stdin_r, em_stdin_w = posix.pipe() |
| 123 | em_stdout_r, em_stdout_w = posix.pipe() |
| 124 | args += ['-shell-serial', 'fdpair:%d:%d' % (em_stdin_r, em_stdout_w)] |
| 125 | else: |
| 126 | args += ['-shell'] |
| 127 | |
David 'Digit' Turner | 171e0d0 | 2010-02-25 14:05:05 -0800 | [diff] [blame] | 128 | # This is a work-around for the ARMv7 emulation bug. |
| 129 | # XXX: It only works by chance, if any ! A real emulation fix is on the way |
| 130 | args += ['-qemu', '-singlestep'] |
| 131 | |
The Android Open Source Project | 88b6079 | 2009-03-03 19:28:42 -0800 | [diff] [blame] | 132 | # Ensure that this environment variable isn't set; |
| 133 | # if it is, the emulator will print the log to stdout. |
| 134 | if os.environ.get('ANDROID_LOG_TAGS'): |
| 135 | del os.environ['ANDROID_LOG_TAGS'] |
| 136 | |
| 137 | try: |
| 138 | # bufsize=1 line-buffered, =0 unbuffered, |
| 139 | # <0 system default (fully buffered) |
| 140 | Trace('Running emulator: %s' % ' '.join(args)) |
| 141 | if _USE_PIPE: |
| 142 | ep = subprocess.Popen(args) |
| 143 | else: |
| 144 | ep = subprocess.Popen(args, close_fds=True, |
| 145 | stdin=subprocess.PIPE, |
| 146 | stdout=subprocess.PIPE, |
| 147 | stderr=subprocess.PIPE) |
| 148 | if ep: |
| 149 | if _USE_PIPE: |
| 150 | # Hijack the Popen.stdin/.stdout fields to point to our |
| 151 | # pipes. These are the same fields that would have been set |
| 152 | # if we called Popen() with stdin=subprocess.PIPE, etc. |
| 153 | # Note that these names are from the point of view of the |
| 154 | # child process. |
| 155 | # |
| 156 | # Since we'll be using select.select() to read data a byte |
| 157 | # at a time, it's important that these files are unbuffered |
| 158 | # (bufsize=0). If Popen() took care of the pipes, they're |
| 159 | # already unbuffered. |
| 160 | ep.stdin = os.fdopen(em_stdin_w, 'w', 0) |
| 161 | ep.stdout = os.fdopen(em_stdout_r, 'r', 0) |
| 162 | return ep |
| 163 | except OSError, e: |
| 164 | print >>sys.stderr, 'Could not start emulator:', e |
| 165 | return None |
| 166 | |
| 167 | |
| 168 | def IsDataAvailable(fo, timeout=0): |
| 169 | """Indicates whether or not data is available to be read from a file object. |
| 170 | |
| 171 | Args: |
| 172 | fo: A file object to read from. |
| 173 | timeout: The number of seconds to wait for data, or zero for no timeout. |
| 174 | |
| 175 | Returns: |
| 176 | True iff data is available to be read. |
| 177 | """ |
| 178 | return select.select([fo], [], [], timeout) == ([fo], [], []) |
| 179 | |
| 180 | |
| 181 | def ConsumeAvailableData(fo): |
| 182 | """Reads data from a file object while it's available. |
| 183 | |
| 184 | Stops when no more data is immediately available or upon reaching EOF. |
| 185 | |
| 186 | Args: |
| 187 | fo: A file object to read from. |
| 188 | |
| 189 | Returns: |
| 190 | An unsigned byte array.array of the data that was read. |
| 191 | """ |
| 192 | buf = array.array('B') |
| 193 | while IsDataAvailable(fo): |
| 194 | try: |
| 195 | buf.fromfile(fo, 1) |
| 196 | except EOFError: |
| 197 | break |
| 198 | return buf |
| 199 | |
| 200 | |
| 201 | def ShowTimeout(timeout, end_time): |
| 202 | """For debugging, display the timeout info. |
| 203 | |
| 204 | Args: |
| 205 | timeout: the timeout in seconds. |
| 206 | end_time: a time.time()-based value indicating when the timeout should |
| 207 | expire. |
| 208 | """ |
| 209 | if _DEBUG_READ: |
| 210 | if timeout: |
| 211 | remaining = end_time - time.time() |
| 212 | Trace('ok, time remaining %.1f of %.1f' % (remaining, timeout)) |
| 213 | else: |
| 214 | Trace('ok (no timeout)') |
| 215 | |
| 216 | |
| 217 | def WaitForString(inf, pattern, timeout=0, max_len=0, eat_to_eol=True, |
| 218 | reset_on_activity=False): |
| 219 | """Reads from a file object and returns when the pattern matches the data. |
| 220 | |
| 221 | Reads a byte at a time to avoid consuming extra data, so do not call |
| 222 | this function when you expect the pattern to match a large amount of data. |
| 223 | |
| 224 | Args: |
| 225 | inf: The file object to read from. |
| 226 | pattern: The string to look for in the input data. |
| 227 | May be a tuple of strings. |
| 228 | timeout: How long to wait, in seconds. No timeout if it evaluates to False. |
| 229 | max_len: Return None if this many bytes have been read without matching. |
| 230 | No upper bound if it evaluates to False. |
| 231 | eat_to_eol: If true, the input data will be consumed until a '\\n' or EOF |
| 232 | is encountered. |
| 233 | reset_on_activity: If True, reset the timeout whenever a character is |
| 234 | read. |
| 235 | |
| 236 | Returns: |
| 237 | The input data matching the expression as an unsigned char array, |
| 238 | or None if the operation timed out or didn't match after max_len bytes. |
| 239 | |
| 240 | Raises: |
| 241 | IOError: An error occurred reading from the input file. |
| 242 | """ |
| 243 | if timeout: |
| 244 | end_time = time.time() + timeout |
| 245 | else: |
| 246 | end_time = 0 |
| 247 | |
| 248 | if _DEBUG_READ: |
| 249 | Trace('WaitForString: "%s", %.1f' % (pattern, timeout)) |
| 250 | |
| 251 | buf = array.array('B') # unsigned char array |
| 252 | eating = False |
| 253 | while True: |
| 254 | if end_time: |
| 255 | remaining = end_time - time.time() |
| 256 | if remaining <= 0: |
| 257 | Trace('Timeout expired after %.1f seconds' % timeout) |
| 258 | return None |
| 259 | else: |
| 260 | remaining = None |
| 261 | |
| 262 | if IsDataAvailable(inf, remaining): |
| 263 | if reset_on_activity and timeout: |
| 264 | end_time = time.time() + timeout |
| 265 | |
| 266 | buf.fromfile(inf, 1) |
| 267 | if _DEBUG_READ: |
| 268 | c = buf.tostring()[-1:] |
| 269 | ci = ord(c) |
| 270 | if ci < 0x20: c = '.' |
| 271 | if _DEBUG_READ > 1: |
| 272 | print 'read [%c] 0x%02x' % (c, ci) |
| 273 | |
| 274 | if not eating: |
| 275 | if buf.tostring().endswith(pattern): |
| 276 | if eat_to_eol: |
| 277 | if _DEBUG_READ > 1: |
| 278 | Trace('Matched; eating to EOL') |
| 279 | eating = True |
| 280 | else: |
| 281 | ShowTimeout(timeout, end_time) |
| 282 | return buf |
| 283 | if _DEBUG_READ > 2: |
| 284 | print '/%s/ ? "%s"' % (pattern, buf.tostring()) |
| 285 | else: |
| 286 | if buf.tostring()[-1:] == '\n': |
| 287 | ShowTimeout(timeout, end_time) |
| 288 | return buf |
| 289 | |
| 290 | if max_len and len(buf) >= max_len: return None |
| 291 | |
| 292 | |
| 293 | def WaitForEmulator(ep, timeout=0): |
| 294 | """Waits for the emulator to start up and print the first prompt. |
| 295 | |
| 296 | Args: |
| 297 | ep: A subprocess.Popen object referring to the emulator process. |
| 298 | timeout: How long to wait, in seconds. No timeout if it evaluates to False. |
| 299 | |
| 300 | Returns: |
| 301 | True on success, False if the timeout occurred. |
| 302 | """ |
| 303 | # Prime the pipe; the emulator doesn't start without this. |
| 304 | print >>ep.stdin, '' |
| 305 | |
| 306 | # Wait until the console is ready and the first prompt appears. |
| 307 | buf = WaitForString(ep.stdout, '#', timeout=timeout, eat_to_eol=False) |
| 308 | if buf: |
| 309 | Trace('Saw the prompt: "%s"' % buf.tostring()) |
| 310 | return True |
| 311 | return False |
| 312 | |
| 313 | |
| 314 | def WaitForPrompt(ep, prompt=None, timeout=0, reset_on_activity=False): |
| 315 | """Blocks until the prompt appears on ep.stdout or the timeout elapses. |
| 316 | |
| 317 | Args: |
| 318 | ep: A subprocess.Popen connection to the emulator process. |
| 319 | prompt: The prompt to wait for. If None, uses ep.prompt. |
| 320 | timeout: How many seconds to wait for the prompt. Waits forever |
| 321 | if timeout is zero. |
| 322 | reset_on_activity: If True, reset the timeout whenever a character is |
| 323 | read. |
| 324 | |
| 325 | Returns: |
| 326 | A string containing the data leading up to the prompt. The string |
| 327 | will always end in '\\n'. Returns None if the prompt was not seen |
| 328 | within the timeout, or if some other error occurred. |
| 329 | """ |
| 330 | if not prompt: prompt = ep.prompt |
| 331 | if prompt: |
| 332 | #Trace('waiting for prompt "%s"' % prompt) |
| 333 | data = WaitForString(ep.stdout, prompt, |
| 334 | timeout=timeout, reset_on_activity=reset_on_activity) |
| 335 | if data: |
| 336 | # data contains everything on ep.stdout up to and including the prompt, |
| 337 | # plus everything up 'til the newline. Scrape out the prompt |
| 338 | # and everything that follows, and ensure that the result ends |
| 339 | # in a newline (which is important if it would otherwise be empty). |
| 340 | s = data.tostring() |
| 341 | i = s.rfind(prompt) |
| 342 | s = s[:i] |
| 343 | if s[-1:] != '\n': |
| 344 | s += '\n' |
| 345 | if _DEBUG_READ: |
| 346 | print 'WaitForPrompt saw """\n%s"""' % s |
| 347 | return s |
| 348 | return None |
| 349 | |
| 350 | |
| 351 | def ReplaceEmulatorPrompt(ep, prompt=None): |
| 352 | """Replaces PS1 in the emulator with a different value. |
| 353 | |
| 354 | This is useful for making the prompt unambiguous; i.e., something |
| 355 | that probably won't appear in the output of another command. |
| 356 | |
| 357 | Assumes that the emulator is already sitting at a prompt, |
| 358 | waiting for shell input. |
| 359 | |
| 360 | Puts the new prompt in ep.prompt. |
| 361 | |
| 362 | Args: |
| 363 | ep: A subprocess.Popen object referring to the emulator process. |
| 364 | prompt: The new prompt to use |
| 365 | |
| 366 | Returns: |
| 367 | True on success, False if the timeout occurred. |
| 368 | """ |
| 369 | if not prompt: |
| 370 | prompt = '-----DEXPREOPT-PROMPT-----' |
| 371 | print >>ep.stdin, 'PS1="%s\n"' % prompt |
| 372 | ep.prompt = prompt |
| 373 | |
| 374 | # Eat the command echo. |
| 375 | data = WaitForPrompt(ep, timeout=2) |
| 376 | if not data: |
| 377 | return False |
| 378 | |
| 379 | # Make sure it's actually there. |
| 380 | return WaitForPrompt(ep, timeout=2) |
| 381 | |
| 382 | |
| 383 | def RunEmulatorCommand(ep, cmd, timeout=0): |
| 384 | """Sends the command to the emulator's shell and waits for the result. |
| 385 | |
| 386 | Assumes that the emulator is already sitting at a prompt, |
| 387 | waiting for shell input. |
| 388 | |
| 389 | Args: |
| 390 | ep: A subprocess.Popen object referring to the emulator process. |
| 391 | cmd: The shell command to run in the emulator. |
| 392 | timeout: The number of seconds to wait for the command to complete, |
| 393 | or zero for no timeout. |
| 394 | |
| 395 | Returns: |
| 396 | If the command ran and returned to the console prompt before the |
| 397 | timeout, returns the output of the command as a string. |
| 398 | Returns None otherwise. |
| 399 | """ |
| 400 | ConsumeAvailableData(ep.stdout) |
| 401 | |
| 402 | Trace('Running "%s"' % cmd) |
| 403 | print >>ep.stdin, '%s' % cmd |
| 404 | |
| 405 | # The console will echo the command. |
| 406 | #Trace('Waiting for echo') |
| 407 | if WaitForString(ep.stdout, cmd, timeout=timeout): |
| 408 | #Trace('Waiting for completion') |
| 409 | return WaitForPrompt(ep, timeout=timeout, reset_on_activity=True) |
| 410 | |
| 411 | return None |
| 412 | |
| 413 | |
| 414 | def ReadFileList(ep, dir_list, timeout=0): |
| 415 | """Returns a list of emulator files in each dir in dir_list. |
| 416 | |
| 417 | Args: |
| 418 | ep: A subprocess.Popen object referring to the emulator process. |
| 419 | dir_list: List absolute paths to directories to read. |
| 420 | timeout: The number of seconds to wait for the command to complete, |
| 421 | or zero for no timeout. |
| 422 | |
| 423 | Returns: |
| 424 | A list of absolute paths to files in the named directories, |
| 425 | in the context of the emulator's filesystem. |
| 426 | None on failure. |
| 427 | """ |
| 428 | ret = [] |
| 429 | for d in dir_list: |
| 430 | output = RunEmulatorCommand(ep, 'ls ' + d, timeout=timeout) |
| 431 | if not output: |
| 432 | Trace('Could not ls ' + d) |
| 433 | return None |
| 434 | ret += ['%s/%s' % (d, f) for f in output.splitlines()] |
| 435 | return ret |
| 436 | |
| 437 | |
| 438 | def DownloadDirectoryHierarchy(ep, src, dest, timeout=0): |
| 439 | """Recursively downloads an emulator directory to the local filesystem. |
| 440 | |
| 441 | Args: |
| 442 | ep: A subprocess.Popen object referring to the emulator process. |
| 443 | src: The path on the emulator's filesystem to download from. |
| 444 | dest: The path on the local filesystem to download to. |
| 445 | timeout: The number of seconds to wait for the command to complete, |
| 446 | or zero for no timeout. (CURRENTLY IGNORED) |
| 447 | |
| 448 | Returns: |
| 449 | True iff the files downloaded successfully, False otherwise. |
| 450 | """ |
| 451 | ConsumeAvailableData(ep.stdout) |
| 452 | |
| 453 | if not os.path.exists(dest): |
| 454 | os.makedirs(dest) |
| 455 | |
| 456 | cmd = 'afar %s' % src |
| 457 | Trace('Running "%s"' % cmd) |
| 458 | print >>ep.stdin, '%s' % cmd |
| 459 | |
| 460 | # The console will echo the command. |
| 461 | #Trace('Waiting for echo') |
| 462 | if not WaitForString(ep.stdout, cmd, timeout=timeout): |
| 463 | return False |
| 464 | |
| 465 | #TODO: use a signal to support timing out? |
| 466 | |
| 467 | # |
| 468 | # Android File Archive format: |
| 469 | # |
| 470 | # magic[5]: 'A' 'F' 'A' 'R' '\n' |
| 471 | # version[4]: 0x00 0x00 0x00 0x01 |
| 472 | # for each file: |
| 473 | # file magic[4]: 'F' 'I' 'L' 'E' |
| 474 | # namelen[4]: Length of file name, including NUL byte (big-endian) |
| 475 | # name[*]: NUL-terminated file name |
| 476 | # datalen[4]: Length of file (big-endian) |
| 477 | # data[*]: Unencoded file data |
| 478 | # adler32[4]: adler32 of the unencoded file data (big-endian) |
| 479 | # file end magic[4]: 'f' 'i' 'l' 'e' |
| 480 | # end magic[4]: 'E' 'N' 'D' 0x00 |
| 481 | # |
| 482 | |
| 483 | # Read the header. |
| 484 | HEADER = array.array('B', 'AFAR\n\000\000\000\001') |
| 485 | buf = array.array('B') |
| 486 | buf.fromfile(ep.stdout, len(HEADER)) |
| 487 | if buf != HEADER: |
| 488 | Trace('Header does not match: "%s"' % buf) |
| 489 | return False |
| 490 | |
| 491 | # Read the file entries. |
| 492 | FILE_START = array.array('B', 'FILE') |
| 493 | FILE_END = array.array('B', 'file') |
| 494 | END = array.array('B', 'END\000') |
| 495 | while True: |
| 496 | # Entry magic. |
| 497 | buf = array.array('B') |
| 498 | buf.fromfile(ep.stdout, 4) |
| 499 | if buf == FILE_START: |
| 500 | # Name length (4 bytes, big endian) |
| 501 | buf = array.array('B') |
| 502 | buf.fromfile(ep.stdout, 4) |
| 503 | (name_len,) = struct.unpack('>I', buf) |
| 504 | #Trace('name len %d' % name_len) |
| 505 | |
| 506 | # Name, NUL-terminated. |
| 507 | buf = array.array('B') |
| 508 | buf.fromfile(ep.stdout, name_len) |
| 509 | buf.pop() # Remove trailing NUL byte. |
| 510 | file_name = buf.tostring() |
| 511 | Trace('FILE: %s' % file_name) |
| 512 | |
| 513 | # File length (4 bytes, big endian) |
| 514 | buf = array.array('B') |
| 515 | buf.fromfile(ep.stdout, 4) |
| 516 | (file_len,) = struct.unpack('>I', buf) |
| 517 | |
| 518 | # File data. |
| 519 | data = array.array('B') |
| 520 | data.fromfile(ep.stdout, file_len) |
| 521 | #Trace('FILE: read %d bytes from %s' % (file_len, file_name)) |
| 522 | |
| 523 | # adler32 (4 bytes, big endian) |
| 524 | buf = array.array('B') |
| 525 | buf.fromfile(ep.stdout, 4) |
| 526 | (adler32,) = struct.unpack('>i', buf) # adler32 wants a signed int ('i') |
| 527 | data_adler32 = zlib.adler32(data) |
| 528 | # Because of a difference in behavior of zlib.adler32 on 32-bit and 64-bit |
| 529 | # systems (one returns signed values, the other unsigned), we take the |
| 530 | # modulo 2**32 of the checksums, and compare those. |
| 531 | # See also http://bugs.python.org/issue1202 |
| 532 | if (adler32 % (2**32)) != (data_adler32 % (2**32)): |
| 533 | Trace('adler32 does not match: calculated 0x%08x != expected 0x%08x' % |
| 534 | (data_adler32, adler32)) |
| 535 | return False |
| 536 | |
| 537 | # File end magic. |
| 538 | buf = array.array('B') |
| 539 | buf.fromfile(ep.stdout, 4) |
| 540 | if buf != FILE_END: |
| 541 | Trace('Unexpected file end magic "%s"' % buf) |
| 542 | return False |
| 543 | |
| 544 | # Write to the output file |
| 545 | out_file_name = dest + '/' + file_name[len(src):] |
| 546 | p = os.path.dirname(out_file_name) |
| 547 | if not os.path.exists(p): os.makedirs(p) |
| 548 | fo = file(out_file_name, 'w+b') |
| 549 | fo.truncate(0) |
| 550 | Trace('FILE: Writing %d bytes to %s' % (len(data), out_file_name)) |
| 551 | data.tofile(fo) |
| 552 | fo.close() |
| 553 | |
| 554 | elif buf == END: |
| 555 | break |
| 556 | else: |
| 557 | Trace('Unexpected magic "%s"' % buf) |
| 558 | return False |
| 559 | |
| 560 | return WaitForPrompt(ep, timeout=timeout, reset_on_activity=True) |
| 561 | |
| 562 | |
| 563 | def ReadBootClassPath(ep, timeout=0): |
| 564 | """Reads and returns the default bootclasspath as a list of files. |
| 565 | |
| 566 | Args: |
| 567 | ep: A subprocess.Popen object referring to the emulator process. |
| 568 | timeout: The number of seconds to wait for the command to complete, |
| 569 | or zero for no timeout. |
| 570 | |
| 571 | Returns: |
| 572 | The bootclasspath as a list of strings. |
| 573 | None on failure. |
| 574 | """ |
| 575 | bcp = RunEmulatorCommand(ep, 'echo $BOOTCLASSPATH', timeout=timeout) |
| 576 | if not bcp: |
| 577 | Trace('Could not find bootclasspath') |
| 578 | return None |
| 579 | return bcp.strip().split(':') # strip trailing newline |
| 580 | |
| 581 | |
| 582 | def RunDexoptOnFileList(ep, files, dest_root, move=False, timeout=0): |
| 583 | """Creates the corresponding .odex file for all jar/apk files in 'files'. |
| 584 | Copies the .odex file to a location under 'dest_root'. If 'move' is True, |
| 585 | the file is moved instead of copied. |
| 586 | |
| 587 | Args: |
| 588 | ep: A subprocess.Popen object referring to the emulator process. |
| 589 | files: The list of files to optimize |
| 590 | dest_root: directory to copy/move odex files to. Must already exist. |
| 591 | move: if True, move rather than copy files |
| 592 | timeout: The number of seconds to wait for the command to complete, |
| 593 | or zero for no timeout. |
| 594 | |
| 595 | Returns: |
| 596 | True on success, False on failure. |
| 597 | """ |
| 598 | for jar_file in files: |
| 599 | if jar_file.endswith('.apk') or jar_file.endswith('.jar'): |
| 600 | odex_file = jar_file[:jar_file.rfind('.')] + '.odex' |
| 601 | cmd = 'dexopt-wrapper %s %s' % (jar_file, odex_file) |
| 602 | if not RunEmulatorCommand(ep, cmd, timeout=timeout): |
| 603 | Trace('"%s" failed' % cmd) |
| 604 | return False |
| 605 | |
| 606 | # Always copy the odex file. There's no cp(1), so we |
| 607 | # cat out to the new file. |
| 608 | dst_odex = dest_root + odex_file |
| 609 | cmd = 'cat %s > %s' % (odex_file, dst_odex) # no cp(1) |
| 610 | if not RunEmulatorCommand(ep, cmd, timeout=timeout): |
| 611 | Trace('"%s" failed' % cmd) |
| 612 | return False |
| 613 | |
| 614 | # Move it if we're asked to. We can't use mv(1) because |
| 615 | # the files tend to move between filesystems. |
| 616 | if move: |
| 617 | cmd = 'rm %s' % odex_file |
| 618 | if not RunEmulatorCommand(ep, cmd, timeout=timeout): |
| 619 | Trace('"%s" failed' % cmd) |
| 620 | return False |
| 621 | return True |
| 622 | |
| 623 | |
| 624 | def InstallCacheFiles(cache_system_dir, out_system_dir): |
| 625 | """Install files in cache_system_dir to the proper places in out_system_dir. |
| 626 | |
| 627 | cache_system_dir contains various files from /system, plus .odex files |
| 628 | for most of the .apk/.jar files that live there. |
| 629 | This function copies each .odex file from the cache dir to the output dir |
| 630 | and removes "classes.dex" from each appropriate .jar/.apk. |
| 631 | |
| 632 | E.g., <cache_system_dir>/app/NotePad.odex would be copied to |
| 633 | <out_system_dir>/app/NotePad.odex, and <out_system_dir>/app/NotePad.apk |
| 634 | would have its classes.dex file removed. |
| 635 | |
| 636 | Args: |
| 637 | cache_system_dir: The directory containing the cache files scraped from |
| 638 | the emulator. |
| 639 | out_system_dir: The local directory that corresponds to "/system" |
| 640 | on the device filesystem. (the root of system.img) |
| 641 | |
| 642 | Returns: |
| 643 | True if everything succeeded, False if any problems occurred. |
| 644 | """ |
| 645 | # First, walk through cache_system_dir and copy every .odex file |
| 646 | # over to out_system_dir, ensuring that the destination directory |
| 647 | # contains the corresponding source file. |
| 648 | for root, dirs, files in os.walk(cache_system_dir): |
| 649 | for name in files: |
| 650 | if name.endswith('.odex'): |
| 651 | odex_file = os.path.join(root, name) |
| 652 | |
| 653 | # Find the path to the .odex file's source apk/jar file. |
| 654 | out_stem = odex_file[len(cache_system_dir):odex_file.rfind('.')] |
| 655 | out_stem = out_system_dir + out_stem; |
| 656 | jar_file = out_stem + '.jar' |
| 657 | if not os.path.exists(jar_file): |
| 658 | jar_file = out_stem + '.apk' |
| 659 | if not os.path.exists(jar_file): |
| 660 | Trace('Cannot find source .jar/.apk for %s: %s' % |
| 661 | (odex_file, out_stem + '.{jar,apk}')) |
| 662 | return False |
| 663 | |
| 664 | # Copy the cache file next to the source file. |
| 665 | cmd = ['cp', odex_file, out_stem + '.odex'] |
| 666 | ret = subprocess.call(cmd) |
| 667 | if ret: # non-zero exit status |
| 668 | Trace('%s failed' % ' '.join(cmd)) |
| 669 | return False |
| 670 | |
| 671 | # Walk through the output /system directory, making sure |
| 672 | # that every .jar/.apk has an odex file. While we do this, |
| 673 | # remove the classes.dex entry from each source archive. |
| 674 | for root, dirs, files in os.walk(out_system_dir): |
| 675 | for name in files: |
| 676 | if name.endswith('.apk') or name.endswith('.jar'): |
| 677 | jar_file = os.path.join(root, name) |
| 678 | odex_file = jar_file[:jar_file.rfind('.')] + '.odex' |
| 679 | if not os.path.exists(odex_file): |
| 680 | if root.endswith('/system/app') or root.endswith('/system/framework'): |
| 681 | Trace('jar/apk %s has no .odex file %s' % (jar_file, odex_file)) |
| 682 | return False |
| 683 | else: |
| 684 | continue |
| 685 | |
| 686 | # Attempting to dexopt a jar with no classes.dex currently |
| 687 | # creates a 40-byte odex file. |
| 688 | # TODO: use a more reliable check |
| 689 | if os.path.getsize(odex_file) > 100: |
| 690 | # Remove classes.dex from the .jar file. |
| 691 | cmd = ['zip', '-dq', jar_file, 'classes.dex'] |
| 692 | ret = subprocess.call(cmd) |
| 693 | if ret: # non-zero exit status |
| 694 | Trace('"%s" failed' % ' '.join(cmd)) |
| 695 | return False |
| 696 | else: |
| 697 | # Some of the apk files don't contain any code. |
| 698 | if not name.endswith('.apk'): |
| 699 | Trace('%s has a zero-length odex file' % jar_file) |
| 700 | return False |
| 701 | cmd = ['rm', odex_file] |
| 702 | ret = subprocess.call(cmd) |
| 703 | if ret: # non-zero exit status |
| 704 | Trace('"%s" failed' % ' '.join(cmd)) |
| 705 | return False |
| 706 | |
| 707 | return True |
| 708 | |
| 709 | |
| 710 | def KillChildProcess(p, sig=signal.SIGTERM, timeout=0): |
| 711 | """Waits for a child process to die without getting stuck in wait(). |
| 712 | |
| 713 | After Jean Brouwers's 2004 post to python-list. |
| 714 | |
| 715 | Args: |
| 716 | p: A subprocess.Popen representing the child process to kill. |
| 717 | sig: The signal to send to the child process. |
| 718 | timeout: How many seconds to wait for the child process to die. |
| 719 | If zero, do not time out. |
| 720 | |
| 721 | Returns: |
| 722 | The exit status of the child process, if it was successfully killed. |
| 723 | The final value of p.returncode if it wasn't. |
| 724 | """ |
| 725 | os.kill(p.pid, sig) |
| 726 | if timeout > 0: |
| 727 | while p.poll() < 0: |
| 728 | if timeout > 0.5: |
| 729 | timeout -= 0.25 |
| 730 | time.sleep(0.25) |
| 731 | else: |
| 732 | os.kill(p.pid, signal.SIGKILL) |
| 733 | time.sleep(0.5) |
| 734 | p.poll() |
| 735 | break |
| 736 | else: |
| 737 | p.wait() |
| 738 | return p.returncode |
| 739 | |
| 740 | |
| 741 | def Trace(msg): |
| 742 | """Prints a message to stdout. |
| 743 | |
| 744 | Args: |
| 745 | msg: The message to print. |
| 746 | """ |
| 747 | #print 'dexpreopt: %s' % msg |
| 748 | when = datetime.datetime.now() |
| 749 | print '%02d:%02d.%d dexpreopt: %s' % (when.minute, when.second, when.microsecond, msg) |
| 750 | |
| 751 | |
| 752 | def KillEmulator(): |
| 753 | """Attempts to kill the emulator process, if it is running. |
| 754 | |
| 755 | Returns: |
| 756 | The exit status of the emulator process, or None if the emulator |
| 757 | was not running or was unable to be killed. |
| 758 | """ |
| 759 | global _emulator_popen |
| 760 | if _emulator_popen: |
| 761 | Trace('Killing emulator') |
| 762 | try: |
| 763 | ret = KillChildProcess(_emulator_popen, sig=signal.SIGINT, timeout=5) |
| 764 | except OSError: |
| 765 | Trace('Could not kill emulator') |
| 766 | ret = None |
| 767 | _emulator_popen = None |
| 768 | return ret |
| 769 | return None |
| 770 | |
| 771 | |
| 772 | def Fail(msg=None): |
| 773 | """Prints an error and causes the process to exit. |
| 774 | |
| 775 | Args: |
| 776 | msg: Additional error string to print (optional). |
| 777 | |
| 778 | Returns: |
| 779 | Does not return. |
| 780 | """ |
| 781 | s = 'dexpreopt: ERROR' |
| 782 | if msg: s += ': %s' % msg |
| 783 | print >>sys.stderr, msg |
| 784 | KillEmulator() |
| 785 | sys.exit(1) |
| 786 | |
| 787 | |
| 788 | def PrintUsage(msg=None): |
| 789 | """Prints commandline usage information for the tool and exits with an error. |
| 790 | |
| 791 | Args: |
| 792 | msg: Additional string to print (optional). |
| 793 | |
| 794 | Returns: |
| 795 | Does not return. |
| 796 | """ |
| 797 | if msg: |
| 798 | print >>sys.stderr, 'dexpreopt: %s', msg |
| 799 | print >>sys.stderr, """Usage: dexpreopt <options> |
| 800 | Required options: |
| 801 | -kernel <kernel file> Kernel to use when running the emulator |
| 802 | -ramdisk <ramdisk.img file> Ramdisk to use when running the emulator |
| 803 | -image <system.img file> System image to use when running the |
| 804 | emulator. /system/app should contain the |
| 805 | .apk files to optimize, and any required |
| 806 | bootclasspath libraries must be present |
| 807 | in the correct locations. |
| 808 | -system <path> The product directory, which usually contains |
| 809 | files like 'system.img' (files other than |
| 810 | the kernel in that directory won't |
| 811 | be used) |
| 812 | -outsystemdir <path> A fully-populated /system directory, ready |
| 813 | to be modified to contain the optimized |
| 814 | files. The appropriate .jar/.apk files |
| 815 | will be stripped of their classes.dex |
| 816 | entries, and the optimized .dex files |
| 817 | will be added alongside the packages |
| 818 | that they came from. |
| 819 | Optional: |
| 820 | -tmpdir <path> If specified, use this directory for |
| 821 | intermediate objects. If not specified, |
| 822 | a unique directory under the system |
| 823 | temp dir is used. |
| 824 | """ |
| 825 | sys.exit(2) |
| 826 | |
| 827 | |
| 828 | def ParseArgs(argv): |
| 829 | """Parses commandline arguments. |
| 830 | |
| 831 | Args: |
| 832 | argv: A list of arguments; typically sys.argv[1:] |
| 833 | |
| 834 | Returns: |
| 835 | A tuple containing two dictionaries; the first contains arguments |
| 836 | that will be passsed to the emulator, and the second contains other |
| 837 | arguments. |
| 838 | """ |
| 839 | parser = optparse.OptionParser() |
| 840 | |
| 841 | parser.add_option('--kernel', help='Passed to emulator') |
| 842 | parser.add_option('--ramdisk', help='Passed to emulator') |
| 843 | parser.add_option('--image', help='Passed to emulator') |
| 844 | parser.add_option('--system', help='Passed to emulator') |
| 845 | parser.add_option('--outsystemdir', help='Destination /system directory') |
| 846 | parser.add_option('--tmpdir', help='Optional temp directory to use') |
| 847 | |
| 848 | options, args = parser.parse_args(args=argv) |
| 849 | if args: PrintUsage() |
| 850 | |
| 851 | emulator_args = {} |
| 852 | other_args = {} |
| 853 | if options.kernel: emulator_args['kernel'] = options.kernel |
| 854 | if options.ramdisk: emulator_args['ramdisk'] = options.ramdisk |
| 855 | if options.image: emulator_args['image'] = options.image |
| 856 | if options.system: emulator_args['system'] = options.system |
| 857 | if options.outsystemdir: other_args['outsystemdir'] = options.outsystemdir |
| 858 | if options.tmpdir: other_args['tmpdir'] = options.tmpdir |
| 859 | |
| 860 | return (emulator_args, other_args) |
| 861 | |
| 862 | |
| 863 | def DexoptEverything(ep, dest_root): |
| 864 | """Logic for finding and dexopting files in the necessary order. |
| 865 | |
| 866 | Args: |
| 867 | ep: A subprocess.Popen object referring to the emulator process. |
| 868 | dest_root: directory to copy/move odex files to |
| 869 | |
| 870 | Returns: |
| 871 | True on success, False on failure. |
| 872 | """ |
| 873 | _extra_tests = False |
| 874 | if _extra_tests: |
| 875 | if not RunEmulatorCommand(ep, 'ls /system/app', timeout=5): |
| 876 | Fail('Could not ls') |
| 877 | |
| 878 | # We're very short on space, so remove a bunch of big stuff that we |
| 879 | # don't need. |
| 880 | cmd = 'rm -r /system/sounds /system/media /system/fonts /system/xbin' |
| 881 | if not RunEmulatorCommand(ep, cmd, timeout=40): |
| 882 | Trace('"%s" failed' % cmd) |
| 883 | return False |
| 884 | |
| 885 | Trace('Read file list') |
| 886 | jar_dirs = ['/system/framework', '/system/app'] |
| 887 | files = ReadFileList(ep, jar_dirs, timeout=5) |
| 888 | if not files: |
| 889 | Fail('Could not list files in %s' % ' '.join(jar_dirs)) |
| 890 | #Trace('File list:\n"""\n%s\n"""' % '\n'.join(files)) |
| 891 | |
| 892 | bcp = ReadBootClassPath(ep, timeout=2) |
| 893 | if not files: |
| 894 | Fail('Could not sort by bootclasspath') |
| 895 | |
| 896 | # Remove bootclasspath entries from the main file list. |
| 897 | for jar in bcp: |
| 898 | try: |
| 899 | files.remove(jar) |
| 900 | except ValueError: |
| 901 | Trace('File list does not contain bootclasspath entry "%s"' % jar) |
| 902 | return False |
| 903 | |
| 904 | # Create the destination directories. |
| 905 | for d in ['', '/system'] + jar_dirs: |
| 906 | cmd = 'mkdir %s%s' % (dest_root, d) |
| 907 | if not RunEmulatorCommand(ep, cmd, timeout=4): |
| 908 | Trace('"%s" failed' % cmd) |
| 909 | return False |
| 910 | |
| 911 | # First, dexopt the bootclasspath. Keep their cache files in place. |
| 912 | Trace('Dexopt %d bootclasspath files' % len(bcp)) |
| 913 | if not RunDexoptOnFileList(ep, bcp, dest_root, timeout=120): |
| 914 | Trace('Could not dexopt bootclasspath') |
| 915 | return False |
| 916 | |
| 917 | # dexopt the rest. To avoid running out of space on the emulator |
| 918 | # volume, move each cache file after it's been created. |
| 919 | Trace('Dexopt %d files' % len(files)) |
| 920 | if not RunDexoptOnFileList(ep, files, dest_root, move=True, timeout=120): |
| 921 | Trace('Could not dexopt files') |
| 922 | return False |
| 923 | |
| 924 | if _extra_tests: |
| 925 | if not RunEmulatorCommand(ep, 'ls /system/app', timeout=5): |
| 926 | Fail('Could not ls') |
| 927 | |
| 928 | return True |
| 929 | |
| 930 | |
| 931 | |
| 932 | def MainInternal(): |
| 933 | """Main function that can be wrapped in a try block. |
| 934 | |
| 935 | Returns: |
| 936 | Nothing. |
| 937 | """ |
| 938 | emulator_args, other_args = ParseArgs(sys.argv[1:]) |
| 939 | |
| 940 | tmp_dir = EnsureTempDir(other_args.get('tmpdir')) |
| 941 | if not tmp_dir: Fail('Could not create temp dir') |
| 942 | |
| 943 | Trace('Creating data image') |
| 944 | userdata = '%s/data.img' % tmp_dir |
| 945 | if not CreateZeroedFile(userdata, 32 * 1024 * 1024): |
| 946 | Fail('Could not create data image file') |
| 947 | emulator_args['userdata'] = userdata |
| 948 | |
| 949 | ep = StartEmulator(**emulator_args) |
| 950 | if not ep: Fail('Could not start emulator') |
| 951 | global _emulator_popen |
| 952 | _emulator_popen = ep |
| 953 | |
| 954 | # TODO: unlink the big userdata file now, since the emulator |
| 955 | # has it open. |
| 956 | |
| 957 | if not WaitForEmulator(ep, timeout=20): Fail('Emulator did not respond') |
| 958 | if not ReplaceEmulatorPrompt(ep): Fail('Could not replace prompt') |
| 959 | |
| 960 | dest_root = '/data/dexpreopt-root' |
| 961 | if not DexoptEverything(ep, dest_root): Fail('Could not dexopt files') |
| 962 | |
| 963 | # Grab the odex files that were left in dest_root. |
| 964 | cache_system_dir = tmp_dir + '/cache-system' |
| 965 | if not DownloadDirectoryHierarchy(ep, dest_root + '/system', |
| 966 | cache_system_dir, |
| 967 | timeout=20): |
| 968 | Fail('Could not download %s/system from emulator' % dest_root) |
| 969 | |
| 970 | if not InstallCacheFiles(cache_system_dir=cache_system_dir, |
| 971 | out_system_dir=other_args['outsystemdir']): |
| 972 | Fail('Could not install files') |
| 973 | |
| 974 | Trace('dexpreopt successful') |
| 975 | # Success! |
| 976 | |
| 977 | |
| 978 | def main(): |
| 979 | try: |
| 980 | MainInternal() |
| 981 | finally: |
| 982 | KillEmulator() |
| 983 | |
| 984 | |
| 985 | if __name__ == '__main__': |
| 986 | main() |