| import getpass |
| from automation.common import job |
| from automation.common import machine_description |
| import os |
| import re |
| import sys |
| import time |
| from utils import utils |
| |
| depot2_dir = "//depot2/" |
| p4_checkout_dir = "perforce2/" |
| version_dir = "/gcctools/chromeos/v14/" |
| p4_version_dir = p4_checkout_dir + version_dir |
| |
| chromeos_root = "chromeos" |
| scripts_dir = "src/scripts" |
| chromeos_scripts_dir = chromeos_root + "/" + scripts_dir |
| |
| tc_pkgs_dir = "output/pkgs" |
| tc_objects_dir = "output/objects" |
| |
| perflab_binary = "/home/mobiletc-prebuild/perflab-checkout/google3/blaze-bin/platforms/performance/brrd/perflab/perflab" |
| perflab_interpreter_arg = "--perflab_interpreter=/home/mobiletc-prebuild/perflab-checkout/google3/blaze-bin/platforms/performance/brrd/run_tools/experiment_job_dag" |
| perflab_brrd_config_arg = "--brrd_config=/home/mobiletc-prebuild/perflab-checkout/google3/platforms/performance/brrd/perflab/util/perflab.cfg" |
| perflab_options = "--alsologtostderr" |
| perflab_command = ("%s %s %s %s " % |
| (perflab_binary, perflab_interpreter_arg, |
| perflab_brrd_config_arg, perflab_options)) |
| |
| def _GetP4ClientSpec(client_name, p4_paths): |
| p4_string = "" |
| for p4_path in p4_paths: |
| p4_string += " -a \"" + (" //" + client_name + "/").join(p4_path) + "\"" |
| |
| return p4_string |
| |
| |
| def GetP4Command(p4_port, p4_paths, revision, checkoutdir, p4_snapshot=""): |
| command = "" |
| |
| if p4_snapshot: |
| command += "mkdir -p " + checkoutdir |
| for p4_path in p4_paths: |
| real_path = p4_path[1] |
| if real_path.endswith("..."): |
| real_path = real_path.replace("/...", "") |
| command += ("; mkdir -p " + checkoutdir + "/" + |
| os.path.dirname(real_path)) |
| command += ("&& rsync -lr " + p4_snapshot + "/" + real_path + |
| " " + checkoutdir + "/" + os.path.dirname(real_path)) |
| return command |
| |
| client_name = "p4-automation-$HOSTNAME-$JOB_ID" |
| command += " export P4CONFIG=.p4config" |
| command += " && mkdir -p " + checkoutdir |
| command += " && cd " + checkoutdir |
| command += " && cp ${HOME}/.p4config ." |
| command += " && chmod u+w .p4config" |
| command += " && echo \"P4PORT=" + p4_port + "\" >> .p4config" |
| command += " && echo \"P4CLIENT=" + client_name + "\" >> .p4config" |
| command += (" && g4 client " + |
| _GetP4ClientSpec(client_name, p4_paths)) |
| command += " && g4 sync ..." |
| command += " && g4 client -d " + client_name |
| command += " && cd -" |
| return command |
| |
| def CreateLinuxJob(command, lock=False): |
| to_return = job.Job(command) |
| to_return.AddRequiredMachine("", "linux", lock) |
| return to_return |
| |
| def CreateP4Job(p4_port, p4_paths, revision, checkoutdir): |
| to_return = CreateLinuxJob(GetP4Command(p4_port, p4_paths, |
| revision, checkoutdir)) |
| return to_return |
| |
| def _GetChromeOSGoldenBuildLocation(): |
| return "/home/mobiletc-prebuild/www/chromeos_builds/" |
| |
| def GetInitialCommand(): |
| return "pwd && uname -a" |
| |
| def GetCopyTreeCommand(source, dest): |
| command = "" |
| command += "mkdir -p " + dest |
| command += "&& cp -pr " + source + "/* " + dest |
| return command |
| |
| def GetP4VersionDirCommand(p4_snapshot=""): |
| p4_port = "perforce2:2666" |
| p4_paths = [] |
| p4_paths.append(("//depot2/gcctools/chromeos/v14/...", "gcctools/chromeos/v14/...")) |
| p4_revision = 1 |
| command = GetP4Command(p4_port, p4_paths, p4_revision, p4_checkout_dir, p4_snapshot) |
| return command |
| |
| def GetP4BenchmarksDirCommand(p4_snapshot=""): |
| p4_port = "perforce2:2666" |
| p4_paths = [] |
| p4_paths.append(("//depot2/third_party/android_bench/v2_0/...", "gcctools/chromeos/v14/third_party/android_bench/v2_0/...")) |
| p4_revision = 1 |
| command = GetP4Command(p4_port, p4_paths, p4_revision, p4_checkout_dir, p4_snapshot) |
| return command |
| |
| def GetTCRootDir(toolchain="trunk"): |
| if toolchain == "trunk": |
| gcctools_prefix = "" |
| elif toolchain == "v1": |
| gcctools_prefix = "branches/chromeos_toolchain_v1_release_branch/" |
| elif toolchain == "v2": |
| gcctools_prefix = "branches/mobile_toolchain_v14_release_branch/" |
| else: |
| utils.AssertExit(False, "Wrong value for toolchain %s" % toolchain) |
| local_path = p4_checkout_dir + gcctools_prefix + "gcctools/" |
| depot_path = depot2_dir + gcctools_prefix + "gcctools/" |
| return depot_path, local_path |
| |
| def _GetToolchainCheckoutCommand(toolchain="trunk", p4_snapshot=""): |
| p4_port = "perforce2:2666" |
| p4_paths = [] |
| p4_paths.append(("//depot2/gcctools/chromeos/v14/...", "gcctools/chromeos/v14/...")) |
| depot_path, local_path = GetTCRootDir(toolchain) |
| short_local_path = "/".join(local_path.split("/")[1:]) |
| p4_paths.append((depot_path + "google_vendor_src_branch/gcc/gcc-4.4.3/...", |
| short_local_path + "google_vendor_src_branch/gcc/gcc-4.4.3/...")) |
| p4_paths.append((depot_path + "google_vendor_src_branch/binutils/binutils-2.20.1-mobile/...", |
| short_local_path + "google_vendor_src_branch/binutils/binutils-2.20.1-mobile/...")) |
| p4_paths.append((depot_path + "google_vendor_src_branch/binutils/binutils-20100303/...", |
| short_local_path + "google_vendor_src_branch/binutils/binutils-20100303/...")) |
| p4_revision = 1 |
| |
| command = GetP4Command(p4_port, p4_paths, |
| p4_revision, p4_checkout_dir, p4_snapshot) |
| return command |
| |
| |
| |
| def CreateBuildTCJob(chromeos_version="top", |
| board="x86-generic", |
| p4_snapshot="", |
| toolchain="trunk"): |
| |
| depot_path, local_path = GetTCRootDir(toolchain) |
| command = GetInitialCommand() |
| command += "&& " + _GetToolchainCheckoutCommand(toolchain, p4_snapshot) |
| |
| # When g4 syncs this file, often times the timestamp of this file is earlier |
| # than that of the file that is its dependency (ldlex.l). |
| # Since we mount the filesystem as r/o in the build, we cannot regenerate |
| # this file (we also link instead of copy in the 9999 ebuild). |
| # Longer-term, we would want to change the fileattr of this file in g4 |
| # so it syncs the timestamp as well as the file contents. |
| # This is a workaround. |
| command += ("&& touch " + local_path + "google_vendor_src_branch/" + |
| "binutils/binutils-2.20.1-mobile/ld/ldlex.c") |
| |
| command += "&& " + _GetSetupChromeOSCommand(chromeos_version) |
| |
| command += "&& " + _GetBuildTCCommand(toolchain, board, False, True) |
| tc_job = CreateLinuxJob(command) |
| return tc_job |
| |
| def _GetMakeChrootCommand(delete=False): |
| make_chroot_args = "" |
| if delete == True: |
| make_chroot_args = " --delete" |
| command = "cd " + chromeos_scripts_dir |
| command += "&& ./make_chroot --fast " + make_chroot_args |
| command += "&& cd -" |
| return command |
| |
| def CreateDejaGNUJob(chromeos_version="top", |
| board="x86-generic", p4_snapshot="", toolchain="trunk"): |
| local_path = GetTCRootDir(toolchain)[1] |
| command = GetInitialCommand() |
| command += "&& " + _GetToolchainCheckoutCommand(toolchain) |
| command += "&& " + _GetSetupChromeOSCommand(chromeos_version) |
| command += "&& " + _GetBuildTCCommand(toolchain, board) |
| command += ("&& " + p4_version_dir + "/run_dejagnu.py" + |
| " --testflags=\"\"" + |
| " --chromeos_root=chromeos" + |
| " --toolchain_root=" + local_path + |
| " --remote=$SECONDARY_MACHINES[0]" + |
| " --board=" + board) |
| command += ("&& " + p4_version_dir + "/summarize_results.py " + local_path + |
| "/output/dejagnu/gcc.log") |
| command += ("&& " + p4_version_dir + "/summarize_results.py " + local_path + |
| "/output/dejagnu/g++.log") |
| to_return = CreateLinuxJob(command) |
| to_return.AddRequiredMachine("", "chromeos", True, False) |
| return to_return |
| |
| def CreateBuildAndTestChromeOSJob(chromeos_version="latest", |
| board="x86-generic", |
| p4_snapshot="", |
| toolchain="trunk", |
| tests=[]): |
| command = GetInitialCommand() |
| # TODO(asharif): Get rid of this hack at some point. |
| command += "&& mkdir -p perforce2/gcctools/google_vendor_src_branch/gcc" |
| command += "&& " + GetP4VersionDirCommand(p4_snapshot) |
| |
| command += "&& " + _GetSetupChromeOSCommand(chromeos_version) |
| command += "&& " + _GetBuildTCCommand(toolchain, board) |
| command += ("&& " + p4_version_dir + "/build_chromeos.py" + |
| " --chromeos_root=" + chromeos_root + |
| " --board=" + board) |
| |
| command += "&& " + _GetImageChromeOSCommand() |
| |
| autotests = "bvt" + " ".join(tests) |
| command += ("&& " + p4_version_dir + "/run_tests.py" + |
| " --remote=$SECONDARY_MACHINES[0] " + |
| " --chromeos_root=" + chromeos_root + |
| " --board=" + board + |
| " " + autotests) |
| command += ("&& " + p4_version_dir + "/summarize_results.py " + p4_version_dir |
| + "logs/run_tests.py.out") |
| |
| to_return = CreateLinuxJob(command, lock=True) |
| |
| to_return.AddRequiredMachine("", "chromeos", True, False) |
| |
| return to_return |
| |
| def _GetImageChromeOSCommand(): |
| command = (p4_version_dir + "/image_chromeos.py" + |
| " --chromeos_root=chromeos" + |
| " --remote=$SECONDARY_MACHINES[0]") |
| return command |
| |
| def _GetSetupChromeOSCommand(version, use_minilayout=True): |
| version_re = "^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$" |
| tarred_re = "(bz2|gz)$" |
| if version == "weekly" or version == "quarterly": |
| location = _GetChromeOSGoldenBuildLocation() + "/" + version |
| utils.AssertExit(os.path.islink(location) == True, |
| "Symlink: " + location + " does not exist.") |
| location_expanded = os.path.realpath(location) |
| version = utils.GetRoot(location_expanded)[1] |
| if (version == "top" or version == "latest" or |
| re.match(version_re, version)): |
| chromeos_version = version |
| elif re.search(tarred_re, version): |
| command = "mkdir " + chromeos_root |
| command += "&& tar xf " + location_expanded + " -C " + chromeos_root |
| return command |
| else: |
| signature_file = "/src/scripts/enter_chroot.sh" |
| signature_file_location = ("/home/mobiletc-prebuild/www/chromeos_builds/" |
| + version + signature_file) |
| utils.AssertExit(os.path.exists(signature_file_location)) |
| command += "rsync -a " + version + "/ chromeos/" |
| return command |
| |
| command = (p4_version_dir + "/setup_chromeos.py" + |
| " --dir=" + chromeos_root + |
| " --version=" + chromeos_version) |
| if use_minilayout == True: |
| command += " --minilayout" |
| return command |
| |
| def _GetBuildTCCommand(toolchain, board, use_binary=True, rebuild=False): |
| local_path = GetTCRootDir(toolchain)[1] |
| command = (p4_version_dir + "/build_tc.py" + |
| " --toolchain_root=" + local_path + |
| " --chromeos_root=" + chromeos_root + |
| " --board=" + board) |
| if use_binary: |
| command += " -B" |
| return command |
| |
| def CreatePerflabJob(chromeos_version, benchmark, board="x86-agz", |
| p4_snapshot="", toolchain="trunk"): |
| toolchain_root = GetTCRootDir("trunk")[1] |
| command = GetInitialCommand() |
| command += "&& " + GetP4VersionDirCommand(p4_snapshot) |
| command += "&& " + GetP4BenchmarksDirCommand(p4_snapshot) |
| |
| command += "&& " + _GetSetupChromeOSCommand(chromeos_version) |
| command += "&& " + _GetBuildTCCommand(toolchain, board) |
| command += ("&& %s --crosstool=$PWD/%s --chromeos_root=$PWD/%s" |
| " --machines=chromeos_x86-agz_1 build %s" % |
| (perflab_command, toolchain_root, chromeos_root, benchmark)) |
| command += ("&& %s --crosstool=$PWD/%s --chromeos_root=$PWD/%s" |
| " --machines=chromeos_x86-agz_1 run %s" % |
| (perflab_command, toolchain_root, chromeos_root, benchmark)) |
| for b in benchmark.split(","): |
| benchmark_log = b.replace("/", "__") |
| benchmark_log += "/results.txt" |
| benchmark_log_path = ("perflab-output/*/*/chromeos_%s/*/*/%s" % |
| (board, |
| benchmark_log)) |
| |
| command += ("&& " + p4_version_dir + "/summarize_results.py " + |
| benchmark_log_path) |
| to_return = CreateLinuxJob(command, lock=True) |
| return to_return |
| |
| |
| def CreateUpdateJob(chromeos_versions, |
| create_image=True, |
| p4_snapshot="", |
| boards="x86-generic"): |
| command = GetInitialCommand() |
| command += "&& " + GetP4VersionDirCommand(p4_snapshot) |
| command += ("&& " + p4_version_dir + "/setup_chromeos.py" + |
| " --dir=" + chromeos_root + |
| " --version=latest") |
| board_list = boards.split(",") |
| for board in board_list: |
| command += ("&& " + p4_version_dir + "/build_chromeos.py" + |
| " --chromeos_root=" + chromeos_root + |
| " --vanilla --board=" + board) |
| |
| dirname = "$(cd chromeos/src/scripts; git describe --tags --always HEAD)" |
| build_location = _GetChromeOSGoldenBuildLocation() + "/" + dirname |
| for board in board_list: |
| board_build = build_location + "/" + board |
| command += "&& mkdir -p " + board_build |
| command += ("&& rsync -a chromeos/src/build/images/" + board + "/" + |
| " " + board_build + "/") |
| |
| for chromeos_version in chromeos_versions.split(","): |
| build_link = chromeos_version |
| command += ("&& ln -fs -T " + dirname + " " + |
| _GetChromeOSGoldenBuildLocation() + chromeos_version) |
| to_return = CreateLinuxJob(command) |
| return to_return |
| |