Port CompareUpstreams tool to Java and improve.
This version runs ~ 50x faster than the .py version, is configured via
environment variables rather than command line parameters, and has
additional features:
* Summarize change comments that occur in each file
* Add support for OpenJDK 9 upstreams.
* Know expected upstream for each file (hard coded), currently 9b113+
for java.util.concurrent and parts of java.util, but 8u121-b13 for
everything else.
* Add functionality to copy files from each upstream
(rather than just 8u121-b13) to $OJLUNI_UPSTREAMS
* As a side effect, this fixes the counting of line numbers: If a file
ends with a newline, Python counts that as an additional (empty) line,
whereas the Java version counts the newline character as belonging to
the line that it terminates; Java's interpretation matches that of
unix tools such as wc -l.
Bug: 35910877
Test: make libcore-compare-upstreams && \
java -jar out/host/linux-x86/framework/libcore-compare-upstreams.jar
Test: make libcore-copy-upstream-files && \
java -jar out/host/linux-x86/framework/libcore-copy-upstream-files.jar \
/tmp/upstreams
Change-Id: I1604b4c4430fe032250f694b7db42456c7653d64
diff --git a/tools/upstream/Android.mk b/tools/upstream/Android.mk
new file mode 100644
index 0000000..bc56f5d
--- /dev/null
+++ b/tools/upstream/Android.mk
@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+LOCAL_PATH := $(call my-dir)
+
+##############################################################
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := libcore-compare-upstreams
+LOCAL_SRC_FILES := $(call all-java-files-under, src/main/java)
+LOCAL_JAR_MANIFEST := src/main/libcore-compare-upstreams.mf
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+##############################################################
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := libcore-copy-upstream-files
+LOCAL_SRC_FILES := $(call all-java-files-under, src/main/java)
+LOCAL_JAR_MANIFEST := src/main/libcore-copy-upstream-files.mf
+include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/tools/upstream/oj_upstream_comparison.py b/tools/upstream/oj_upstream_comparison.py
deleted file mode 100755
index 76d63aa..0000000
--- a/tools/upstream/oj_upstream_comparison.py
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2017 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helps compare openjdk_java_files contents against upstream file contents.
-
-Outputs a tab-separated table comparing each openjdk_java_files entry
-against OpenJDK upstreams. This can help verify updates to later upstreams
-or focus attention towards files that may have been missed in a previous
-update (http://b/36461944) or are otherwise surprising (http://b/36429512).
-
- - Identifies each file as identical to, different from or missing from
- each upstream; diffs are not produced.
- - Optionally, copies all openjdk_java_files from the default upstream
- (eg. OpenJDK8u121-b13) to a new directory, for easy directory comparison
- using e.g. kdiff3, which allows inspecting detailed diffs.
- - The ANDROID_BUILD_TOP environment variable must be set to point to the
- AOSP root directory (parent of libcore).
- - Run with -h command line argument to get usage instructions.
-
-To check out upstreams OpenJDK 7u40, 8u60 and 8u121-b13, run:
-
-mkdir openjdk
-cd openjdk
-hg clone http://hg.openjdk.java.net/jdk7u/jdk7u40/ 7u40
-(cd !$ ; sh get_source.sh)
-hg clone http://hg.openjdk.java.net/jdk8u/jdk8u 8u121-b13
-(cd !$ ; hg update -r jdk8u121-b13 && sh get_source.sh)
-hg clone http://hg.openjdk.java.net/jdk8u/jdk8u60/ 8u60
-(cd !$ ; sh get_source.sh)
-
-The newly created openjdk directory is then a suitable argument for the
---upstream_root parameter.
-"""
-
-import argparse
-import csv
-import filecmp
-import os
-import re
-import shutil
-import sys
-
-def rel_paths_from_makefile(build_top):
- """Returns the list of relative paths to .java files parsed from openjdk_java_files.mk"""
- list_file = os.path.join(build_top, "libcore", "openjdk_java_files.mk")
-
- result = []
- with open(list_file, "r") as f:
- for line in f:
- match = re.match("\s+ojluni/src/main/java/(.+\.java)\s*\\\s*", line)
- if match:
- path = match.group(1)
- # convert / to the appropriate separator (e.g. \ on Windows), just in case
- path = os.path.normpath(path)
- result.append(path)
- return result
-
-def ojluni_path(build_top, rel_path):
- """The full path of the file at the given rel_path in ojluni"""
- return os.path.join(build_top, "libcore", "ojluni", "src", "main", "java", rel_path)
-
-def upstream_path(upstream_root, upstream, rel_path):
- """The full path of the file at the given rel_path in the given upstream"""
- source_dirs = [
- "jdk/src/share/classes",
- "jdk/src/solaris/classes"
- ]
- for source_dir in source_dirs:
- source_dir = os.path.normpath(source_dir)
- result = os.path.join(upstream_root, upstream, source_dir, rel_path)
- if os.path.exists(result):
- return result
- return None
-
-# For lists of length N and M, respectively, this runs in time O(N*M).
-# This could be improved to O(D*(N+M)) for lists with distance <= D by
-# only considering array elements within D cells of the diagonal.
-def edit_distance(a, b):
- """
- Computes the line-based edit distance between two lists, i.e.
- the smallest number of list items to delete, insert or replace
- that would transform the content of one list into the other.
- """
- prev_cost = range(0, len(b) + 1)
- for end_a in range(1, len(a) + 1):
- # For each valid index i, prev_cost[i] is the edit distance between
- # a[:end_a-1] and b[:i].
- # We now calculate cur_cost[end_b] as the edit distance between
- # a[:end_a] and b[:end_b]
- cur_cost = [end_a]
- for end_b in range(1, len(b) + 1):
- c = min(
- cur_cost[-1] + 1, # append item from b
- prev_cost[end_b] + 1, # append item from a
- # match or replace item
- prev_cost[end_b - 1] + (0 if a[end_a - 1] == b[end_b - 1] else 1)
- )
- cur_cost.append(c)
- prev_cost = cur_cost
- return prev_cost[-1]
-
-def compare_to_upstreams_and_save(out_file, build_top, upstream_root, upstreams, rel_paths):
- """
- Prints tab-separated values comparing ojluni files vs. each
- upstream, for each of the rel_paths, suitable for human
- analysis in a spreadsheet.
- This includes whether the corresponding upstream file is
- missing, identical, or by how many lines it differs, and
- a guess as to the correct upstream based on minimal line
- difference (ties broken in favor of upstreams that occur
- earlier in the list).
- """
- writer = csv.writer(out_file, delimiter='\t')
- writer.writerow(["rel_path", "guessed_upstream"] + upstreams)
- for rel_path in rel_paths:
- ojluni_file = ojluni_path(build_top, rel_path)
- upstream_comparisons = []
- best_distance = sys.maxint
- guessed_upstream = ""
- for upstream in upstreams:
- upstream_file = upstream_path(upstream_root, upstream, rel_path)
- if upstream_file is None:
- upstream_comparison = "missing"
- else:
- if filecmp.cmp(upstream_file, ojluni_file, shallow=False):
- distance = 0
- upstream_comparison = "identical"
- else:
- with open(upstream_file) as f:
- lines_a = f.readlines()
- with open(ojluni_file) as f:
- lines_b = f.readlines()
- distance = edit_distance(lines_a, lines_b)
- # 0% for identical files
- # 100% for totally different files or where one file is empty
- percent_different = 100.0 * distance / max(len(lines_a), len(lines_b))
- upstream_comparison = "%.1f%% different (%d lines)" % (percent_different, distance)
- if distance < best_distance:
- best_distance = distance
- guessed_upstream = upstream
- upstream_comparisons.append(upstream_comparison)
- writer.writerow([rel_path, guessed_upstream ] + upstream_comparisons)
-
-def copy_files(rel_paths, upstream_root, upstream, output_dir):
- """Copies files at the given rel_paths from upstream to output_dir"""
- for rel_path in rel_paths:
- upstream_file = upstream_path(upstream_root, upstream, rel_path)
- if upstream_file is not None:
- out_file = os.path.join(output_dir, rel_path)
- out_dir = os.path.dirname(out_file)
- if not os.path.exists(out_dir):
- os.makedirs(out_dir)
- shutil.copyfile(upstream_file, out_file)
-
-def main():
- parser = argparse.ArgumentParser(
- description="Check openjdk_java_files contents against upstream file contents.")
- parser.add_argument("--upstream_root",
- help="Path below where upstream sources are checked out. This should be a "
- "directory with one child directory for each upstream (select the "
- "upstreams to compare against via --upstreams).",
- required=True,)
- parser.add_argument("--upstreams",
- default="8u121-b13,8u60,7u40",
- help="Comma separated list of subdirectory names of --upstream_root that "
- "each hold one upstream.")
- parser.add_argument("--output_dir",
- help="(optional) path where default upstream sources should be copied to; "
- "this path must not yet exist and will be created. "
- "The default upstream is the one that occurs first in --upstreams.")
- parser.add_argument("--build_top",
- default=os.environ.get('ANDROID_BUILD_TOP'),
- help="Path where Android sources are checked out (defaults to $ANDROID_BUILD_TOP).")
- args = parser.parse_args()
- if args.output_dir is not None and os.path.exists(args.output_dir):
- raise Exception("Output dir already exists: " + args.output_dir)
-
- upstreams = [upstream.strip() for upstream in args.upstreams.split(',')]
- default_upstream = upstreams[0]
- for upstream in upstreams:
- upstream_path = os.path.join(args.upstream_root, upstream)
- if not os.path.exists(upstream_path):
- raise Exception("Upstream not found: " + upstream_path)
-
- rel_paths = rel_paths_from_makefile(args.build_top)
-
- compare_to_upstreams_and_save(
- sys.stdout, args.build_top, args.upstream_root, upstreams, rel_paths)
-
- if args.output_dir is not None:
- copy_files(rel_paths, args.upstream_root, default_upstream, args.output_dir)
-
-if __name__ == '__main__':
- main()
diff --git a/tools/upstream/src/main/java/libcore/CompareUpstreams.java b/tools/upstream/src/main/java/libcore/CompareUpstreams.java
new file mode 100644
index 0000000..bd80b82
--- /dev/null
+++ b/tools/upstream/src/main/java/libcore/CompareUpstreams.java
@@ -0,0 +1,260 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package libcore;
+
+import java.io.*;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Helps compare openjdk_java_files contents against upstream file contents.
+ *
+ * Outputs a tab-separated table comparing each openjdk_java_files entry
+ * against OpenJDK upstreams. This can help verify updates to later upstreams
+ * or focus attention towards files that may have been missed in a previous
+ * update (http://b/36461944) or are otherwise surprising (http://b/36429512).
+ *
+ * - Identifies each file as identical to, different from or missing from
+ * each upstream; diffs are not produced.
+ * - Optionally, copies all openjdk_java_files from the default upstream
+ * (eg. OpenJDK8u121-b13) to a new directory, for easy directory comparison
+ * using e.g. kdiff3, which allows inspecting detailed diffs.
+ * - The ANDROID_BUILD_TOP environment variable must be set to point to the
+ * AOSP root directory (parent of libcore).
+ *
+ * To check out upstreams OpenJDK 7u40, 8u60 and 8u121-b13, run:
+ *
+ * mkdir ~/openjdk
+ * cd ~/openjdk
+ * export OPENJDK_HOME=$PWD
+ * hg clone http://hg.openjdk.java.net/jdk7u/jdk7u40/ 7u40
+ * (cd !$ ; sh get_source.sh)
+ * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u 8u121-b13
+ * (cd !$ ; hg update -r jdk8u121-b13 && sh get_source.sh)
+ * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u60/ 8u60
+ * (cd !$ ; sh get_source.sh)
+ *
+ * To get the 9b113+ upstream, follow the instructions from the commit
+ * message of AOSP libcore commit 29957558cf0db700bfaae360a80c42dc3871d0e5
+ * at https://android-review.googlesource.com/c/304056/
+ */
+public class CompareUpstreams {
+
+ private final StandardRepositories standardRepositories;
+
+ public CompareUpstreams(StandardRepositories standardRepositories) {
+ this.standardRepositories = Objects.requireNonNull(standardRepositories);
+ }
+
+ private static Map<String, Integer> androidChangedComments(List<String> lines) {
+ List<String> problems = new ArrayList<>();
+ Map<String, Integer> result = new LinkedHashMap<>();
+ Pattern pattern = Pattern.compile(
+ "// (BEGIN |END |)Android-((?:changed|added|removed|note)(?:: )?.*)$");
+ for (String line : lines) {
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ String type = matcher.group(1);
+ if (type.equals("END")) {
+ continue;
+ }
+ String match = matcher.group(2);
+ if (match.isEmpty()) {
+ match = "[empty comment]";
+ }
+ Integer oldCount = result.get(match);
+ if (oldCount == null) {
+ oldCount = 0;
+ }
+ result.put(match, oldCount + 1);
+ } else if (line.contains("Android-")) {
+ problems.add(line);
+ }
+ }
+ if (!problems.isEmpty()) {
+ throw new IllegalArgumentException(problems.toString());
+ }
+ return result;
+ }
+
+ private static String androidChangedCommentsSummary(List<String> lines) {
+ Map<String, Integer> map = androidChangedComments(lines);
+ List<String> comments = new ArrayList<>(map.keySet());
+ Collections.sort(comments, Comparator.comparing(map::get).reversed());
+ List<String> result = new ArrayList<>();
+ for (String comment : comments) {
+ int count = map.get(comment);
+ if (count == 1) {
+ result.add(comment);
+ } else {
+ result.add(comment + " (x" + count + ")");
+ }
+ }
+ return escapeTsv(String.join("\n", result));
+ }
+
+ /**
+ * Computes the edit distance of two lists, i.e. the smallest number of list items to delete,
+ * insert or replace that would transform the content of one list into the other.
+ */
+ private <T> int editDistance(List<T> a, List<T> b) {
+ int numB = b.size();
+ int[] prevCost = new int[numB + 1];
+ for (int i = 0; i <= numB; i++) {
+ prevCost[i] = i;
+ }
+ int[] curCost = new int[numB + 1];
+ for (int endA = 1; endA <= a.size(); endA++) {
+ // For each valid index i, prevCost[i] is the edit distance between
+ // a.subList(0, endA-1) and b.sublist(0, i).
+ // We now calculate curCost[end_b] as the edit distance between
+ // a.subList(0, endA) and b.subList(0, endB)
+ curCost[0] = endA;
+ for (int endB = 1; endB <= numB; endB++) {
+ boolean endsMatch = a.get(endA - 1).equals(b.get(endB - 1));
+ curCost[endB] = min(
+ curCost[endB - 1] + 1, // append item from b
+ prevCost[endB] + 1, // append item from a
+ prevCost[endB - 1] + (endsMatch ? 0 : 1)); // match or replace item
+ }
+ int[] tmp = curCost;
+ curCost = prevCost;
+ prevCost = tmp;
+ }
+ return prevCost[numB];
+ }
+
+ private static int min(int a, int b, int c) {
+ if (a < b) {
+ return a < c ? a : c;
+ } else {
+ return b < c ? b : c;
+ }
+ }
+
+ private static String escapeTsv(String value) {
+ if (value.contains("\t")) {
+ throw new IllegalArgumentException(value); // tsv doesn't support escaping tabs
+ }
+ return "\"" + value.replace("\"", "\"\"") + "\"";
+ }
+
+ private static void printTsv(PrintStream out, List<String> values) {
+ out.println(String.join("\t", values));
+ }
+
+ /**
+ * Prints tab-separated values comparing ojluni files vs. each
+ * upstream, for each of the rel_paths, suitable for human
+ * analysis in a spreadsheet.
+ * This includes whether the corresponding upstream file is
+ * missing, identical, or by how many lines it differs, and
+ * a guess as to the correct upstream based on minimal line
+ * difference (ties broken in favor of upstreams that occur
+ * earlier in the list).
+ */
+ private void run(PrintStream out, List<Path> relPaths) throws IOException {
+ // upstreams are in decreasing order of preference
+ List<String> headers = new ArrayList<>();
+ headers.addAll(Arrays.asList(
+ "rel_path", "expected_upstream", "guessed_upstream", "changes", "vs. expected"));
+ for (Repository upstream : standardRepositories.historicUpstreams()) {
+ headers.add(upstream.name());
+ }
+ headers.add("diff");
+ printTsv(out, headers);
+ for (Path relPath : relPaths) {
+ Repository expectedUpstream = standardRepositories.currentUpstream(relPath);
+ out.print(relPath + "\t");
+ Path ojluniFile = standardRepositories.ojluni().absolutePath(relPath);
+ List<String> linesB = Util.readLines(ojluniFile);
+ int bestDistance = Integer.MAX_VALUE;
+ Repository guessedUpstream = null;
+ List<Repository> upstreams = new ArrayList<>();
+ upstreams.add(expectedUpstream);
+ upstreams.addAll(standardRepositories.historicUpstreams());
+ List<String> comparisons = new ArrayList<>(upstreams.size());
+ for (Repository upstream : upstreams) {
+ final String comparison;
+ Path upstreamFile = upstream.absolutePath(relPath);
+ if (upstreamFile == null) {
+ comparison = "missing";
+ } else {
+ List<String> linesA = Util.readLines(upstreamFile);
+ int distance = editDistance(linesA, linesB);
+ if (distance == 0) {
+ comparison = "identical";
+ } else {
+ double percentDifferent = 100.0 * distance / Math
+ .max(linesA.size(), linesB.size());
+ comparison = String
+ .format(Locale.US, "%.1f%% different (%d lines)", percentDifferent,
+ distance);
+ }
+ if (distance < bestDistance) {
+ bestDistance = distance;
+ guessedUpstream = upstream;
+ }
+ }
+ comparisons.add(comparison);
+ }
+ String changedCommentsSummary = androidChangedCommentsSummary(linesB);
+
+ String diffCommand = "";
+ if (!comparisons.get(0).equals("identical")) {
+ Path expectedUpstreamPath = expectedUpstream.pathFromRepository(relPath);
+ if (expectedUpstreamPath != null) {
+ diffCommand = String.format(Locale.US, "meld \"%s\" \"%s\"",
+ "${ANDROID_BUILD_TOP}/libcore/"
+ + standardRepositories.ojluni().pathFromRepository(relPath),
+ "${OJLUNI_UPSTREAMS}/" + expectedUpstream.name() + "/" + relPath);
+ //"${OPENJDK_HOME}/" + expectedUpstreamPath;
+ } else {
+ diffCommand = "FILE MISSING";
+ }
+ }
+ List<String> values = new ArrayList<>();
+ values.add(expectedUpstream.name());
+ values.add(guessedUpstream == null ? "" : guessedUpstream.name());
+ values.add(changedCommentsSummary);
+ values.addAll(comparisons);
+ values.add(diffCommand);
+ printTsv(out, values);
+ }
+ }
+
+ public void run() throws IOException {
+ List<Path> relPaths = standardRepositories.ojluni().loadRelPathsFromMakefile();
+ run(System.out, relPaths);
+ }
+
+ public static void main(String[] args) throws IOException {
+ StandardRepositories standardRepositories = StandardRepositories.fromEnv();
+ CompareUpstreams action = new CompareUpstreams(standardRepositories);
+ action.run();
+ }
+}
diff --git a/tools/upstream/src/main/java/libcore/CopyUpstreamFiles.java b/tools/upstream/src/main/java/libcore/CopyUpstreamFiles.java
new file mode 100644
index 0000000..007914f
--- /dev/null
+++ b/tools/upstream/src/main/java/libcore/CopyUpstreamFiles.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package libcore;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class CopyUpstreamFiles {
+
+ private final StandardRepositories standardRepositories;
+ private final Path outputDir;
+
+ private CopyUpstreamFiles(StandardRepositories standardRepositories, Path outputDir) {
+ this.standardRepositories = Objects.requireNonNull(standardRepositories);
+ this.outputDir = Objects.requireNonNull(outputDir);
+ }
+
+ public void run() throws IOException {
+ List<Path> relPaths = standardRepositories.ojluni().loadRelPathsFromMakefile();
+ if (outputDir.toFile().exists()) {
+ throw new IOException(outputDir + " already exists");
+ } else {
+ boolean success = outputDir.toFile().mkdir();
+ if (!success) {
+ throw new IOException("Failed to create directory " + outputDir);
+ }
+ }
+ for (Path relPath : relPaths) {
+ Repository expectedUpstream = standardRepositories.currentUpstream(relPath);
+ for (Repository upstream : standardRepositories.upstreams()) {
+ Path upstreamFile = upstream.absolutePath(relPath);
+ if (upstreamFile != null) {
+ Path outputFile = outputDir
+ .resolve(upstream.name())
+ .resolve(relPath);
+ copyFile(upstreamFile, outputFile);
+ if (upstream.equals(expectedUpstream)) {
+ copyFile(upstreamFile, outputDir.resolve("expected").resolve(relPath));
+ }
+ }
+ }
+ }
+ }
+
+ private void copyFile(Path from, Path to) throws IOException {
+ if (!from.toFile().canRead()) {
+ throw new IOException("Error reading " + from);
+ }
+ Path toDir = to.getParent();
+ if (!toDir.toFile().exists()) {
+ boolean success = toDir.toFile().mkdirs();
+ if (!success) {
+ throw new IOException("Failed to create directory " + toDir);
+ }
+ }
+ Files.copy(from, to);
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1) {
+ throw new IllegalArgumentException(Arrays.asList(args).toString());
+ }
+ Path outputDir = new File(args[0]).toPath();
+ StandardRepositories standardRepositories = StandardRepositories.fromEnv();
+ new CopyUpstreamFiles(standardRepositories, outputDir).run();
+ }
+}
diff --git a/tools/upstream/src/main/java/libcore/Repository.java b/tools/upstream/src/main/java/libcore/Repository.java
new file mode 100644
index 0000000..7aac3e2
--- /dev/null
+++ b/tools/upstream/src/main/java/libcore/Repository.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package libcore;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A set of .java files (either from ojluni or from an upstream).
+ */
+abstract class Repository {
+
+ protected final Path rootPath;
+ protected final String name;
+
+ protected Repository(Path rootPath, String name) {
+ this.rootPath = Objects.requireNonNull(rootPath);
+ this.name = Objects.requireNonNull(name);
+ if (!rootPath.toFile().isDirectory()) {
+ throw new IllegalArgumentException("Missing or not a directory: " + rootPath);
+ }
+ }
+
+ /**
+ * @param relPath a relative path of a .java file in the repository, e.g.
+ * "java/util/ArrayList.java".
+ * @return the path of the indicated file (either absolute, or relative to the current
+ * working directory), or null if the file does not exist in this Repository.
+ */
+ public final Path absolutePath(Path relPath) {
+ Path p = pathFromRepository(relPath);
+ return p == null ? null : rootPath.resolve(p).toAbsolutePath();
+ }
+
+ public abstract Path pathFromRepository(Path relPath);
+
+ /**
+ * @return A human readable name to identify this repository, suitable for use as a
+ * directory name.
+ */
+ public final String name() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name() + " repository";
+ }
+
+ /**
+ * A checkout of the hg repository of OpenJDK 9 or higher, located in the
+ * subdirectory {@code upstreamName} under the directory {@code upstreamRoot}.
+ */
+ public static Repository openJdk9(Path upstreamRoot, String upstreamName) {
+ List<String> sourceDirs = Arrays.asList(
+ "jdk/src/java.base/share/classes",
+ "jdk/src/java.logging/share/classes",
+ "jdk/src/java.prefs/share/classes",
+ "jdk/src/java.sql/share/classes",
+ "jdk/src/java.desktop/share/classes",
+ "jdk/src/java.base/solaris/classes",
+ "jdk/src/java.base/unix/classes",
+ "jdk/src/java.prefs/unix/classes",
+ "jdk/src/jdk.unsupported/share/classes",
+ "jdk/src/jdk.net/share/classes",
+ "jdk/src/java.base/linux/classes"
+ );
+ return new OpenJdkRepository(upstreamRoot, upstreamName, sourceDirs);
+ }
+
+ /**
+ * A checkout of the hg repository of OpenJDK 8 or earlier, located in the
+ * subdirectory {@code upstreamName} under the directory {@code upstreamRoot}.
+ */
+ public static Repository openJdkLegacy(Path upstreamRoot, String upstreamName) {
+ List<String> sourceDirs = Arrays.asList("jdk/src/share/classes", "jdk/src/solaris/classes");
+ return new OpenJdkRepository(upstreamRoot, upstreamName, sourceDirs);
+ }
+
+ /**
+ * Checkouts of hg repositories of OpenJDK 8 or earlier, located in the
+ * respective {@code upstreamNames} subdirectories under the join parent
+ * directory {@code upstreamRoot}.
+ */
+ public static List<Repository> openJdkLegacy(Path upstreamRoot, List<String> upstreamNames) {
+ List<String> sourceDirs = Arrays.asList("jdk/src/share/classes", "jdk/src/solaris/classes");
+ List<Repository> result = new ArrayList<>();
+ for (String upstreamName : upstreamNames) {
+ result.add(new OpenJdkRepository(upstreamRoot, upstreamName, sourceDirs));
+ }
+ return Collections.unmodifiableList(result);
+ }
+
+ static class OjluniRepository extends Repository {
+
+ /**
+ * The repository of ojluni java files belonging to the Android sources under
+ * {@code buildTop}.
+ *
+ * @param buildTop The root path of an Android checkout, as identified by the
+ * {@quote ANDROID_BUILD_TOP} environment variable.
+ */
+ public OjluniRepository(Path buildTop) {
+ super(buildTop.resolve("libcore"), "ojluni");
+ }
+
+
+ @Override
+ public Path pathFromRepository(Path relPath) {
+ return Paths.get("ojluni/src/main/java").resolve(relPath);
+ }
+
+ /**
+ * Returns the list of relative paths to .java files parsed from openjdk_java_files.mk
+ */
+ public List<Path> loadRelPathsFromMakefile() throws IOException {
+ List<Path> result = new ArrayList<>();
+ Path makefile = rootPath.resolve("openjdk_java_files.mk");
+ Pattern pattern = Pattern.compile("\\s+ojluni/src/main/java/(.+\\.java)\\s*\\\\\\s*");
+ for (String line : Util.readLines(makefile)) {
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.matches()) {
+ Path path = new File(matcher.group(1)).toPath();
+ result.add(path);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "libcore ojluni";
+ }
+ }
+
+ static class OpenJdkRepository extends Repository {
+ private final List<String> sourceDirs;
+
+ public OpenJdkRepository(Path upstreamRoot, String name, List<String> sourceDirs) {
+ super(upstreamRoot.resolve(name), name);
+ this.sourceDirs = Objects.requireNonNull(sourceDirs);
+ }
+
+ @Override
+ public Path pathFromRepository(Path relPath) {
+ for (String sourceDir : sourceDirs) {
+ Path repositoryRelativePath = Paths.get(sourceDir).resolve(relPath);
+ Path file = rootPath.resolve(repositoryRelativePath);
+ if (file.toFile().exists()) {
+ return repositoryRelativePath;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "OpenJDK " + name;
+ }
+ }
+
+
+}
diff --git a/tools/upstream/src/main/java/libcore/StandardRepositories.java b/tools/upstream/src/main/java/libcore/StandardRepositories.java
new file mode 100644
index 0000000..36478dd
--- /dev/null
+++ b/tools/upstream/src/main/java/libcore/StandardRepositories.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package libcore;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import libcore.Repository.OjluniRepository;
+
+import static libcore.Repository.openJdk9;
+import static libcore.Repository.openJdkLegacy;
+
+public class StandardRepositories {
+
+ private final List<Repository> historicUpstreams;
+ private final Repository defaultUpstream;
+ private final Repository jsr166Upstream;
+ private final OjluniRepository ojluni;
+
+ private StandardRepositories(Path buildTop, Path upstreamRoot) {
+ this.historicUpstreams = openJdkLegacy(upstreamRoot, Arrays.asList("8u60", "7u40"));
+ this.defaultUpstream = openJdkLegacy(upstreamRoot, "8u121-b13");
+ this.jsr166Upstream = openJdk9(upstreamRoot, "9b113+");
+ this.ojluni = new OjluniRepository(buildTop);
+ }
+
+ public List<Repository> historicUpstreams() {
+ return historicUpstreams;
+ }
+
+ public OjluniRepository ojluni() {
+ return ojluni;
+ }
+
+ /**
+ * Returns all upstream repository snapshots, in order from latest to earliest.
+ */
+ public List<Repository> upstreams() {
+ List<Repository> upstreams = new ArrayList<>(Arrays.asList(
+ jsr166Upstream, defaultUpstream));
+ upstreams.addAll(historicUpstreams);
+ return Collections.unmodifiableList(upstreams);
+ }
+
+ public static StandardRepositories fromEnv() {
+ Path androidBuildTop = Paths.get(getEnvOrThrow("ANDROID_BUILD_TOP"));
+ Path upstreamRoot = Paths.get(getEnvOrThrow("OPENJDK_HOME"));
+ return new StandardRepositories(androidBuildTop, upstreamRoot);
+ }
+
+ private static String getEnvOrThrow(String name) {
+ String result = System.getenv(name);
+ if (result == null) {
+ throw new IllegalStateException("Environment variable undefined: " + name);
+ }
+ return result;
+ }
+
+ private static final Set<String> juFilesFromJsr166 = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList(
+ "AbstractQueue",
+ "ArrayDeque",
+ "ArrayPrefixHelpers",
+ "Deque",
+ "Map",
+ "NavigableMap",
+ "NavigableSet",
+ "PriorityQueue",
+ "Queue",
+ "SplittableRandom"
+ )));
+
+ public Repository currentUpstream(Path relPath) {
+ boolean isJsr166 = relPath.toString().startsWith("java/util/concurrent");
+ String ju = "java/util/";
+ String suffix = ".java";
+ if (!isJsr166 && relPath.startsWith(ju)) {
+ String name = relPath.toString().substring(ju.length());
+ if (name.endsWith(suffix)) {
+ name = name.substring(0, name.length() - suffix.length());
+ isJsr166 = juFilesFromJsr166.contains(name);
+ }
+ }
+ return isJsr166 ? jsr166Upstream : defaultUpstream;
+ }
+
+}
diff --git a/tools/upstream/src/main/java/libcore/Util.java b/tools/upstream/src/main/java/libcore/Util.java
new file mode 100644
index 0000000..d213080
--- /dev/null
+++ b/tools/upstream/src/main/java/libcore/Util.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package libcore;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+class Util {
+ private Util() {
+ }
+
+ public static List<String> readLines(Path path) throws IOException {
+ List<String> result = new ArrayList<>();
+ try (BufferedReader reader = new BufferedReader(new FileReader(path.toFile()))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ result.add(line);
+ }
+ }
+ return result;
+ }
+
+}
diff --git a/tools/upstream/src/main/libcore-compare-upstreams.mf b/tools/upstream/src/main/libcore-compare-upstreams.mf
new file mode 100644
index 0000000..12db91e
--- /dev/null
+++ b/tools/upstream/src/main/libcore-compare-upstreams.mf
@@ -0,0 +1 @@
+Main-Class: libcore.CompareUpstreams
diff --git a/tools/upstream/src/main/libcore-copy-upstream-files.mf b/tools/upstream/src/main/libcore-copy-upstream-files.mf
new file mode 100644
index 0000000..7babdbc
--- /dev/null
+++ b/tools/upstream/src/main/libcore-copy-upstream-files.mf
@@ -0,0 +1 @@
+Main-Class: libcore.CopyUpstreamFiles
diff --git a/tools/upstream/upstream-tool.iml b/tools/upstream/upstream-tool.iml
new file mode 100644
index 0000000..f76df2b
--- /dev/null
+++ b/tools/upstream/upstream-tool.iml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" inherit-compiler-output="true">
+ <exclude-output />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ </component>
+</module>
\ No newline at end of file