| #!/usr/bin/env python |
| # |
| # Copyright (C) 2018 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """ |
| Generate API lists for non-SDK API enforcement. |
| |
| usage: generate-hiddenapi-lists.py [-h] |
| --input-public INPUT_PUBLIC |
| --input-private INPUT_PRIVATE |
| [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]] |
| [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]] |
| [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]] |
| --output-whitelist OUTPUT_WHITELIST |
| --output-light-greylist OUTPUT_LIGHT_GREYLIST |
| --output-dark-greylist OUTPUT_DARK_GREYLIST |
| --output-blacklist OUTPUT_BLACKLIST |
| """ |
| import argparse |
| import os |
| import sys |
| import re |
| |
| def get_args(): |
| """Parses command line arguments. |
| |
| Returns: |
| Namespace: dictionary of parsed arguments |
| """ |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--input-public', required=True, help='List of all public members') |
| parser.add_argument('--input-private', required=True, help='List of all private members') |
| parser.add_argument( |
| '--input-whitelists', nargs='*', |
| help='Lists of members to force on whitelist') |
| parser.add_argument( |
| '--input-greylists', nargs='*', |
| help='Lists of members to force on light greylist') |
| parser.add_argument( |
| '--input-blacklists', nargs='*', |
| help='Lists of members to force on blacklist') |
| parser.add_argument('--output-whitelist', required=True) |
| parser.add_argument('--output-light-greylist', required=True) |
| parser.add_argument('--output-dark-greylist', required=True) |
| parser.add_argument('--output-blacklist', required=True) |
| return parser.parse_args() |
| |
| def read_lines(filename): |
| """Reads entire file and return it as a list of lines. |
| |
| Lines which begin with a hash are ignored. |
| |
| Args: |
| filename (string): Path to the file to read from. |
| |
| Returns: |
| list: Lines of the loaded file as a list of strings. |
| """ |
| with open(filename, 'r') as f: |
| return filter(lambda line: not line.startswith('#'), f.readlines()) |
| |
| def write_lines(filename, lines): |
| """Writes list of lines into a file, overwriting the file it it exists. |
| |
| Args: |
| filename (string): Path to the file to be writting into. |
| lines (list): List of strings to write into the file. |
| """ |
| with open(filename, 'w') as f: |
| f.writelines(lines) |
| |
| def move_between_sets(subset, src, dst, source = "<unknown>"): |
| """Removes a subset of elements from one set and add it to another. |
| |
| Args: |
| subset (set): The subset of `src` to be moved from `src` to `dst`. |
| src (set): Source set. Must be a superset of `subset`. |
| dst (set): Destination set. Must be disjoint with `subset`. |
| """ |
| assert src.issuperset(subset), ( |
| "Error processing: {}\n" |
| "The following entries were not found:\n" |
| "{}" |
| "Please visit go/hiddenapi for more information.").format( |
| source, "".join(map(lambda x: " " + str(x), subset.difference(src)))) |
| assert dst.isdisjoint(subset) |
| # Order matters if `src` and `subset` are the same object. |
| dst.update(subset) |
| src.difference_update(subset) |
| |
| def get_package_name(signature): |
| """Returns the package name prefix of a class member signature. |
| |
| Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/" |
| |
| Args: |
| signature (string): Member signature |
| |
| Returns |
| string: Package name of the given member |
| """ |
| class_name_end = signature.find("->") |
| assert class_name_end != -1, "Invalid signature: {}".format(signature) |
| package_name_end = signature.rfind("/", 0, class_name_end) |
| assert package_name_end != -1, "Invalid signature: {}".format(signature) |
| return signature[:package_name_end + 1] |
| |
| def all_package_names(*args): |
| """Returns a set of packages names in given lists of member signatures. |
| |
| Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]), |
| set([ "Lpkg1/ClassC;->baz()Z" ]) ] |
| return value = set([ "Lpkg1/", "Lpkg2" ]) |
| |
| Args: |
| *args (list): List of sets to iterate over and extract the package names |
| of its elements (member signatures) |
| |
| Returns: |
| set: All package names extracted from the given lists of signatures. |
| """ |
| packages = set() |
| for arg in args: |
| packages = packages.union(map(get_package_name, arg)) |
| return packages |
| |
| def move_all(src, dst): |
| """Moves all elements of one set to another. |
| |
| Args: |
| src (set): Source set. Will become empty. |
| dst (set): Destination set. Will contain all elements of `src`. |
| """ |
| move_between_sets(src, src, dst) |
| |
| def move_from_files(filenames, src, dst): |
| """Loads member signatures from a list of files and moves them to a given set. |
| |
| Opens files in `filenames`, reads all their lines and moves those from `src` |
| set to `dst` set. |
| |
| Args: |
| filenames (list): List of paths to files to be loaded. |
| src (set): Set that loaded lines should be moved from. |
| dst (set): Set that loaded lines should be moved to. |
| """ |
| if filenames: |
| for filename in filenames: |
| move_between_sets(set(read_lines(filename)), src, dst, filename) |
| |
| def move_serialization(src, dst): |
| """Moves all members matching serialization API signatures between given sets. |
| |
| Args: |
| src (set): Set that will be searched for serialization API and that API |
| will be removed from it. |
| dst (set): Set that serialization API will be moved to. |
| """ |
| serialization_patterns = [ |
| r'readObject\(Ljava/io/ObjectInputStream;\)V', |
| r'readObjectNoData\(\)V', |
| r'readResolve\(\)Ljava/lang/Object;', |
| r'serialVersionUID:J', |
| r'serialPersistentFields:\[Ljava/io/ObjectStreamField;', |
| r'writeObject\(Ljava/io/ObjectOutputStream;\)V', |
| r'writeReplace\(\)Ljava/lang/Object;', |
| ] |
| regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$') |
| move_between_sets(filter(lambda api: regex.match(api), src), src, dst) |
| |
| def move_from_packages(packages, src, dst): |
| """Moves all members of given package names from one set to another. |
| |
| Args: |
| packages (list): List of string package names. |
| src (set): Set that will be searched for API matching one of the given |
| package names. Surch API will be removed from the set. |
| dst (set): Set that matching API will be moved to. |
| """ |
| move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst) |
| |
| def main(argv): |
| args = get_args() |
| |
| # Initialize API sets by loading lists of public and private API. Public API |
| # are all members resolvable from SDK API stubs, other members are private. |
| # As an optimization, skip the step of moving public API from a full set of |
| # members and start with a populated whitelist. |
| whitelist = set(read_lines(args.input_public)) |
| uncategorized = set(read_lines(args.input_private)) |
| light_greylist = set() |
| dark_greylist = set() |
| blacklist = set() |
| |
| # Assert that there is no overlap between public and private API. |
| assert whitelist.isdisjoint(uncategorized) |
| num_all_api = len(whitelist) + len(uncategorized) |
| |
| # Read all files which manually assign members to specific lists. |
| move_from_files(args.input_whitelists, uncategorized, whitelist) |
| move_from_files(args.input_greylists, uncategorized, light_greylist) |
| move_from_files(args.input_blacklists, uncategorized, blacklist) |
| |
| # Iterate over all uncategorized members and move serialization API to whitelist. |
| move_serialization(uncategorized, whitelist) |
| |
| # Extract package names of members from whitelist and light greylist, which |
| # are assumed to have been finalized at this point. Assign all uncategorized |
| # members from the same packages to the dark greylist. |
| dark_greylist_packages = all_package_names(whitelist, light_greylist) |
| move_from_packages(dark_greylist_packages, uncategorized, dark_greylist) |
| |
| # Assign all uncategorized members to the blacklist. |
| move_all(uncategorized, blacklist) |
| |
| # Assert we have not missed anything. |
| assert whitelist.isdisjoint(light_greylist) |
| assert whitelist.isdisjoint(dark_greylist) |
| assert whitelist.isdisjoint(blacklist) |
| assert light_greylist.isdisjoint(dark_greylist) |
| assert light_greylist.isdisjoint(blacklist) |
| assert dark_greylist.isdisjoint(blacklist) |
| assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist) |
| |
| # Write final lists to disk. |
| write_lines(args.output_whitelist, whitelist) |
| write_lines(args.output_light_greylist, light_greylist) |
| write_lines(args.output_dark_greylist, dark_greylist) |
| write_lines(args.output_blacklist, blacklist) |
| |
| if __name__ == "__main__": |
| main(sys.argv) |