| # Copyright (C) 2018 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """Utility for ICU4C code generation""" |
| |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| |
| import logging |
| import os |
| import site |
| import sys |
| import textwrap |
| from collections import deque |
| |
| import jinja2 |
| |
| THIS_DIR = os.path.dirname(os.path.realpath(__file__)) |
| ANDROID_TOP = os.path.realpath(os.path.join(THIS_DIR, '../../../..')) |
| |
| JINJA_ENV = jinja2.Environment(loader=jinja2.FileSystemLoader( |
| os.path.join(THIS_DIR, 'jinja_templates'))) |
| JINJA_ENV.trim_blocks = True |
| JINJA_ENV.lstrip_blocks = True |
| |
| def generate_shim(functions, includes, suffix, template_file): |
| """Generates the library source file from the given functions.""" |
| data = { |
| 'functions': functions, |
| 'icu_headers': includes, |
| 'suffix': suffix, |
| } |
| return JINJA_ENV.get_template(template_file).render(data) |
| |
| def generate_symbol_txt(shim_functions, extra_function_names, template_file): |
| """Generates the symbol txt file from the given functions.""" |
| data = { |
| # Each shim_function is given a suffix. |
| 'shim_functions' : shim_functions, |
| # Each extra function name is included as given. |
| 'extra_function_names': extra_function_names, |
| } |
| return JINJA_ENV.get_template(template_file).render(data) |
| |
| def get_allowlisted_apis(allowlist_file): |
| """Return all allowlisted API in allowlist_file""" |
| allowlisted_apis = set() |
| with open(os.path.join(THIS_DIR, allowlist_file), 'r') as file: |
| for line in file: |
| line = line.strip() |
| if line and not line.startswith("#"): |
| allowlisted_apis.add(line) |
| return allowlisted_apis |
| |
| def android_path(*args): |
| """Returns the absolute path to a directory within the Android tree.""" |
| return os.path.join(ANDROID_TOP, *args) |
| |
| |
| # TODO: Include clang bindings in prebuilt package. http://b/119270767 |
| site.addsitedir(android_path('external/clang/bindings/python')) |
| import clang.cindex # pylint: disable=import-error,wrong-import-position |
| |
| # TODO: Do not hardcode clang version. http://b/119270767 |
| CLANG_REVISION = 'r383902c' |
| CLANG_LIB_VERSION = '11git' |
| CLANG_HEADER_VERSION = '11.0.3' |
| CLANG_PATH = android_path('prebuilts/clang/host/linux-x86/clang-%s' % CLANG_REVISION) |
| |
| |
| class Function: |
| """A visible function found in an ICU header.""" |
| |
| def __init__(self, name, result_type, params, is_variadic, module): |
| self.name = name |
| self.result_type = result_type |
| self.params = params |
| self.is_variadic = is_variadic |
| self.va_list_insert_position = -1 |
| |
| # callee will be used in dlsym and may be identical to others for |
| # functions with variable argument lists. |
| self.callee = self.name |
| if self.is_variadic: |
| self.last_param = self.params[-1][1] |
| self.handle = 'handle_' + module |
| self.return_void = self.result_type == 'void' |
| |
| @property |
| def param_str(self): |
| """Returns a string usable as a parameter list in a function decl.""" |
| params = [] |
| for param_type, param_name in self.params: |
| if '[' in param_type: |
| # `int foo[42]` will be a param_type of `int [42]` and a |
| # param_name of `foo`. We need to put these back in the right |
| # order. |
| param_name += param_type[param_type.find('['):] |
| param_type = param_type[:param_type.find('[')] |
| params.append('{} {}'.format(param_type, param_name)) |
| if self.is_variadic: |
| params.append('...') |
| return ', '.join(params) |
| |
| @property |
| def arg_str(self): |
| """Returns a string usable as an argument list in a function call.""" |
| args = [] |
| for _, param_name in self.params: |
| args.append(param_name) |
| if self.is_variadic: |
| if self.va_list_insert_position >= 0: |
| args.insert(self.va_list_insert_position, 'args') |
| else: |
| raise ValueError(textwrap.dedent("""\ |
| {}({}) is variadic, but has no valid \ |
| inserted position""".format( |
| self.name, |
| self.param_str))) |
| return ', '.join(args) |
| |
| def set_variadic_callee(self, callee, inserted_position): |
| """Set variadic callee with callee name and inserted position""" |
| if self.is_variadic: |
| self.callee = callee |
| self.va_list_insert_position = inserted_position |
| |
| |
| def logger(): |
| """Returns the module level logger.""" |
| return logging.getLogger(__name__) |
| |
| |
| class DeclaredFunctionsParser: |
| """Parser to get declared functions from ICU4C headers. """ |
| |
| def __init__(self, decl_filters, allowlisted_decl_filter): |
| """ |
| Args: |
| decl_filters: A list of filters for declared functions. |
| allowlisted_decl_filter: A list of allowlisting filters for declared functions. |
| If the function is allowlisted here, the function will not filtered by the filter added |
| in decl_filters |
| """ |
| self.decl_filters = decl_filters |
| self.allowlisted_decl_filters = allowlisted_decl_filter |
| self.va_functions_mapping = {} |
| self.ignored_include_dependency = {} |
| |
| # properties to store the parsing result |
| self.all_headers = [] |
| self.all_header_paths_to_copy = set() |
| self.all_declared_functions = [] |
| self.seen_functions = set() |
| self.all_header_to_function_names = {} |
| |
| # Configures libclang to load in our environment |
| # Set up LD_LIBRARY_PATH to include libclang.so, libLLVM.so, etc. Note |
| # that setting LD_LIBRARY_PATH with os.putenv() sometimes doesn't help. |
| # clang.cindex.Config.set_library_path(os.path.join(CLANG_PATH, 'lib64')) |
| clang.cindex.Config.set_library_file( |
| os.path.join(CLANG_PATH, 'lib64', 'libclang.so.%s' % CLANG_LIB_VERSION)) |
| |
| def set_va_functions_mapping(self, mapping): |
| """Set mapping from a variable argument function to an implementation. |
| |
| Functions w/ variable argument lists (...) need special care to call |
| their corresponding v- versions that accept a va_list argument. Note that |
| although '...' will always appear as the last parameter, its v- version |
| may put the va_list arg in a different place. Hence we provide an index |
| to indicate the position. |
| e.g. 'umsg_format': ('umsg_vformat', 3) means in the wrapper function of |
| 'umsg_format', it will call 'umsg_vformat' instead, with the va_list arg |
| inserted as the 3rd argument.""" |
| self.va_functions_mapping = mapping |
| |
| def set_ignored_include_dependency(self, mapping): |
| """ |
| A sample mapping is { "ulocdata.h" : [ "uloc.h", "ures.h" ] }. |
| The include dependencies will explicitly be ignored when producing header_paths_to_copy. |
| """ |
| self.ignored_include_dependency = mapping |
| |
| @property |
| def header_includes(self): |
| """Return all headers declaring the functions returned in get_all_declared_functions. |
| |
| If all functions in the header are filtered, the header is not included in here.""" |
| return [DeclaredFunctionsParser.short_header_path(header) for header in self.all_headers] |
| |
| @property |
| def header_paths_to_copy(self): |
| """Return all headers needed to be copied""" |
| return self.all_header_paths_to_copy |
| |
| @property |
| def declared_functions(self): |
| """Return all declared functions after filtering""" |
| return self.all_declared_functions |
| |
| @property |
| def header_to_function_names(self): |
| """Return the mapping from the header file name to a list of function names in the file""" |
| return self.all_header_to_function_names |
| |
| @staticmethod |
| def get_cflags(): |
| """Returns the cflags that should be used for parsing.""" |
| clang_flags = [ |
| '-x', |
| 'c', |
| '-std=c99', |
| '-DU_DISABLE_RENAMING=1', |
| '-DU_SHOW_CPLUSPLUS_API=0', |
| '-DU_HIDE_DRAFT_API', |
| '-DU_HIDE_DEPRECATED_API', |
| '-DU_HIDE_INTERNAL_API', |
| '-DANDROID_LINK_SHARED_ICU4C', |
| ] |
| |
| include_dirs = [ |
| # TODO: Do not hardcode clang version. http://b/119270767 |
| os.path.join(CLANG_PATH, 'lib64/clang/', CLANG_HEADER_VERSION, 'include/'), |
| android_path('bionic/libc/include'), |
| android_path('external/icu/android_icu4c/include'), |
| android_path('external/icu/icu4c/source/common'), |
| android_path('external/icu/icu4c/source/i18n'), |
| ] |
| |
| for include_dir in include_dirs: |
| clang_flags.append('-I' + include_dir) |
| return clang_flags |
| |
| @staticmethod |
| def get_all_cpp_headers(): |
| """Return all C++ header names in icu4c/source/test/hdrtst/cxxfiles.txt""" |
| cpp_headers = [] |
| with open(android_path('external/icu/tools/icu4c_srcgen/cxxfiles.txt'), 'r') as file: |
| for line in file: |
| line = line.strip() |
| if not line.startswith("#"): |
| cpp_headers.append(line) |
| return cpp_headers |
| |
| def parse(self): |
| """Parse the headers and collect the declared functions after filtering |
| and the headers containing the functions.""" |
| index = clang.cindex.Index.create() |
| |
| icu_modules = ( |
| 'common', |
| 'i18n', |
| ) |
| header_dependencies = {} |
| for module in icu_modules: |
| path = android_path(android_path('external/icu/icu4c/source', module, 'unicode')) |
| files = [os.path.join(path, f) |
| for f in os.listdir(path) if f.endswith('.h')] |
| |
| for file_path in files: |
| base_header_name = os.path.basename(file_path) |
| # Ignore C++ headers. |
| if base_header_name in DeclaredFunctionsParser.get_all_cpp_headers(): |
| continue |
| |
| tunit = index.parse(file_path, DeclaredFunctionsParser.get_cflags()) |
| DeclaredFunctionsParser.handle_diagnostics(tunit) |
| header_dependencies[file_path] = [file_inclusion.include.name for file_inclusion |
| in tunit.get_includes()] |
| visible_functions = self.get_visible_functions( |
| tunit.cursor, module, file_path) |
| self.all_header_to_function_names[base_header_name] = \ |
| [f.name for f in visible_functions] |
| for function in visible_functions: |
| self.seen_functions.add(function.name) |
| self.all_declared_functions.append(function) |
| if visible_functions: |
| self.all_headers.append(file_path) |
| |
| # Sort to produce an deterministic output |
| self.all_declared_functions = sorted(self.all_declared_functions, key=lambda f: f.name) |
| self.all_headers = sorted(self.all_headers) |
| |
| # Build the headers required for using your restricted API set, and put the set into |
| # all_header_files_to_copy. |
| # header_dependencies is a map from icu4c header file path to a list of included headers. |
| # The key must be a ICU4C header, but the value could contain non-ICU4C headers, e.g. |
| # { |
| # ".../icu4c/source/common/unicode/utype.h": [ |
| # ".../icu4c/source/common/unicode/uversion.h", |
| # ".../bionic/libc/include/ctype.h", |
| # ], |
| # ... |
| # } |
| file_queue = deque() |
| file_processed = set() |
| for header in self.all_headers: |
| file_queue.appendleft(header) |
| self.all_header_paths_to_copy.add(header) |
| while file_queue: |
| file = file_queue.pop() |
| file_basename = os.path.basename(file) |
| if file in file_processed: |
| continue |
| file_processed.add(file) |
| for header in header_dependencies[file]: |
| header_basename = os.path.basename(header) |
| # Skip this header if this dependency is explicitly ignored |
| if file_basename in self.ignored_include_dependency and \ |
| header_basename in self.ignored_include_dependency[file_basename]: |
| continue |
| if header in header_dependencies: # Do not care non-icu4c headers |
| self.all_header_paths_to_copy.add(header) |
| file_queue.appendleft(header) |
| |
| @staticmethod |
| def handle_diagnostics(tunit): |
| """Prints compiler diagnostics to stdout. Exits if errors occurred.""" |
| errors = 0 |
| for diag in tunit.diagnostics: |
| if diag.severity == clang.cindex.Diagnostic.Fatal: |
| level = logging.CRITICAL |
| errors += 1 |
| elif diag.severity == clang.cindex.Diagnostic.Error: |
| level = logging.ERROR |
| errors += 1 |
| elif diag.severity == clang.cindex.Diagnostic.Warning: |
| level = logging.WARNING |
| elif diag.severity == clang.cindex.Diagnostic.Note: |
| level = logging.INFO |
| logger().log( |
| level, '%s:%s:%s %s', diag.location.file, diag.location.line, |
| diag.location.column, diag.spelling) |
| if errors: |
| sys.exit('Errors occurred during parsing. Exiting.') |
| |
| def get_visible_functions(self, cursor, module, file_name): |
| """Returns a list of all visible functions in a header file.""" |
| functions = [] |
| for child in cursor.get_children(): |
| if self.should_process_decl(child, file_name): |
| functions.append(self.from_cursor(child, module)) |
| return functions |
| |
| def should_process_decl(self, decl, file_name): |
| """Returns True if this function needs to be processed.""" |
| if decl.kind != clang.cindex.CursorKind.FUNCTION_DECL: |
| return False |
| if decl.location.file.name != file_name: |
| return False |
| if decl.spelling in self.seen_functions: |
| return False |
| if not DeclaredFunctionsParser.is_function_visible(decl): |
| return False |
| for allowlisted_decl_filter in self.allowlisted_decl_filters: |
| if allowlisted_decl_filter(decl): |
| return True |
| for decl_filter in self.decl_filters: |
| if not decl_filter(decl): |
| return False |
| return True |
| |
| @staticmethod |
| def is_function_visible(decl): |
| """Returns True if the function has default visibility.""" |
| visible = False |
| vis_attrs = DeclaredFunctionsParser.get_children_by_kind( |
| decl, clang.cindex.CursorKind.VISIBILITY_ATTR) |
| for child in vis_attrs: |
| visible = child.spelling == 'default' |
| return visible |
| |
| @staticmethod |
| def get_children_by_kind(cursor, kind): |
| """Returns a generator of cursor's children of a specific kind.""" |
| for child in cursor.get_children(): |
| if child.kind == kind: |
| yield child |
| |
| @staticmethod |
| def short_header_path(name): |
| """Trim the given file name to 'unicode/xyz.h'.""" |
| return name[name.rfind('unicode/'):] |
| |
| def from_cursor(self, cursor, module): |
| """Creates a Function object from the decl at the cursor.""" |
| if cursor.type.kind != clang.cindex.TypeKind.FUNCTIONPROTO: |
| raise ValueError(textwrap.dedent("""\ |
| {}'s type kind is {}, expected TypeKind.FUNCTIONPROTO. |
| {} Line {} Column {}""".format( |
| cursor.spelling, |
| cursor.type.kind, |
| cursor.location.file, |
| cursor.location.line, |
| cursor.location.column))) |
| |
| name = cursor.spelling |
| result_type = cursor.result_type.spelling |
| is_variadic = cursor.type.is_function_variadic() |
| params = [] |
| for arg in cursor.get_arguments(): |
| params.append((arg.type.spelling, arg.spelling)) |
| function = Function(name, result_type, params, is_variadic, module) |
| # For variadic function, set the callee and va_list position |
| if function.is_variadic and function.name in self.va_functions_mapping: |
| va_func = self.va_functions_mapping[function.name] |
| function.set_variadic_callee(va_func[0], va_func[1]) |
| return function |
| |
| |
| class StableDeclarationFilter: |
| """Return true if it's @stable API""" |
| def __call__(self, decl): |
| """Returns True if the given decl has a doxygen stable tag.""" |
| if not decl.raw_comment: |
| return False |
| if '@stable' in decl.raw_comment: |
| return True |
| return False |
| |
| |
| class AllowlistedDeclarationFilter: |
| """A filter for allowlisting function declarations.""" |
| def __init__(self, allowlisted_function_names): |
| self.allowlisted_function_names = allowlisted_function_names |
| |
| def __call__(self, decl): |
| """Returns True if the given decl is allowlisted""" |
| return decl.spelling in self.allowlisted_function_names |
| |
| |
| class BlocklistedlistedDeclarationFilter: |
| """A filter for blocklisting function declarations.""" |
| def __init__(self, blocklisted_function_names): |
| self.blocklisted_function_names = blocklisted_function_names |
| |
| def __call__(self, decl): |
| """Returns True if the given decl is nor blocklisted""" |
| return decl.spelling not in self.blocklisted_function_names |
| |
| |
| # Functions w/ variable argument lists (...) need special care to call |
| # their corresponding v- versions that accept a va_list argument. Note that |
| # although '...' will always appear as the last parameter, its v- version |
| # may put the va_list arg in a different place. Hence we provide an index |
| # to indicate the position. |
| # |
| # e.g. 'umsg_format': ('umsg_vformat', 3) means in the wrapper function of |
| # 'umsg_format', it will call 'umsg_vformat' instead, with the va_list arg |
| # inserted as the 3rd argument. |
| |
| # We need to insert the va_list (named args) at the position |
| # indicated by the KNOWN_VA_FUNCTIONS map. |
| KNOWN_VA_FUNCTIONS = { |
| 'u_formatMessage': ('u_vformatMessage', 5), |
| 'u_parseMessage': ('u_vparseMessage', 5), |
| 'u_formatMessageWithError': ('u_vformatMessageWithError', 6), |
| 'u_parseMessageWithError': ('u_vparseMessageWithError', 5), |
| 'umsg_format': ('umsg_vformat', 3), |
| 'umsg_parse': ('umsg_vparse', 4), |
| 'utrace_format': ('utrace_vformat', 4), |
| } |
| |
| # The following functions are not @stable |
| ALLOWLISTED_FUNCTION_NAMES = ( |
| # Not intended to be called directly, but are used by @stable macros. |
| 'utf8_nextCharSafeBody', |
| 'utf8_appendCharSafeBody', |
| 'utf8_prevCharSafeBody', |
| 'utf8_back1SafeBody', |
| ) |