| #!/usr/bin/env python |
| # |
| # Copyright (C) 2018 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| """Generate ICU stable C API wrapper source. |
| |
| |
| This script parses all the header files specified by the ICU module names. For |
| each function in the allowlist, it generates the NDK headers, and shim functions |
| to shim.cpp, which in turn calls the real implementation at runtime. |
| The tool relies on libclang to parse header files. |
| |
| Reference to ICU4C stable C APIs: |
| http://icu-project.org/apiref/icu4c/files.html |
| """ |
| from __future__ import absolute_import |
| from __future__ import print_function |
| |
| import logging |
| import os |
| import re |
| import shutil |
| import subprocess |
| |
| from genutil import ( |
| android_path, |
| generate_shim, |
| generate_symbol_txt, |
| get_allowlisted_apis, |
| AllowlistedDeclarationFilter, |
| DeclaredFunctionsParser, |
| StableDeclarationFilter, |
| ) |
| |
| # No suffix for ndk shim |
| SYMBOL_SUFFIX = '' |
| |
| SECRET_PROCESSING_TOKEN = "@@@SECRET@@@" |
| |
| DOC_BLOCK_COMMENT = r"\/\*\*(?:\*(?!\/)|[^*])*\*\/[ ]*\n" |
| TILL_CLOSE_PARENTHESIS = r"[^)^;]*\)" |
| STABLE_MACRO = r"(?:U_STABLE|U_CAPI)" |
| STABLE_FUNCTION_DECLARATION = r"^(" + DOC_BLOCK_COMMENT + STABLE_MACRO \ |
| + TILL_CLOSE_PARENTHESIS + ");$" |
| NONSTABLE_FUNCTION_DECLARATION = r"^(" + DOC_BLOCK_COMMENT + r"(U_INTERNAL|U_DEPRECATED|U_DRAFT)" \ |
| + TILL_CLOSE_PARENTHESIS + ");$" |
| |
| REGEX_STABLE_FUNCTION_DECLARATION = re.compile(STABLE_FUNCTION_DECLARATION, re.MULTILINE) |
| REGEX_NONSTABLE_FUNCTION_DECLARATION = re.compile(NONSTABLE_FUNCTION_DECLARATION, re.MULTILINE) |
| |
| def get_allowlisted_regex_string(decl_names): |
| """Return a regex in string to capture the C function declarations in the decl_names list""" |
| tag = "|".join(decl_names) |
| return r"(" + DOC_BLOCK_COMMENT + STABLE_MACRO + r"[^(]*(?=" + tag + r")(" + tag + ")" \ |
| + r"\("+ TILL_CLOSE_PARENTHESIS +");$" |
| |
| def get_replacement_adding_api_level_macro(api_level): |
| """Return the replacement string adding the NDK C macro |
| guarding C function declaration by the api_level""" |
| return r"#if !defined(__ANDROID__) || __ANDROID_API__ >= {0}\n\n" \ |
| r"\1 __INTRODUCED_IN({0});\n\n" \ |
| r"#endif // !defined(__ANDROID__) || __ANDROID_API__ >= {0}".format(api_level) |
| |
| def modify_func_declarations(src_path, dst_path, decl_names): |
| """Process the source file, |
| remove the C function declarations not in the decl_names, |
| add guard the functions listed in decl_names by the API level, |
| and output to the dst_path """ |
| allowlist_regex_string = get_allowlisted_regex_string(decl_names) |
| allowlist_decl_regex = re.compile('^' + allowlist_regex_string, re.MULTILINE) |
| secret_allowlist_decl_regex = re.compile('^' + SECRET_PROCESSING_TOKEN |
| + allowlist_regex_string, re.MULTILINE) |
| with open(src_path, "r") as file: |
| src = file.read() |
| |
| # Remove all non-stable function declarations |
| modified = REGEX_NONSTABLE_FUNCTION_DECLARATION.sub('', src) |
| |
| # Insert intermediate token to all functions in the allowlist |
| if decl_names: |
| modified = allowlist_decl_regex.sub(SECRET_PROCESSING_TOKEN + r"\1;", modified) |
| # Remove all other stable declarations not in the allowlist |
| modified = REGEX_STABLE_FUNCTION_DECLARATION.sub('', modified) |
| # Insert C macro and annotation to indicate the API level to each functions in the allowlist |
| modified = secret_allowlist_decl_regex.sub( |
| get_replacement_adding_api_level_macro(31), modified) |
| |
| with open(dst_path, "w") as out: |
| out.write(modified) |
| def remove_ignored_includes(file_path, include_list): |
| """ |
| Remove the included header, i.e. #include lines, listed in include_list from the file_path |
| header. |
| """ |
| |
| # Do nothing if the list is empty |
| if not include_list: |
| return |
| |
| tag = "|".join(include_list) |
| |
| with open(file_path, "r") as file: |
| content = file.read() |
| |
| regex = re.compile(r"^#include \"unicode\/(" + tag + ")\"\n", re.MULTILINE) |
| content = regex.sub('', content) |
| |
| with open(file_path, "w") as out: |
| out.write(content) |
| |
| def copy_header_only_files(): |
| """Copy required header only files""" |
| base_src_path = android_path('external/icu/icu4c/source/') |
| base_dest_path = android_path('external/icu/libicu/ndk_headers/unicode/') |
| with open(android_path('external/icu/tools/icu4c_srcgen/libicu_required_header_only_files.txt'), |
| 'r') as in_file: |
| header_only_files = [ |
| base_src_path + line.strip() for line in in_file.readlines() if not line.startswith('#') |
| ] |
| |
| for src_path in header_only_files: |
| dest_path = base_dest_path + os.path.basename(src_path) |
| cmd = ['sed', |
| "s/U_SHOW_CPLUSPLUS_API/LIBICU_U_SHOW_CPLUSPLUS_API/g", |
| src_path |
| ] |
| |
| with open(dest_path, "w") as destfile: |
| subprocess.check_call(cmd, stdout=destfile) |
| |
| def copy_cts_headers(): |
| """Copy headers from common/ and i18n/ to cts_headers/ for compiling cintltst as CTS.""" |
| dst_folder = android_path('external/icu/libicu/cts_headers') |
| if os.path.exists(dst_folder): |
| shutil.rmtree(dst_folder) |
| os.mkdir(dst_folder) |
| os.mkdir(os.path.join(dst_folder, 'unicode')) |
| |
| shutil.copyfile(android_path('external/icu/android_icu4c/include/uconfig_local.h'), |
| android_path('external/icu/libicu/cts_headers/uconfig_local.h')) |
| |
| header_subfolders = ( |
| 'common', |
| 'common/unicode', |
| 'i18n', |
| 'i18n/unicode', |
| ) |
| for subfolder in header_subfolders: |
| path = android_path('external/icu/icu4c/source', subfolder) |
| files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.h')] |
| |
| for src_path in files: |
| base_header_name = os.path.basename(src_path) |
| dst_path = dst_folder |
| if subfolder.endswith('unicode'): |
| dst_path = os.path.join(dst_path, 'unicode') |
| dst_path = os.path.join(dst_path, base_header_name) |
| |
| shutil.copyfile(src_path, dst_path) |
| |
| def get_rename_macro_regex(decl_names): |
| """Return a regex in string to capture the C macro defining the name in the decl_names list""" |
| tag = "|".join(decl_names) |
| return re.compile(r"^(#define (?:" + tag + r") .*)$", re.MULTILINE) |
| |
| def generate_cts_headers(decl_names): |
| """Generate headers for compiling cintltst as CTS.""" |
| copy_cts_headers() |
| |
| # Disable all C macro renaming the NDK functions in order to test the functions in the CTS |
| urename_path = android_path('external/icu/libicu/cts_headers/unicode/urename.h') |
| with open(urename_path, "r") as file: |
| src = file.read() |
| |
| regex = get_rename_macro_regex(decl_names) |
| modified = regex.sub(r"// \1", src) |
| |
| with open(urename_path, "w") as out: |
| out.write(modified) |
| |
| IGNORED_INCLUDE_DEPENDENCY = { |
| "ulocdata.h": ["ures.h", "uset.h", ], |
| "ustring.h": ["uiter.h", ], |
| } |
| |
| def main(): |
| """Parse the ICU4C headers and generate the shim libicu.""" |
| logging.basicConfig(level=logging.DEBUG) |
| |
| allowlisted_apis = get_allowlisted_apis('libicu_export.txt') |
| decl_filters = [StableDeclarationFilter()] |
| decl_filters.append(AllowlistedDeclarationFilter(allowlisted_apis)) |
| parser = DeclaredFunctionsParser(decl_filters, []) |
| parser.set_ignored_include_dependency(IGNORED_INCLUDE_DEPENDENCY) |
| |
| parser.parse() |
| |
| includes = parser.header_includes |
| functions = parser.declared_functions |
| header_to_function_names = parser.header_to_function_names |
| |
| # The shim has the allowlisted functions only |
| functions = [f for f in functions if f.name in allowlisted_apis] |
| |
| headers_folder = android_path('external/icu/libicu/ndk_headers/unicode') |
| if os.path.exists(headers_folder): |
| shutil.rmtree(headers_folder) |
| os.mkdir(headers_folder) |
| |
| with open(android_path('external/icu/libicu/src/shim.cpp'), |
| 'w') as out_file: |
| out_file.write(generate_shim(functions, includes, SYMBOL_SUFFIX, 'libicu_shim.cpp.j2') |
| .encode('utf8')) |
| |
| with open(android_path('external/icu/libicu/libicu.map.txt'), 'w') as out_file: |
| out_file.write(generate_symbol_txt(functions, [], 'libicu.map.txt.j2') |
| .encode('utf8')) |
| |
| # Process the C headers and put them into the ndk folder. |
| for src_path in parser.header_paths_to_copy: |
| basename = os.path.basename(src_path) |
| dst_path = os.path.join(headers_folder, basename) |
| modify_func_declarations(src_path, dst_path, header_to_function_names[basename]) |
| # Remove #include lines from the header files. |
| if basename in IGNORED_INCLUDE_DEPENDENCY: |
| remove_ignored_includes(dst_path, IGNORED_INCLUDE_DEPENDENCY[basename]) |
| |
| copy_header_only_files() |
| |
| generate_cts_headers(allowlisted_apis) |
| |
| # Apply documentation patches by the following shell script |
| subprocess.check_call( |
| [android_path('external/icu/tools/icu4c_srcgen/doc_patches/apply_patches.sh')]) |
| |
| if __name__ == '__main__': |
| main() |