Merge changes from topic "dpgroup"
* changes:
Add dynamic partition info to target files
BOARD_{GROUP}_SIZE must not be empty.
Add _a and _b to group names to super.
diff --git a/Changes.md b/Changes.md
index 4aa7ea2..2d5cd97 100644
--- a/Changes.md
+++ b/Changes.md
@@ -1,5 +1,22 @@
# Build System Changes for Android.mk Writers
+## `BUILD_NUMBER` removal from Android.mk {#BUILD_NUMBER}
+
+`BUILD_NUMBER` should not be used directly in Android.mk files, as it would
+trigger them to be re-read every time the `BUILD_NUMBER` changes (which it does
+on every build server build). If possible, just remove the use so that your
+builds are more reproducible. If you do need it, use `BUILD_NUMBER_FROM_FILE`:
+
+``` make
+$(LOCAL_BUILT_MODULE):
+ mytool --build_number $(BUILD_NUMBER_FROM_FILE) -o $@
+```
+
+That will expand out to a subshell that will read the current `BUILD_NUMBER`
+whenever it's run. It will not re-run your command if the build number has
+changed, so incremental builds will have the build number from the last time
+the particular output was rebuilt.
+
## `DIST_DIR`, `dist_goal`, and `dist-for-goals` {#dist}
`DIST_DIR` and `dist_goal` are no longer available when reading Android.mk
diff --git a/core/Makefile b/core/Makefile
index ebca3f7..6721757 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -1206,7 +1206,7 @@
ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true)
INTERNAL_USERIMAGES_DEPS := $(SIMG2IMG)
-INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK)
+INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK) $(TUNE2FS)
ifeq ($(TARGET_USERIMAGES_USE_F2FS),true)
INTERNAL_USERIMAGES_DEPS += $(MKF2FSUSERIMG) $(MAKE_F2FS)
endif
@@ -3100,6 +3100,7 @@
$(HOST_OUT_EXECUTABLES)/mke2fs \
$(HOST_OUT_EXECUTABLES)/mkuserimg_mke2fs \
$(HOST_OUT_EXECUTABLES)/e2fsdroid \
+ $(HOST_OUT_EXECUTABLES)/tune2fs \
$(HOST_OUT_EXECUTABLES)/mksquashfsimage.sh \
$(HOST_OUT_EXECUTABLES)/mksquashfs \
$(HOST_OUT_EXECUTABLES)/mkf2fsuserimg.sh \
diff --git a/core/android_manifest.mk b/core/android_manifest.mk
index 8608ca1..c3af942 100644
--- a/core/android_manifest.mk
+++ b/core/android_manifest.mk
@@ -71,6 +71,11 @@
ifeq ($(LOCAL_PRIVATE_PLATFORM_APIS),true)
my_manifest_fixer_flags += --uses-non-sdk-api
endif
+
+ifeq (true,$(LOCAL_PREFER_INTEGRITY))
+ my_manifest_fixer_flags += --prefer-integrity
+endif
+
$(fixed_android_manifest): PRIVATE_MANIFEST_FIXER_FLAGS := $(my_manifest_fixer_flags)
# These two libs are added as optional dependencies (<uses-library> with
# android:required set to false). This is because they haven't existed in pre-P
diff --git a/core/autogen_test_config.mk b/core/autogen_test_config.mk
index 4b912fb..a01d80f 100644
--- a/core/autogen_test_config.mk
+++ b/core/autogen_test_config.mk
@@ -34,16 +34,9 @@
endif
# Auto generating test config file for native test
$(autogen_test_config_file): PRIVATE_MODULE_NAME := $(LOCAL_MODULE)
-$(autogen_test_config_file): PRIVATE_RUN_UID := $(LOCAL_RUN_TEST_AS)
$(autogen_test_config_file) : $(autogen_test_config_template)
@echo "Auto generating test config $(notdir $@)"
$(hide) sed 's&{MODULE}&$(PRIVATE_MODULE_NAME)&g' $< > $@
-ifneq ($(LOCAL_RUN_TEST_AS),)
- $(hide) sed -i 's&{UID_OPTION}&<option name="run-test-as" value="$(PRIVATE_RUN_UID)" />&g' $@
-else
- $(hide) sed -i '/{UID_OPTION}/d' $@
-endif
-
my_auto_generate_config := true
else
# Auto generating test config file for instrumentation test
diff --git a/core/clear_vars.mk b/core/clear_vars.mk
index a036267..7f8e9c3 100644
--- a/core/clear_vars.mk
+++ b/core/clear_vars.mk
@@ -213,6 +213,7 @@
LOCAL_PREBUILT_OBJ_FILES:=
LOCAL_PREBUILT_STATIC_JAVA_LIBRARIES:=
LOCAL_PREBUILT_STRIP_COMMENTS:=
+LOCAL_PREFER_INTEGRITY:=
LOCAL_PRESUBMIT_DISABLED:=
LOCAL_PRIVATE_PLATFORM_APIS:=
LOCAL_PRIVILEGED_MODULE:=
@@ -244,7 +245,6 @@
LOCAL_RMTYPEDEFS:=
LOCAL_RRO_THEME:=
LOCAL_RTTI_FLAG:=
-LOCAL_RUN_TEST_AS :=
LOCAL_SANITIZE:=
LOCAL_SANITIZE_DIAG:=
LOCAL_SANITIZE_RECOVER:=
diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk
index 62597d1..6a892e2 100644
--- a/core/dex_preopt_odex_install.mk
+++ b/core/dex_preopt_odex_install.mk
@@ -3,6 +3,12 @@
# Output variables: LOCAL_DEX_PREOPT, LOCAL_UNCOMPRESS_DEX, built_odex,
# dexpreopt_boot_jar_module
+ifeq (true,$(LOCAL_PREFER_INTEGRITY))
+ LOCAL_UNCOMPRESS_DEX := true
+else
+ LOCAL_UNCOMPRESS_DEX :=
+endif
+
# We explicitly uncompress APKs of privileged apps, and used by
# privileged apps
ifneq (true,$(DONT_UNCOMPRESS_PRIV_APPS_DEXS))
@@ -49,7 +55,7 @@
endif
# If we have product-specific config for this module?
-ifeq (disable,$(DEXPREOPT.$(TARGET_PRODUCT).$(LOCAL_MODULE).CONFIG))
+ifneq (,$(filter $(LOCAL_MODULE),$(DEXPREOPT_DISABLED_MODULES)))
LOCAL_DEX_PREOPT :=
endif
diff --git a/core/main.mk b/core/main.mk
index 6ff5f93..461c3b4 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -66,12 +66,15 @@
$(shell mkdir -p $(OUT_DIR) && \
echo -n $(BUILD_NUMBER) > $(OUT_DIR)/build_number.txt)
BUILD_NUMBER_FILE := $(OUT_DIR)/build_number.txt
+.KATI_READONLY := BUILD_NUMBER_FILE
+$(KATI_obsolete_var BUILD_NUMBER,See https://android.googlesource.com/platform/build/+/master/Changes.md#BUILD_NUMBER)
ifeq ($(HOST_OS),darwin)
DATE_FROM_FILE := date -r $(BUILD_DATETIME_FROM_FILE)
else
DATE_FROM_FILE := date -d @$(BUILD_DATETIME_FROM_FILE)
endif
+.KATI_READONLY := DATE_FROM_FILE
# Pick a reasonable string to use to identify files.
ifeq ($(strip $(HAS_BUILD_NUMBER)),false)
@@ -81,6 +84,7 @@
else
FILE_NAME_TAG := $(file <$(BUILD_NUMBER_FILE))
endif
+.KATI_READONLY := FILE_NAME_TAG
# Make an empty directory, which can be used to make empty jars
EMPTY_DIRECTORY := $(OUT_DIR)/empty
diff --git a/core/native_test_config_template.xml b/core/native_test_config_template.xml
index 077223e..a88d57c 100644
--- a/core/native_test_config_template.xml
+++ b/core/native_test_config_template.xml
@@ -23,7 +23,6 @@
</target_preparer>
<test class="com.android.tradefed.testtype.GTest" >
- {UID_OPTION}
<option name="native-test-device-path" value="/data/local/tmp" />
<option name="module-name" value="{MODULE}" />
</test>
diff --git a/core/product_config.mk b/core/product_config.mk
index 27af09e..577bafe 100644
--- a/core/product_config.mk
+++ b/core/product_config.mk
@@ -423,6 +423,7 @@
# Resolve and setup per-module dex-preopt configs.
PRODUCT_DEX_PREOPT_MODULE_CONFIGS := \
$(strip $(PRODUCTS.$(INTERNAL_PRODUCT).PRODUCT_DEX_PREOPT_MODULE_CONFIGS))
+DEXPREOPT_DISABLED_MODULES :=
# If a module has multiple setups, the first takes precedence.
_pdpmc_modules :=
$(foreach c,$(PRODUCT_DEX_PREOPT_MODULE_CONFIGS),\
@@ -431,7 +432,9 @@
$(eval _pdpmc_modules += $(m))\
$(eval cf := $(patsubst $(m)=%,%,$(c)))\
$(eval cf := $(subst $(_PDPMC_SP_PLACE_HOLDER),$(space),$(cf)))\
- $(eval DEXPREOPT.$(TARGET_PRODUCT).$(m).CONFIG := $(cf))))
+ $(if $(filter disable,$(cf)),\
+ $(eval DEXPREOPT_DISABLED_MODULES += $(m)),\
+ $(eval DEXPREOPT.$(TARGET_PRODUCT).$(m).CONFIG := $(cf)))))
_pdpmc_modules :=
# Resolve and setup per-module sanitizer configs.
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 2f978fa..7a884e0 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -123,6 +123,11 @@
$(call add_json_list, Platform_systemsdk_versions, $(PLATFORM_SYSTEMSDK_VERSIONS))
$(call add_json_bool, Malloc_not_svelte, $(call invert_bool,$(filter true,$(MALLOC_SVELTE))))
$(call add_json_str, Override_rs_driver, $(OVERRIDE_RS_DRIVER))
+$(call add_json_bool, UncompressPrivAppDex, $(call invert_bool,$(filter true,$(DONT_UNCOMPRESS_PRIV_APPS_DEXS))))
+$(call add_json_list, ModulesLoadedByPrivilegedModules, $(PRODUCT_LOADED_BY_PRIVILEGED_MODULES))
+$(call add_json_bool, DefaultStripDex, $(call invert_bool,$(filter nostripping,$(DEX_PREOPT_DEFAULT))))
+$(call add_json_bool, DisableDexPreopt, $(filter false,$(WITH_DEXPREOPT)))
+$(call add_json_list, DisableDexPreoptModules, $(DEXPREOPT_DISABLED_MODULES))
$(call add_json_bool, Product_is_iot, $(filter true,$(PRODUCT_IOT)))
diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py
index 4a013c2..b88171f 100755
--- a/tools/releasetools/build_image.py
+++ b/tools/releasetools/build_image.py
@@ -54,23 +54,53 @@
"""Returns the number of bytes that "path" occupies on host.
Args:
- path: The directory or file to calculate size on
+ path: The directory or file to calculate size on.
Returns:
- The number of bytes.
-
- Raises:
- BuildImageError: On error.
+ The number of bytes based on a 1K block_size.
"""
- env_copy = os.environ.copy()
- env_copy["POSIXLY_CORRECT"] = "1"
- cmd = ["du", "-s", path]
+ cmd = ["du", "-k", "-s", path]
+ output = common.RunAndCheckOutput(cmd, verbose=False)
+ return int(output.split()[0]) * 1024
+
+
+def GetInodeUsage(path):
+ """Returns the number of inodes that "path" occupies on host.
+
+ Args:
+ path: The directory or file to calculate inode number on.
+
+ Returns:
+ The number of inodes used.
+ """
+ cmd = ["find", path, "-print"]
+ output = common.RunAndCheckOutput(cmd, verbose=False)
+ # increase by > 4% as number of files and directories is not whole picture.
+ return output.count('\n') * 25 // 24
+
+
+def GetFilesystemCharacteristics(sparse_image_path):
+ """Returns various filesystem characteristics of "sparse_image_path".
+
+ Args:
+ sparse_image_path: The file to analyze.
+
+ Returns:
+ The characteristics dictionary.
+ """
+ unsparse_image_path = UnsparseImage(sparse_image_path, replace=False)
+
+ cmd = ["tune2fs", "-l", unsparse_image_path]
try:
- output = common.RunAndCheckOutput(cmd, verbose=False, env=env_copy)
- except common.ExternalError:
- raise BuildImageError("Failed to get disk usage:\n{}".format(output))
- # POSIX du returns number of blocks with block size 512
- return int(output.split()[0]) * 512
+ output = common.RunAndCheckOutput(cmd, verbose=False)
+ finally:
+ os.remove(unsparse_image_path)
+ fs_dict = {}
+ for line in output.splitlines():
+ fields = line.split(":")
+ if len(fields) == 2:
+ fs_dict[fields[0].strip()] = fields[1].strip()
+ return fs_dict
def UnsparseImage(sparse_image_path, replace=True):
@@ -121,6 +151,10 @@
if prop_dict["mount_point"] != "system":
return origin_in, fs_config
+ if "first_pass" in prop_dict:
+ prop_dict["mount_point"] = "/"
+ return prop_dict["first_pass"]
+
# Construct a staging directory of the root file system.
in_dir = common.MakeTempDir()
root_dir = prop_dict.get("root_dir")
@@ -144,6 +178,7 @@
with open(fs_config) as fr:
fw.writelines(fr.readlines())
fs_config = merged_fs_config
+ prop_dict["first_pass"] = (in_dir, fs_config)
return in_dir, fs_config
@@ -175,7 +210,7 @@
m = ext4fs_stats.match(last_line)
used_blocks = int(m.groupdict().get('used_blocks'))
total_blocks = int(m.groupdict().get('total_blocks'))
- headroom_blocks = int(prop_dict['partition_headroom']) / BLOCK_SIZE
+ headroom_blocks = int(prop_dict['partition_headroom']) // BLOCK_SIZE
adjusted_blocks = total_blocks - headroom_blocks
if used_blocks > adjusted_blocks:
mount_point = prop_dict["mount_point"]
@@ -202,6 +237,7 @@
Raises:
BuildImageError: On build image failures.
"""
+ original_mount_point = prop_dict["mount_point"]
in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict)
build_command = []
@@ -233,7 +269,8 @@
size = GetDiskUsage(in_dir)
logger.info(
"The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB)
- size += int(prop_dict.get("partition_reserved_size", 0))
+ # If not specified, give us 16MB margin for GetDiskUsage error ...
+ size += int(prop_dict.get("partition_reserved_size", BYTES_IN_MB * 16))
# Round this up to a multiple of 4K so that avbtool works
size = common.RoundUpTo4K(size)
# Adjust partition_size to add more space for AVB footer, to prevent
@@ -244,6 +281,35 @@
lambda x: verity_utils.AVBCalcMaxImageSize(
avbtool, avb_footer_type, x, avb_signing_args))
prop_dict["partition_size"] = str(size)
+ if fs_type.startswith("ext"):
+ if "extfs_inode_count" not in prop_dict:
+ prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir))
+ logger.info(
+ "First Pass based on estimates of %d MB and %s inodes.",
+ size // BYTES_IN_MB, prop_dict["extfs_inode_count"])
+ prop_dict["mount_point"] = original_mount_point
+ BuildImage(in_dir, prop_dict, out_file, target_out)
+ fs_dict = GetFilesystemCharacteristics(out_file)
+ block_size = int(fs_dict.get("Block size", "4096"))
+ free_size = int(fs_dict.get("Free blocks", "0")) * block_size
+ reserved_size = int(prop_dict.get("partition_reserved_size", 0))
+ if free_size <= reserved_size:
+ logger.info(
+ "Not worth reducing image %d <= %d.", free_size, reserved_size)
+ else:
+ size -= free_size
+ size += reserved_size
+ if block_size <= 4096:
+ size = common.RoundUpTo4K(size)
+ else:
+ size = ((size + block_size - 1) // block_size) * block_size
+ extfs_inode_count = prop_dict["extfs_inode_count"]
+ inodes = int(fs_dict.get("Inode count", extfs_inode_count))
+ inodes -= int(fs_dict.get("Free inodes", "0"))
+ prop_dict["extfs_inode_count"] = str(inodes)
+ prop_dict["partition_size"] = str(size)
+ logger.info(
+ "Allocating %d Inodes for %s.", inodes, out_file)
logger.info(
"Allocating %d MB for %s.", size // BYTES_IN_MB, out_file)
@@ -363,7 +429,7 @@
int(prop_dict.get("partition_reserved_size", 0)),
int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB))
print(
- "The max image size for filsystem files is {} bytes ({} MB), out of a "
+ "The max image size for filesystem files is {} bytes ({} MB), out of a "
"total partition size of {} bytes ({} MB).".format(
int(prop_dict["image_size"]),
int(prop_dict["image_size"]) // BYTES_IN_MB,
@@ -677,7 +743,7 @@
glob_dict = LoadGlobalDict(glob_dict_file)
if "mount_point" in glob_dict:
- # The caller knows the mount point and provides a dictionay needed by
+ # The caller knows the mount point and provides a dictionary needed by
# BuildImage().
image_properties = glob_dict
else:
diff --git a/tools/releasetools/test_build_image.py b/tools/releasetools/test_build_image.py
index 634c6b1..1cebd0c 100644
--- a/tools/releasetools/test_build_image.py
+++ b/tools/releasetools/test_build_image.py
@@ -19,7 +19,7 @@
import common
from build_image import (
- BuildImageError, CheckHeadroom, SetUpInDirAndFsConfig)
+ BuildImageError, CheckHeadroom, GetFilesystemCharacteristics, SetUpInDirAndFsConfig)
from test_utils import ReleaseToolsTestCase
@@ -176,3 +176,25 @@
self.assertIn('fs-config-system\n', fs_config_data)
self.assertIn('fs-config-root\n', fs_config_data)
self.assertEqual('/', prop_dict['mount_point'])
+
+ def test_GetFilesystemCharacteristics(self):
+ input_dir = common.MakeTempDir()
+ output_image = common.MakeTempFile(suffix='.img')
+ command = ['mkuserimg_mke2fs', input_dir, output_image, 'ext4',
+ '/system', '409600', '-j', '0']
+ proc = common.Run(command)
+ ext4fs_output, _ = proc.communicate()
+ self.assertEqual(0, proc.returncode)
+
+ output_file = common.MakeTempFile(suffix='.img')
+ cmd = ["img2simg", output_image, output_file]
+ p = common.Run(cmd)
+ p.communicate()
+ self.assertEqual(0, p.returncode)
+
+ fs_dict = GetFilesystemCharacteristics(output_file)
+ self.assertEqual(int(fs_dict['Block size']), 4096)
+ self.assertGreaterEqual(int(fs_dict['Free blocks']), 0) # expect ~88
+ self.assertGreater(int(fs_dict['Inode count']), 0) # expect ~64
+ self.assertGreaterEqual(int(fs_dict['Free inodes']), 0) # expect ~53
+ self.assertGreater(int(fs_dict['Inode count']), int(fs_dict['Free inodes']))