minigbm: refactor to add unified resolve_format_and_use_flags helper

It makes more sense to resovle the format and use_flags together to
avoid duplicates.

Add the helper to vc4 and dumb_driver backends since that's missed
from prior frontend refactoring.

Split the resolve helper in virtgpu_virgl for the 3d and 2d paths to
make code logic cleaner.

BUG=b:199524294
TEST=CQ

Change-Id: I89b633ea484f0fc5bb9b4e0548cad017b9970cc6
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/minigbm/+/3200143
Tested-by: Yiwei Zhang <zzyiwei@chromium.org>
Tested-by: Jason Macnak <natsu@google.com>
Reviewed-by: Chia-I Wu <olv@google.com>
Reviewed-by: Jason Macnak <natsu@google.com>
Reviewed-by: Dominik Behr <dbehr@chromium.org>
Commit-Queue: Yiwei Zhang <zzyiwei@chromium.org>
diff --git a/amdgpu.c b/amdgpu.c
index 9b964a1..3ebf1ab 100644
--- a/amdgpu.c
+++ b/amdgpu.c
@@ -785,8 +785,7 @@
 	.bo_map = amdgpu_map_bo,
 	.bo_unmap = amdgpu_unmap_bo,
 	.bo_invalidate = amdgpu_bo_invalidate,
-	.resolve_format = drv_resolve_format_helper,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 	.num_planes_from_modifier = dri_num_planes_from_modifier,
 };
 
diff --git a/cros_gralloc/cros_gralloc_driver.cc b/cros_gralloc/cros_gralloc_driver.cc
index a2d47b4..fea4e1a 100644
--- a/cros_gralloc/cros_gralloc_driver.cc
+++ b/cros_gralloc/cros_gralloc_driver.cc
@@ -136,11 +136,9 @@
 	uint64_t resolved_use_flags;
 	struct combination *combo;
 
-	resolved_format = drv_resolve_format(drv_, descriptor->drm_format, descriptor->use_flags);
-	if (resolved_format == DRM_FORMAT_NONE)
-		return false;
+	drv_resolve_format_and_use_flags(drv_, descriptor->drm_format, descriptor->use_flags,
+					 &resolved_format, &resolved_use_flags);
 
-	resolved_use_flags = drv_resolve_use_flags(drv_, resolved_format, descriptor->use_flags);
 	/*
 	 * This unmask is a backup in the case DRM_FORMAT_FLEX_IMPLEMENTATION_DEFINED is resolved
 	 * to non-YUV formats.
@@ -562,9 +560,15 @@
 	return buffer->get_reserved_region(reserved_region_addr, reserved_region_size);
 }
 
-uint32_t cros_gralloc_driver::get_resolved_drm_format(uint32_t drm_format, uint64_t usage)
+uint32_t cros_gralloc_driver::get_resolved_drm_format(uint32_t drm_format, uint64_t use_flags)
 {
-	return drv_resolve_format(drv_, drm_format, usage);
+	uint32_t resolved_format;
+	uint64_t resolved_use_flags;
+
+	drv_resolve_format_and_use_flags(drv_, drm_format, use_flags, &resolved_format,
+					 &resolved_use_flags);
+
+	return resolved_format;
 }
 
 cros_gralloc_buffer *cros_gralloc_driver::get_buffer(cros_gralloc_handle_t hnd)
diff --git a/cros_gralloc/cros_gralloc_driver.h b/cros_gralloc/cros_gralloc_driver.h
index 8403ff6..d1456a8 100644
--- a/cros_gralloc/cros_gralloc_driver.h
+++ b/cros_gralloc/cros_gralloc_driver.h
@@ -39,7 +39,7 @@
 	int32_t get_reserved_region(buffer_handle_t handle, void **reserved_region_addr,
 				    uint64_t *reserved_region_size);
 
-	uint32_t get_resolved_drm_format(uint32_t drm_format, uint64_t usage);
+	uint32_t get_resolved_drm_format(uint32_t drm_format, uint64_t use_flags);
 
 	void for_each_handle(const std::function<void(cros_gralloc_handle_t)> &function);
 
diff --git a/drv.c b/drv.c
index 3a18fd0..39a6df2 100644
--- a/drv.c
+++ b/drv.c
@@ -710,20 +710,13 @@
 	return (fourcc_internal == DRM_FORMAT_YVU420_ANDROID) ? DRM_FORMAT_YVU420 : fourcc_internal;
 }
 
-uint32_t drv_resolve_format(struct driver *drv, uint32_t format, uint64_t use_flags)
+void drv_resolve_format_and_use_flags(struct driver *drv, uint32_t format, uint64_t use_flags,
+				      uint32_t *out_format, uint64_t *out_use_flags)
 {
-	if (drv->backend->resolve_format)
-		return drv->backend->resolve_format(format, use_flags);
+	assert(drv->backend->resolve_format_and_use_flags);
 
-	return format;
-}
-
-uint64_t drv_resolve_use_flags(struct driver *drv, uint32_t format, uint64_t use_flags)
-{
-	if (drv->backend->resolve_use_flags)
-		return drv->backend->resolve_use_flags(drv, format, use_flags);
-
-	return use_flags;
+	drv->backend->resolve_format_and_use_flags(drv, format, use_flags, out_format,
+						   out_use_flags);
 }
 
 uint32_t drv_num_buffers_per_bo(struct bo *bo)
diff --git a/drv.h b/drv.h
index 037cc01..9687025 100644
--- a/drv.h
+++ b/drv.h
@@ -185,7 +185,8 @@
 
 uint32_t drv_stride_from_format(uint32_t format, uint32_t width, size_t plane);
 
-uint32_t drv_resolve_format(struct driver *drv, uint32_t format, uint64_t use_flags);
+void drv_resolve_format_and_use_flags(struct driver *drv, uint32_t format, uint64_t use_flags,
+				      uint32_t *out_format, uint64_t *out_use_flags);
 
 uint64_t drv_resolve_use_flags(struct driver *drv, uint32_t format, uint64_t use_flags);
 
diff --git a/drv_helpers.c b/drv_helpers.c
index 3874251..0b6cb29 100644
--- a/drv_helpers.c
+++ b/drv_helpers.c
@@ -573,27 +573,30 @@
 	return false;
 }
 
-uint32_t drv_resolve_format_helper(uint32_t format, uint64_t use_flags)
+void drv_resolve_format_and_use_flags_helper(struct driver *drv, uint32_t format,
+					     uint64_t use_flags, uint32_t *out_format,
+					     uint64_t *out_use_flags)
 {
+	*out_format = format;
+	*out_use_flags = use_flags;
 	switch (format) {
 	case DRM_FORMAT_FLEX_IMPLEMENTATION_DEFINED:
 		/* Common camera implementation defined format. */
-		if (use_flags & (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE))
-			return DRM_FORMAT_NV12;
-		/* A common hack: See b/28671744 */
-		return DRM_FORMAT_XBGR8888;
+		if (use_flags & (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE)) {
+			*out_format = DRM_FORMAT_NV12;
+		} else {
+			/* HACK: See b/28671744 */
+			*out_format = DRM_FORMAT_XBGR8888;
+		}
+		break;
 	case DRM_FORMAT_FLEX_YCbCr_420_888:
 		/* Common flexible video format. */
-		return DRM_FORMAT_NV12;
+		*out_format = DRM_FORMAT_NV12;
+		break;
+	case DRM_FORMAT_YVU420_ANDROID:
+		*out_use_flags &= ~BO_USE_SCANOUT;
+		break;
 	default:
-		return format;
+		break;
 	}
 }
-
-uint64_t drv_resolve_use_flags_helper(struct driver *drv, uint32_t format, uint64_t use_flags)
-{
-	if (format == DRM_FORMAT_YVU420_ANDROID)
-		return use_flags & ~BO_USE_SCANOUT;
-
-	return use_flags;
-}
diff --git a/drv_helpers.h b/drv_helpers.h
index 2913fb3..edb69bf 100644
--- a/drv_helpers.h
+++ b/drv_helpers.h
@@ -44,7 +44,8 @@
 uint64_t drv_pick_modifier(const uint64_t *modifiers, uint32_t count,
 			   const uint64_t *modifier_order, uint32_t order_count);
 bool drv_has_modifier(const uint64_t *list, uint32_t count, uint64_t modifier);
-uint32_t drv_resolve_format_helper(uint32_t format, uint64_t use_flags);
-uint64_t drv_resolve_use_flags_helper(struct driver *drv, uint32_t format, uint64_t use_flags);
+void drv_resolve_format_and_use_flags_helper(struct driver *drv, uint32_t format,
+					     uint64_t use_flags, uint32_t *out_format,
+					     uint64_t *out_use_flags);
 
 #endif
diff --git a/drv_priv.h b/drv_priv.h
index 791fc56..63f7804 100644
--- a/drv_priv.h
+++ b/drv_priv.h
@@ -91,8 +91,9 @@
 	int (*bo_unmap)(struct bo *bo, struct vma *vma);
 	int (*bo_invalidate)(struct bo *bo, struct mapping *mapping);
 	int (*bo_flush)(struct bo *bo, struct mapping *mapping);
-	uint32_t (*resolve_format)(uint32_t format, uint64_t use_flags);
-	uint64_t (*resolve_use_flags)(struct driver *drv, uint32_t format, uint64_t use_flags);
+	void (*resolve_format_and_use_flags)(struct driver *drv, uint32_t format,
+					     uint64_t use_flags, uint32_t *out_format,
+					     uint64_t *out_use_flags);
 	size_t (*num_planes_from_modifier)(struct driver *drv, uint32_t format, uint64_t modifier);
 	int (*resource_info)(struct bo *bo, uint32_t strides[DRV_MAX_PLANES],
 			     uint32_t offsets[DRV_MAX_PLANES], uint64_t *format_modifier);
diff --git a/dumb_driver.c b/dumb_driver.c
index d94ca4e..baceca1 100644
--- a/dumb_driver.c
+++ b/dumb_driver.c
@@ -17,6 +17,7 @@
 		.bo_import = drv_prime_bo_import,                                                  \
 		.bo_map = drv_dumb_bo_map,                                                         \
 		.bo_unmap = drv_bo_munmap,                                                         \
+		.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,           \
 	};
 
 static const uint32_t scanout_render_formats[] = { DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888,
diff --git a/i915.c b/i915.c
index c4002ef..10a1ec2 100644
--- a/i915.c
+++ b/i915.c
@@ -684,8 +684,7 @@
 	.bo_unmap = drv_bo_munmap,
 	.bo_invalidate = i915_bo_invalidate,
 	.bo_flush = i915_bo_flush,
-	.resolve_format = drv_resolve_format_helper,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 };
 
 #endif
diff --git a/mediatek.c b/mediatek.c
index 81f9441..9d59fca 100644
--- a/mediatek.c
+++ b/mediatek.c
@@ -275,36 +275,51 @@
 	return 0;
 }
 
-static uint32_t mediatek_resolve_format(uint32_t format, uint64_t use_flags)
+static void mediatek_resolve_format_and_use_flags(struct driver *drv, uint32_t format,
+						  uint64_t use_flags, uint32_t *out_format,
+						  uint64_t *out_use_flags)
 {
+	*out_format = format;
+	*out_use_flags = use_flags;
 	switch (format) {
 	case DRM_FORMAT_FLEX_IMPLEMENTATION_DEFINED:
 #ifdef MTK_MT8183
 		/* Only MT8183 Camera subsystem offers private reprocessing
 		 * capability. CAMERA_READ indicates the buffer is intended for
 		 * reprocessing and hence given the private format for MTK. */
-		if (use_flags & BO_USE_CAMERA_READ)
-			return DRM_FORMAT_MTISP_SXYZW10;
+		if (use_flags & BO_USE_CAMERA_READ) {
+			*out_format = DRM_FORMAT_MTISP_SXYZW10;
+			break;
+		}
 #endif
-		if (use_flags & BO_USE_CAMERA_WRITE)
-			return DRM_FORMAT_NV12;
+		if (use_flags & BO_USE_CAMERA_WRITE) {
+			*out_format = DRM_FORMAT_NV12;
+			break;
+		}
 
-		/*HACK: See b/28671744 */
-		return DRM_FORMAT_XBGR8888;
+		/* HACK: See b/28671744 */
+		*out_format = DRM_FORMAT_XBGR8888;
+		break;
 	case DRM_FORMAT_FLEX_YCbCr_420_888:
 #if defined(MTK_MT8183) || defined(MTK_MT8192) || defined(MTK_MT8195)
 		// TODO(hiroh): Switch to use NV12 for video decoder on MT8173 as well.
 		if (use_flags & (BO_USE_HW_VIDEO_DECODER)) {
-			return DRM_FORMAT_NV12;
+			*out_format = DRM_FORMAT_NV12;
+			break;
 		}
 #endif
 		if (use_flags &
 		    (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE | BO_USE_HW_VIDEO_ENCODER)) {
-			return DRM_FORMAT_NV12;
+			*out_format = DRM_FORMAT_NV12;
+			break;
 		}
-		return DRM_FORMAT_YVU420;
+		*out_format = DRM_FORMAT_YVU420;
+		break;
+	case DRM_FORMAT_YVU420_ANDROID:
+		*out_use_flags &= ~BO_USE_SCANOUT;
+		break;
 	default:
-		return format;
+		break;
 	}
 }
 
@@ -319,8 +334,7 @@
 	.bo_unmap = mediatek_bo_unmap,
 	.bo_invalidate = mediatek_bo_invalidate,
 	.bo_flush = mediatek_bo_flush,
-	.resolve_format = mediatek_resolve_format,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = mediatek_resolve_format_and_use_flags,
 };
 
 #endif
diff --git a/msm.c b/msm.c
index ac9a15b..a07c1c4 100644
--- a/msm.c
+++ b/msm.c
@@ -384,7 +384,6 @@
 	.bo_import = drv_prime_bo_import,
 	.bo_map = msm_bo_map,
 	.bo_unmap = drv_bo_munmap,
-	.resolve_format = drv_resolve_format_helper,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 };
 #endif /* DRV_MSM */
diff --git a/rockchip.c b/rockchip.c
index 5de4977..89063ee 100644
--- a/rockchip.c
+++ b/rockchip.c
@@ -271,8 +271,7 @@
 	.bo_unmap = rockchip_bo_unmap,
 	.bo_invalidate = rockchip_bo_invalidate,
 	.bo_flush = rockchip_bo_flush,
-	.resolve_format = drv_resolve_format_helper,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 };
 
 #endif
diff --git a/vc4.c b/vc4.c
index 9f6a679..820e4fe 100644
--- a/vc4.c
+++ b/vc4.c
@@ -131,6 +131,7 @@
 	.bo_destroy = drv_gem_bo_destroy,
 	.bo_map = vc4_bo_map,
 	.bo_unmap = drv_bo_munmap,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 };
 
 #endif
diff --git a/virtgpu_cross_domain.c b/virtgpu_cross_domain.c
index 160704e..e131ed2 100644
--- a/virtgpu_cross_domain.c
+++ b/virtgpu_cross_domain.c
@@ -422,6 +422,5 @@
 	.bo_destroy = drv_gem_bo_destroy,
 	.bo_map = cross_domain_bo_map,
 	.bo_unmap = drv_bo_munmap,
-	.resolve_format = drv_resolve_format_helper,
-	.resolve_use_flags = drv_resolve_use_flags_helper,
+	.resolve_format_and_use_flags = drv_resolve_format_and_use_flags_helper,
 };
diff --git a/virtgpu_virgl.c b/virtgpu_virgl.c
index 9745288..becce8b 100644
--- a/virtgpu_virgl.c
+++ b/virtgpu_virgl.c
@@ -945,66 +945,93 @@
 	return 0;
 }
 
-static uint32_t virgl_resolve_format(uint32_t format, uint64_t use_flags)
+static void virgl_3d_resolve_format_and_use_flags(struct driver *drv, uint32_t format,
+						  uint64_t use_flags, uint32_t *out_format,
+						  uint64_t *out_use_flags)
 {
+	*out_format = format;
+	*out_use_flags = use_flags;
 	switch (format) {
 	case DRM_FORMAT_FLEX_IMPLEMENTATION_DEFINED:
 		/* Camera subsystem requires NV12. */
-		if (use_flags & (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE))
-			return DRM_FORMAT_NV12;
-		/*HACK: See b/28671744 */
-		return DRM_FORMAT_XBGR8888;
+		if (use_flags & (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE)) {
+			*out_format = DRM_FORMAT_NV12;
+		} else {
+			/* HACK: See b/28671744 */
+			*out_format = DRM_FORMAT_XBGR8888;
+		}
+		break;
 	case DRM_FORMAT_FLEX_YCbCr_420_888:
-		/*
-		 * All of our host drivers prefer NV12 as their flexible media format.
-		 * If that changes, this will need to be modified.
-		 */
-		if (params[param_3d].value)
-			return DRM_FORMAT_NV12;
-		else
-			return DRM_FORMAT_YVU420_ANDROID;
+		/* All of our host drivers prefer NV12 as their flexible media format.
+		 * If that changes, this will need to be modified. */
+		*out_format = DRM_FORMAT_NV12;
+		/* fallthrough */
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_XRGB8888:
+		/* These are the scanout capable formats to the guest. Strip scanout use_flag if the
+		 * host does not natively support scanout on the requested format. */
+		if ((use_flags & BO_USE_SCANOUT) &&
+		    !virgl_supports_combination_natively(drv, format, BO_USE_SCANOUT))
+			*out_use_flags &= ~BO_USE_SCANOUT;
+		break;
+	case DRM_FORMAT_YVU420_ANDROID:
+		*out_use_flags &= ~BO_USE_SCANOUT;
+		/* HACK: See b/172389166. Also see gbm_bo_create. */
+		*out_use_flags |= BO_USE_LINEAR;
+		break;
 	default:
-		return format;
+		break;
 	}
 }
 
-static uint64_t virgl_resolve_use_flags(struct driver *drv, uint32_t format, uint64_t use_flags)
+static void virgl_2d_resolve_format_and_use_flags(uint32_t format, uint64_t use_flags,
+						  uint32_t *out_format, uint64_t *out_use_flags)
 {
-	if (format == DRM_FORMAT_YVU420_ANDROID) {
-		use_flags &= ~BO_USE_SCANOUT;
-		/*
-		 * HACK: See b/172389166. This is for HAL_PIXEL_FORMAT_YV12 buffers allocated by
-		 * arcvm. None of our platforms can display YV12, so we can treat as a SW buffer.
-		 * Remove once this can be intelligently resolved in the guest. Also see
-		 * gbm_bo_create.
-		 */
-		use_flags |= BO_USE_LINEAR;
-		return use_flags;
-	}
+	*out_format = format;
+	*out_use_flags = use_flags;
 
-	if (params[param_3d].value) {
-		switch (format) {
-		/* formats need to support scanout */
-		case DRM_FORMAT_ABGR8888:
-		case DRM_FORMAT_ARGB8888:
-		case DRM_FORMAT_RGB565:
-		case DRM_FORMAT_XBGR8888:
-		case DRM_FORMAT_XRGB8888:
-		case DRM_FORMAT_NV12:
-			/* strip scanout use_flag if necessary */
-			if ((use_flags & BO_USE_SCANOUT) &&
-			    !virgl_supports_combination_natively(drv, format, BO_USE_SCANOUT))
-				return use_flags & ~BO_USE_SCANOUT;
-			break;
-		default:
-			break;
+	/* HACK: See crrev/c/1849773 */
+	if (format != DRM_FORMAT_XRGB8888)
+		*out_use_flags &= ~BO_USE_SCANOUT;
+
+	switch (format) {
+	case DRM_FORMAT_FLEX_IMPLEMENTATION_DEFINED:
+		/* Camera subsystem requires NV12. */
+		if (use_flags & (BO_USE_CAMERA_READ | BO_USE_CAMERA_WRITE)) {
+			*out_format = DRM_FORMAT_NV12;
+		} else {
+			/* HACK: See b/28671744 */
+			*out_format = DRM_FORMAT_XBGR8888;
 		}
-	} else {
-		if (format != DRM_FORMAT_XRGB8888)
-			return use_flags & ~BO_USE_SCANOUT;
+		break;
+	case DRM_FORMAT_FLEX_YCbCr_420_888:
+		*out_format = DRM_FORMAT_YVU420_ANDROID;
+		/* fallthrough */
+	case DRM_FORMAT_YVU420_ANDROID:
+		*out_use_flags &= ~BO_USE_SCANOUT;
+		/* HACK: See b/172389166. Also see gbm_bo_create. */
+		*out_use_flags |= BO_USE_LINEAR;
+		break;
+	default:
+		break;
 	}
+}
 
-	return use_flags;
+static void virgl_resolve_format_and_use_flags(struct driver *drv, uint32_t format,
+					       uint64_t use_flags, uint32_t *out_format,
+					       uint64_t *out_use_flags)
+{
+	if (params[param_3d].value) {
+		return virgl_3d_resolve_format_and_use_flags(drv, format, use_flags, out_format,
+							     out_use_flags);
+	} else {
+		return virgl_2d_resolve_format_and_use_flags(format, use_flags, out_format,
+							     out_use_flags);
+	}
 }
 
 static int virgl_resource_info(struct bo *bo, uint32_t strides[DRV_MAX_PLANES],
@@ -1058,7 +1085,7 @@
 				       .bo_unmap = drv_bo_munmap,
 				       .bo_invalidate = virgl_bo_invalidate,
 				       .bo_flush = virgl_bo_flush,
-				       .resolve_format = virgl_resolve_format,
-				       .resolve_use_flags = virgl_resolve_use_flags,
+				       .resolve_format_and_use_flags =
+					   virgl_resolve_format_and_use_flags,
 				       .resource_info = virgl_resource_info,
 				       .get_max_texture_2d_size = virgl_get_max_texture_2d_size };