Merge "Toggle for RenderEngine to cleanup texture memory."
diff --git a/libs/renderengine/gl/GLESRenderEngine.cpp b/libs/renderengine/gl/GLESRenderEngine.cpp
index 79c3654..672ffae 100644
--- a/libs/renderengine/gl/GLESRenderEngine.cpp
+++ b/libs/renderengine/gl/GLESRenderEngine.cpp
@@ -407,6 +407,23 @@
     mImageManager = std::make_unique<ImageManager>(this);
     mImageManager->initThread();
     mDrawingBuffer = createFramebuffer();
+    sp<GraphicBuffer> buf =
+            new GraphicBuffer(1, 1, PIXEL_FORMAT_RGBA_8888, 1,
+                              GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_TEXTURE, "placeholder");
+
+    const status_t err = buf->initCheck();
+    if (err != OK) {
+        ALOGE("Error allocating placeholder buffer: %d", err);
+        return;
+    }
+    mPlaceholderBuffer = buf.get();
+    EGLint attributes[] = {
+            EGL_NONE,
+    };
+    mPlaceholderImage = eglCreateImageKHR(mEGLDisplay, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID,
+                                          mPlaceholderBuffer, attributes);
+    ALOGE_IF(mPlaceholderImage == EGL_NO_IMAGE_KHR, "Failed to create placeholder image: %#x",
+             eglGetError());
 }
 
 GLESRenderEngine::~GLESRenderEngine() {
@@ -421,6 +438,7 @@
         eglDestroyImageKHR(mEGLDisplay, expired);
         DEBUG_EGL_IMAGE_TRACKER_DESTROY();
     }
+    eglDestroyImageKHR(mEGLDisplay, mPlaceholderImage);
     mImageCache.clear();
     eglMakeCurrent(mEGLDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
     eglTerminate(mEGLDisplay);
@@ -587,6 +605,9 @@
 }
 
 void GLESRenderEngine::deleteTextures(size_t count, uint32_t const* names) {
+    for (int i = 0; i < count; ++i) {
+        mTextureView.erase(names[i]);
+    }
     glDeleteTextures(count, names);
 }
 
@@ -644,6 +665,7 @@
         }
 
         bindExternalTextureImage(texName, *cachedImage->second);
+        mTextureView.insert_or_assign(texName, buffer->getId());
     }
 
     // Wait for the new buffer to be ready.
@@ -885,7 +907,7 @@
     glBindFramebuffer(GL_FRAMEBUFFER, 0);
 }
 
-bool GLESRenderEngine::cleanupPostRender() {
+bool GLESRenderEngine::cleanupPostRender(CleanupMode mode) {
     ATRACE_CALL();
 
     if (mPriorResourcesCleaned ||
@@ -894,6 +916,30 @@
         return false;
     }
 
+    // This is a bit of a band-aid fix for FrameCaptureProcessor, as we should
+    // not need to keep memory around if we don't need to do so.
+    if (mode == CleanupMode::CLEAN_ALL) {
+        // TODO: SurfaceFlinger memory utilization may benefit from resetting
+        // texture bindings as well. Assess if it does and there's no performance regression
+        // when rebinding the same image data to the same texture, and if so then its mode
+        // behavior can be tweaked.
+        if (mPlaceholderImage != EGL_NO_IMAGE_KHR) {
+            for (auto [textureName, bufferId] : mTextureView) {
+                if (bufferId && mPlaceholderImage != EGL_NO_IMAGE_KHR) {
+                    glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureName);
+                    glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES,
+                                                 static_cast<GLeglImageOES>(mPlaceholderImage));
+                    mTextureView[textureName] = std::nullopt;
+                    checkErrors();
+                }
+            }
+        }
+        {
+            std::lock_guard<std::mutex> lock(mRenderingMutex);
+            mImageCache.clear();
+        }
+    }
+
     // Bind the texture to placeholder so that backing image data can be freed.
     GLFramebuffer* glFramebuffer = static_cast<GLFramebuffer*>(getFramebufferForDrawing());
     glFramebuffer->allocateBuffers(1, 1, mPlaceholderDrawBuffer);
@@ -1618,6 +1664,16 @@
     return cachedImage != mImageCache.end();
 }
 
+bool GLESRenderEngine::isTextureNameKnownForTesting(uint32_t texName) {
+    const auto& entry = mTextureView.find(texName);
+    return entry != mTextureView.end();
+}
+
+std::optional<uint64_t> GLESRenderEngine::getBufferIdForTextureNameForTesting(uint32_t texName) {
+    const auto& entry = mTextureView.find(texName);
+    return entry != mTextureView.end() ? entry->second : std::nullopt;
+}
+
 bool GLESRenderEngine::isFramebufferImageCachedForTesting(uint64_t bufferId) {
     std::lock_guard<std::mutex> lock(mFramebufferImageCacheMutex);
     return std::any_of(mFramebufferImageCache.cbegin(), mFramebufferImageCache.cend(),
diff --git a/libs/renderengine/gl/GLESRenderEngine.h b/libs/renderengine/gl/GLESRenderEngine.h
index 245bfd7..2c6eae2 100644
--- a/libs/renderengine/gl/GLESRenderEngine.h
+++ b/libs/renderengine/gl/GLESRenderEngine.h
@@ -75,7 +75,7 @@
                         const std::vector<const LayerSettings*>& layers,
                         const sp<GraphicBuffer>& buffer, const bool useFramebufferCache,
                         base::unique_fd&& bufferFence, base::unique_fd* drawFence) override;
-    bool cleanupPostRender() override;
+    bool cleanupPostRender(CleanupMode mode) override;
 
     EGLDisplay getEGLDisplay() const { return mEGLDisplay; }
     // Creates an output image for rendering to
@@ -86,6 +86,12 @@
     // Test-only methods
     // Returns true iff mImageCache contains an image keyed by bufferId
     bool isImageCachedForTesting(uint64_t bufferId) EXCLUDES(mRenderingMutex);
+    // Returns true iff texName was previously generated by RenderEngine and was
+    // not destroyed.
+    bool isTextureNameKnownForTesting(uint32_t texName);
+    // Returns the buffer ID of the content bound to texName, or nullopt if no
+    // such mapping exists.
+    std::optional<uint64_t> getBufferIdForTextureNameForTesting(uint32_t texName);
     // Returns true iff mFramebufferImageCache contains an image keyed by bufferId
     bool isFramebufferImageCachedForTesting(uint64_t bufferId)
             EXCLUDES(mFramebufferImageCacheMutex);
@@ -224,6 +230,8 @@
 
     // Cache of GL images that we'll store per GraphicBuffer ID
     std::unordered_map<uint64_t, std::unique_ptr<Image>> mImageCache GUARDED_BY(mRenderingMutex);
+    std::unordered_map<uint32_t, std::optional<uint64_t>> mTextureView;
+
     // Mutex guarding rendering operations, so that:
     // 1. GL operations aren't interleaved, and
     // 2. Internal state related to rendering that is potentially modified by
@@ -237,6 +245,11 @@
     // ensure that we align on a word. Allocating 16 bytes will provide a
     // guarantee that we don't clobber memory.
     uint32_t mPlaceholderDrawBuffer[4];
+    // Placeholder buffer and image, similar to mPlaceholderDrawBuffer, but
+    // instead these are intended for cleaning up texture memory with the
+    // GL_TEXTURE_EXTERNAL_OES target.
+    ANativeWindowBuffer* mPlaceholderBuffer = nullptr;
+    EGLImage mPlaceholderImage = EGL_NO_IMAGE_KHR;
     sp<Fence> mLastDrawFence;
     // Store a separate boolean checking if prior resources were cleaned up, as
     // devices that don't support native sync fences can't rely on a last draw
diff --git a/libs/renderengine/include/renderengine/RenderEngine.h b/libs/renderengine/include/renderengine/RenderEngine.h
index b137023..09a0f65 100644
--- a/libs/renderengine/include/renderengine/RenderEngine.h
+++ b/libs/renderengine/include/renderengine/RenderEngine.h
@@ -120,14 +120,25 @@
     // Returns NO_ERROR when binds successfully, NO_MEMORY when there's no memory for allocation.
     virtual status_t bindFrameBuffer(Framebuffer* framebuffer) = 0;
     virtual void unbindFrameBuffer(Framebuffer* framebuffer) = 0;
+
+    enum class CleanupMode {
+        CLEAN_OUTPUT_RESOURCES,
+        CLEAN_ALL,
+    };
     // Clean-up method that should be called on the main thread after the
     // drawFence returned by drawLayers fires. This method will free up
     // resources used by the most recently drawn frame. If the frame is still
     // being drawn, then this call is silently ignored.
     //
+    // If mode is CLEAN_OUTPUT_RESOURCES, then only resources related to the
+    // output framebuffer are cleaned up, including the sibling texture.
+    //
+    // If mode is CLEAN_ALL, then we also cleanup resources related to any input
+    // buffers.
+    //
     // Returns true if resources were cleaned up, and false if we didn't need to
     // do any work.
-    virtual bool cleanupPostRender() = 0;
+    virtual bool cleanupPostRender(CleanupMode mode = CleanupMode::CLEAN_OUTPUT_RESOURCES) = 0;
 
     // queries
     virtual size_t getMaxTextureSize() const = 0;
diff --git a/libs/renderengine/include/renderengine/mock/RenderEngine.h b/libs/renderengine/include/renderengine/mock/RenderEngine.h
index d0343ba..e03dd58 100644
--- a/libs/renderengine/include/renderengine/mock/RenderEngine.h
+++ b/libs/renderengine/include/renderengine/mock/RenderEngine.h
@@ -56,7 +56,7 @@
     MOCK_CONST_METHOD0(isProtected, bool());
     MOCK_CONST_METHOD0(supportsProtectedContent, bool());
     MOCK_METHOD1(useProtectedContext, bool(bool));
-    MOCK_METHOD0(cleanupPostRender, bool());
+    MOCK_METHOD1(cleanupPostRender, bool(CleanupMode mode));
     MOCK_METHOD6(drawLayers,
                  status_t(const DisplaySettings&, const std::vector<const LayerSettings*>&,
                           const sp<GraphicBuffer>&, const bool, base::unique_fd&&,
diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp
index 8ab2746..ba17143 100644
--- a/libs/renderengine/tests/RenderEngineTest.cpp
+++ b/libs/renderengine/tests/RenderEngineTest.cpp
@@ -83,6 +83,7 @@
         }
         for (uint32_t texName : mTexNames) {
             sRE->deleteTextures(1, &texName);
+            EXPECT_FALSE(sRE->isTextureNameKnownForTesting(texName));
         }
     }
 
@@ -1422,10 +1423,44 @@
     if (fd >= 0) {
         sync_wait(fd, -1);
     }
-
     // Only cleanup the first time.
-    EXPECT_TRUE(sRE->cleanupPostRender());
-    EXPECT_FALSE(sRE->cleanupPostRender());
+    EXPECT_TRUE(sRE->cleanupPostRender(
+            renderengine::RenderEngine::CleanupMode::CLEAN_OUTPUT_RESOURCES));
+    EXPECT_FALSE(sRE->cleanupPostRender(
+            renderengine::RenderEngine::CleanupMode::CLEAN_OUTPUT_RESOURCES));
+}
+
+TEST_F(RenderEngineTest, cleanupPostRender_whenCleaningAll_replacesTextureMemory) {
+    renderengine::DisplaySettings settings;
+    settings.physicalDisplay = fullscreenRect();
+    settings.clip = fullscreenRect();
+
+    std::vector<const renderengine::LayerSettings*> layers;
+    renderengine::LayerSettings layer;
+    layer.geometry.boundaries = fullscreenRect().toFloatRect();
+    BufferSourceVariant<ForceOpaqueBufferVariant>::fillColor(layer, 1.0f, 0.0f, 0.0f, this);
+    layer.alpha = 1.0;
+    layers.push_back(&layer);
+
+    base::unique_fd fence;
+    sRE->drawLayers(settings, layers, mBuffer, true, base::unique_fd(), &fence);
+
+    const int fd = fence.get();
+    if (fd >= 0) {
+        sync_wait(fd, -1);
+    }
+
+    uint64_t bufferId = layer.source.buffer.buffer->getId();
+    uint32_t texName = layer.source.buffer.textureName;
+    EXPECT_TRUE(sRE->isImageCachedForTesting(bufferId));
+    EXPECT_EQ(bufferId, sRE->getBufferIdForTextureNameForTesting(texName));
+
+    EXPECT_TRUE(sRE->cleanupPostRender(renderengine::RenderEngine::CleanupMode::CLEAN_ALL));
+
+    // Now check that our view of memory is good.
+    EXPECT_FALSE(sRE->isImageCachedForTesting(bufferId));
+    EXPECT_EQ(std::nullopt, sRE->getBufferIdForTextureNameForTesting(bufferId));
+    EXPECT_TRUE(sRE->isTextureNameKnownForTesting(texName));
 }
 
 } // namespace android
diff --git a/libs/renderengine/tests/RenderEngineThreadedTest.cpp b/libs/renderengine/tests/RenderEngineThreadedTest.cpp
index 69a0e19..97c7442 100644
--- a/libs/renderengine/tests/RenderEngineThreadedTest.cpp
+++ b/libs/renderengine/tests/RenderEngineThreadedTest.cpp
@@ -178,14 +178,20 @@
 }
 
 TEST_F(RenderEngineThreadedTest, cleanupPostRender_returnsFalse) {
-    EXPECT_CALL(*mRenderEngine, cleanupPostRender()).WillOnce(Return(false));
-    status_t result = mThreadedRE->cleanupPostRender();
+    EXPECT_CALL(*mRenderEngine,
+                cleanupPostRender(renderengine::RenderEngine::CleanupMode::CLEAN_ALL))
+            .WillOnce(Return(false));
+    status_t result =
+            mThreadedRE->cleanupPostRender(renderengine::RenderEngine::CleanupMode::CLEAN_ALL);
     ASSERT_EQ(false, result);
 }
 
 TEST_F(RenderEngineThreadedTest, cleanupPostRender_returnsTrue) {
-    EXPECT_CALL(*mRenderEngine, cleanupPostRender()).WillOnce(Return(true));
-    status_t result = mThreadedRE->cleanupPostRender();
+    EXPECT_CALL(*mRenderEngine,
+                cleanupPostRender(renderengine::RenderEngine::CleanupMode::CLEAN_ALL))
+            .WillOnce(Return(true));
+    status_t result =
+            mThreadedRE->cleanupPostRender(renderengine::RenderEngine::CleanupMode::CLEAN_ALL);
     ASSERT_EQ(true, result);
 }
 
diff --git a/libs/renderengine/threaded/RenderEngineThreaded.cpp b/libs/renderengine/threaded/RenderEngineThreaded.cpp
index ad61718..d4184fd 100644
--- a/libs/renderengine/threaded/RenderEngineThreaded.cpp
+++ b/libs/renderengine/threaded/RenderEngineThreaded.cpp
@@ -361,14 +361,14 @@
     return resultFuture.get();
 }
 
-bool RenderEngineThreaded::cleanupPostRender() {
+bool RenderEngineThreaded::cleanupPostRender(CleanupMode mode) {
     std::promise<bool> resultPromise;
     std::future<bool> resultFuture = resultPromise.get_future();
     {
         std::lock_guard lock(mThreadMutex);
-        mFunctionCalls.push([&resultPromise](renderengine::RenderEngine& instance) {
+        mFunctionCalls.push([&resultPromise, mode](renderengine::RenderEngine& instance) {
             ATRACE_NAME("REThreaded::cleanupPostRender");
-            bool returnValue = instance.cleanupPostRender();
+            bool returnValue = instance.cleanupPostRender(mode);
             resultPromise.set_value(returnValue);
         });
     }
@@ -400,4 +400,4 @@
 
 } // namespace threaded
 } // namespace renderengine
-} // namespace android
\ No newline at end of file
+} // namespace android
diff --git a/libs/renderengine/threaded/RenderEngineThreaded.h b/libs/renderengine/threaded/RenderEngineThreaded.h
index ec18e1f..86a49e9 100644
--- a/libs/renderengine/threaded/RenderEngineThreaded.h
+++ b/libs/renderengine/threaded/RenderEngineThreaded.h
@@ -62,7 +62,7 @@
     bool isProtected() const override;
     bool supportsProtectedContent() const override;
     bool useProtectedContext(bool useProtectedContext) override;
-    bool cleanupPostRender() override;
+    bool cleanupPostRender(CleanupMode mode) override;
 
     status_t drawLayers(const DisplaySettings& display,
                         const std::vector<const LayerSettings*>& layers,
@@ -94,4 +94,4 @@
 };
 } // namespace threaded
 } // namespace renderengine
-} // namespace android
\ No newline at end of file
+} // namespace android