Merge "overlay: Fix rotator output buffer size."
diff --git a/liboverlay/overlayMdp.cpp b/liboverlay/overlayMdp.cpp
index 8b40ad7..8f2e2b6 100644
--- a/liboverlay/overlayMdp.cpp
+++ b/liboverlay/overlayMdp.cpp
@@ -241,7 +241,8 @@
     return true;
 }
 
-//Adjust width, height, format if rotator is used.
+//Adjust width, height if rotator is used post transform calcs.
+//At this point the format is already updated by updateSrcFormat
 void MdpCtrl::adjustSrcWhf(const bool& rotUsed) {
     if(rotUsed) {
         utils::Whf whf = getSrcWhf();
@@ -250,25 +251,15 @@
             whf.w = utils::alignup(whf.w, 64);
             whf.h = utils::alignup(whf.h, 32);
         }
-        /*For example: If original format is tiled, rotator outputs non-tiled,
-         *so update mdp's src fmt to that.
-         */
-        whf.format = utils::getRotOutFmt(whf.format);
         setSrcWhf(whf);
-        /* The above format will be overwritten in function updateSrcformat
-         * after doing rotator start. Format is then set depending on
-         * whether the fastyuv mode is used by the rotator.
-         */
     }
 }
 
+//Update src format if rotator used based on rotator's destination format.
 void MdpCtrl::updateSrcformat(const uint32_t& inputformat) {
-    int version = qdutils::MDPVersion::getInstance().getMDPVersion();
-    if ((version >= qdutils::MDP_V4_2) && (version < qdutils::MDSS_V5)) {
-        utils::Whf whf = getSrcWhf();
-        whf.format =  inputformat;
-        setSrcWhf(whf);
-    }
+    utils::Whf whf = getSrcWhf();
+    whf.format =  inputformat;
+    setSrcWhf(whf);
 }
 
 void MdpCtrl::dump() const {
diff --git a/liboverlay/overlayMdpRot.cpp b/liboverlay/overlayMdpRot.cpp
index dc8ef09..c9843f2 100755
--- a/liboverlay/overlayMdpRot.cpp
+++ b/liboverlay/overlayMdpRot.cpp
@@ -108,8 +108,6 @@
 
     mRotImgInfo.dst.width = whf.w;
     mRotImgInfo.dst.height = whf.h;
-
-    mBufSize = awhf.size;
 }
 
 inline void MdpRot::setFlags(const utils::eMdpFlags& flags) {
@@ -154,6 +152,12 @@
     return true;
 }
 
+uint32_t MdpRot::calcOutputBufSize() {
+    ovutils::Whf destWhf(mRotImgInfo.dst.width,
+            mRotImgInfo.dst.height, mRotImgInfo.dst.format);
+    return Rotator::calcOutputBufSize(destWhf);
+}
+
 bool MdpRot::open_i(uint32_t numbufs, uint32_t bufsz)
 {
     OvMem mem;
@@ -198,8 +202,9 @@
 
 bool MdpRot::remap(uint32_t numbufs) {
     // if current size changed, remap
-    if(mBufSize == mMem.curr().size()) {
-        ALOGE_IF(DEBUG_OVERLAY, "%s: same size %d", __FUNCTION__, mBufSize);
+    uint32_t opBufSize = calcOutputBufSize();
+    if(opBufSize == mMem.curr().size()) {
+        ALOGE_IF(DEBUG_OVERLAY, "%s: same size %d", __FUNCTION__, opBufSize);
         return true;
     }
 
@@ -208,12 +213,12 @@
 
     // ++mMem will make curr to be prev, and prev will be curr
     ++mMem;
-    if(!open_i(numbufs, mBufSize)) {
+    if(!open_i(numbufs, opBufSize)) {
         ALOGE("%s Error could not open", __FUNCTION__);
         return false;
     }
     for (uint32_t i = 0; i < numbufs; ++i) {
-        mMem.curr().mRotOffset[i] = i * mBufSize;
+        mMem.curr().mRotOffset[i] = i * opBufSize;
     }
     return true;
 }
@@ -226,7 +231,6 @@
     ovutils::memset0(mMem.prev().mRotOffset);
     mMem.curr().mCurrOffset = 0;
     mMem.prev().mCurrOffset = 0;
-    mBufSize = 0;
     mOrientation = utils::OVERLAY_TRANSFORM_0;
 }
 
diff --git a/liboverlay/overlayMdssRot.cpp b/liboverlay/overlayMdssRot.cpp
index 0da653c..fd747dd 100644
--- a/liboverlay/overlayMdssRot.cpp
+++ b/liboverlay/overlayMdssRot.cpp
@@ -96,8 +96,6 @@
 
     mRotInfo.dst_rect.w = whf.w;
     mRotInfo.dst_rect.h = whf.h;
-
-    mBufSize = awhf.size;
 }
 
 inline void MdssRot::setDownscale(int ds) {}
@@ -131,7 +129,6 @@
 
 bool MdssRot::commit() {
     doTransform();
-    setBufSize(mRotInfo.src.format);
     mRotInfo.flags |= MDSS_MDP_ROT_ONLY;
     if(!overlay::mdp_wrapper::setOverlay(mFd.getFD(), mRotInfo)) {
         ALOGE("MdssRot commit failed!");
@@ -199,9 +196,11 @@
 }
 
 bool MdssRot::remap(uint32_t numbufs) {
-    // if current size changed, remap
-    if(mBufSize == mMem.curr().size()) {
-        ALOGE_IF(DEBUG_OVERLAY, "%s: same size %d", __FUNCTION__, mBufSize);
+    // Calculate the size based on rotator's dst format, w and h.
+    uint32_t opBufSize = calcOutputBufSize();
+    // If current size changed, remap
+    if(opBufSize == mMem.curr().size()) {
+        ALOGE_IF(DEBUG_OVERLAY, "%s: same size %d", __FUNCTION__, opBufSize);
         return true;
     }
 
@@ -210,12 +209,12 @@
 
     // ++mMem will make curr to be prev, and prev will be curr
     ++mMem;
-    if(!open_i(numbufs, mBufSize)) {
+    if(!open_i(numbufs, opBufSize)) {
         ALOGE("%s Error could not open", __FUNCTION__);
         return false;
     }
     for (uint32_t i = 0; i < numbufs; ++i) {
-        mMem.curr().mRotOffset[i] = i * mBufSize;
+        mMem.curr().mRotOffset[i] = i * opBufSize;
     }
     return true;
 }
@@ -251,7 +250,6 @@
     ovutils::memset0(mMem.prev().mRotOffset);
     mMem.curr().mCurrOffset = 0;
     mMem.prev().mCurrOffset = 0;
-    mBufSize = 0;
     mOrientation = utils::OVERLAY_TRANSFORM_0;
 }
 
@@ -264,25 +262,17 @@
     ALOGE("== Dump MdssRot end ==");
 }
 
-void MdssRot::setBufSize(int format) {
+uint32_t MdssRot::calcOutputBufSize() {
+    uint32_t opBufSize = 0;
+    ovutils::Whf destWhf(mRotInfo.dst_rect.w, mRotInfo.dst_rect.h,
+            mRotInfo.src.format); //mdss src and dst formats are same.
 
-    switch (format) {
-        case MDP_Y_CBCR_H2V2_VENUS:
-            mBufSize = VENUS_BUFFER_SIZE(COLOR_FMT_NV12, mRotInfo.dst_rect.w,
-                                         mRotInfo.dst_rect.h);
-            break;
-
-        case MDP_Y_CR_CB_GH2V2:
-            int alignedw = utils::align(mRotInfo.dst_rect.w, 16);
-            int alignedh = mRotInfo.dst_rect.h;
-            mBufSize = (alignedw*alignedh) +
-                (utils::align(alignedw/2, 16) * (alignedh/2))*2;
-            mBufSize = utils::align(mBufSize, 4096);
-            break;
-    }
+    opBufSize = Rotator::calcOutputBufSize(destWhf);
 
     if (mRotInfo.flags & utils::OV_MDP_SECURE_OVERLAY_SESSION)
-        mBufSize = utils::align(mBufSize, SIZE_1M);
+        opBufSize = utils::align(opBufSize, SIZE_1M);
+
+    return opBufSize;
 }
 
 void MdssRot::getDump(char *buf, size_t len) const {
diff --git a/liboverlay/overlayRotator.cpp b/liboverlay/overlayRotator.cpp
index 956ea2b..90a1e7a 100644
--- a/liboverlay/overlayRotator.cpp
+++ b/liboverlay/overlayRotator.cpp
@@ -20,6 +20,7 @@
 #include "overlayRotator.h"
 #include "overlayUtils.h"
 #include "mdp_version.h"
+#include "gr.h"
 
 namespace ovutils = overlay::utils;
 
@@ -39,6 +40,15 @@
     }
 }
 
+uint32_t Rotator::calcOutputBufSize(const utils::Whf& destWhf) {
+    //dummy aligned w & h.
+    int alW = 0, alH = 0;
+    int halFormat = ovutils::getHALFormat(destWhf.format);
+    //A call into gralloc/memalloc
+    return getBufferSizeAndDimensions(
+            destWhf.w, destWhf.h, halFormat, alW, alH);
+}
+
 int Rotator::getRotatorHwType() {
     int mdpVersion = qdutils::MDPVersion::getInstance().getMDPVersion();
     if (mdpVersion == qdutils::MDSS_V5)
diff --git a/liboverlay/overlayRotator.h b/liboverlay/overlayRotator.h
index ee1610c..bd2fd7e 100644
--- a/liboverlay/overlayRotator.h
+++ b/liboverlay/overlayRotator.h
@@ -67,6 +67,7 @@
 
 protected:
     explicit Rotator() {}
+    static uint32_t calcOutputBufSize(const utils::Whf& destWhf);
 
 private:
     /*Returns rotator h/w type */
@@ -145,13 +146,14 @@
     void doTransform();
     /* reset underlying data, basically memset 0 */
     void reset();
-
     /* return true if current rotator config is different
      * than last known config */
     bool rotConfChanged() const;
-
     /* save mRotImgInfo to be last known good config*/
     void save();
+    /* Calculates the rotator's o/p buffer size post the transform calcs and
+     * knowing the o/p format depending on whether fastYuv is enabled or not */
+    uint32_t calcOutputBufSize();
 
     /* rot info*/
     msm_rotator_img_info mRotImgInfo;
@@ -165,8 +167,6 @@
     OvFD mFd;
     /* Rotator memory manager */
     RotMem mMem;
-    /* Single Rotator buffer size */
-    uint32_t mBufSize;
 
     friend Rotator* Rotator::getRotator();
 };
@@ -208,7 +208,9 @@
     void doTransform();
     /* reset underlying data, basically memset 0 */
     void reset();
-    void setBufSize(int format);
+    /* Calculates the rotator's o/p buffer size post the transform calcs and
+     * knowing the o/p format depending on whether fastYuv is enabled or not */
+    uint32_t calcOutputBufSize();
 
     /* MdssRot info structure */
     mdp_overlay   mRotInfo;
@@ -220,8 +222,6 @@
     OvFD mFd;
     /* Rotator memory manager */
     RotMem mMem;
-    /* Single Rotator buffer size */
-    uint32_t mBufSize;
     /* Enable/Disable Mdss Rot*/
     bool mEnabled;
 
diff --git a/liboverlay/overlayUtils.cpp b/liboverlay/overlayUtils.cpp
index 3880976..de10c9f 100644
--- a/liboverlay/overlayUtils.cpp
+++ b/liboverlay/overlayUtils.cpp
@@ -243,7 +243,50 @@
             //HAL_PIXEL_FORMAT_YCrCb_420_SP_ADRENO    = 0x7FA30C01
             //HAL_PIXEL_FORMAT_R_8                    = 0x10D
             //HAL_PIXEL_FORMAT_RG_88                  = 0x10E
-            ALOGE("%s: Unsupported format = 0x%x", __func__, format);
+            ALOGE("%s: Unsupported HAL format = 0x%x", __func__, format);
+            return -1;
+    }
+    // not reached
+    return -1;
+}
+
+//Takes mdp format as input and translates to equivalent HAL format
+//Refer to graphics.h, gralloc_priv.h, msm_mdp.h for formats.
+int getHALFormat(int mdpFormat) {
+    switch (mdpFormat) {
+        //From graphics.h
+        case MDP_RGBA_8888:
+            return HAL_PIXEL_FORMAT_RGBA_8888;
+        case MDP_RGBX_8888:
+            return HAL_PIXEL_FORMAT_RGBX_8888;
+        case MDP_RGB_888:
+            return HAL_PIXEL_FORMAT_RGB_888;
+        case MDP_RGB_565:
+            return HAL_PIXEL_FORMAT_RGB_565;
+        case MDP_BGRA_8888:
+            return HAL_PIXEL_FORMAT_BGRA_8888;
+        case MDP_Y_CR_CB_GH2V2:
+            return HAL_PIXEL_FORMAT_YV12;
+        case MDP_Y_CBCR_H2V1:
+            return HAL_PIXEL_FORMAT_YCbCr_422_SP;
+        case MDP_Y_CRCB_H2V2:
+            return HAL_PIXEL_FORMAT_YCrCb_420_SP;
+
+        //From gralloc_priv.h
+        case MDP_Y_CBCR_H2V2_TILE:
+            return HAL_PIXEL_FORMAT_YCbCr_420_SP_TILED;
+        case MDP_Y_CBCR_H2V2:
+            return HAL_PIXEL_FORMAT_YCbCr_420_SP;
+        case MDP_Y_CRCB_H2V1:
+            return HAL_PIXEL_FORMAT_YCrCb_422_SP;
+        case MDP_Y_CBCR_H1V1:
+            return HAL_PIXEL_FORMAT_YCbCr_444_SP;
+        case MDP_Y_CRCB_H1V1:
+            return HAL_PIXEL_FORMAT_YCrCb_444_SP;
+        case MDP_Y_CBCR_H2V2_VENUS:
+            return HAL_PIXEL_FORMAT_YCbCr_420_SP_VENUS;
+        default:
+            ALOGE("%s: Unsupported MDP format = 0x%x", __func__, mdpFormat);
             return -1;
     }
     // not reached
diff --git a/liboverlay/overlayUtils.h b/liboverlay/overlayUtils.h
index 946748e..45d1924 100644
--- a/liboverlay/overlayUtils.h
+++ b/liboverlay/overlayUtils.h
@@ -413,7 +413,8 @@
 };
 
 int getMdpFormat(int format);
-int getRotOutFmt(uint32_t format);
+int getHALFormat(int mdpFormat);
+
 /* flip is upside down and such. V, H flip
  * rotation is 90, 180 etc
  * It returns MDP related enum/define that match rot+flip*/
@@ -588,29 +589,6 @@
     return -1;
 }
 
-inline int getRotOutFmt(uint32_t format) {
-
-    if (isMdssRotator())
-        return format;
-
-    switch (format) {
-        case MDP_Y_CRCB_H2V2_TILE:
-            return MDP_Y_CRCB_H2V2;
-        case MDP_Y_CBCR_H2V2_TILE:
-            return MDP_Y_CBCR_H2V2;
-        case MDP_Y_CB_CR_H2V2:
-            return MDP_Y_CBCR_H2V2;
-        case MDP_Y_CR_CB_GH2V2:
-            return MDP_Y_CRCB_H2V2;
-        default:
-            return format;
-    }
-    // not reached
-    OVASSERT(false, "%s not reached", __FUNCTION__);
-    return -1;
-}
-
-
 inline uint32_t getColorFormat(uint32_t format)
 {
     return (format == HAL_PIXEL_FORMAT_YV12) ?