fix(render): change int to float to avoid precision loss

alibaba · Jun 5, 2020 · 526f199 · 526f199
1 parent 0becd1e
commit 526f199
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 50 deletions.
diff --git a/framework/render/video/glRender/CV420PProgramContext.cpp b/framework/render/video/glRender/CV420PProgramContext.cpp
@@ -322,41 +322,41 @@ void CV420PProgramContext::updateDrawRegion() {
         return;
     }
 
-    int windowWidth = mWindowWidth;
-    int windowHeight = mWindowHeight;
-    int off_x = 0;
-    int off_y = 0;
-    int w = mWindowWidth;
-    int h = mWindowHeight;
-    int realWidth = 0;
-    int realHeight = 0;
+    float windowWidth = mWindowWidth;
+    float windowHeight = mWindowHeight;
+    float off_x = 0;
+    float off_y = 0;
+    float w = mWindowWidth;
+    float h = mWindowHeight;
+    float realWidth = 0;
+    float realHeight = 0;
 
     if (mRotate == IVideoRender::Rotate::Rotate_90 ||
         mRotate == IVideoRender::Rotate::Rotate_270) {
         realWidth = mFrameHeight;
-        realHeight = static_cast<int>(mFrameHeight * mDar);
+        realHeight = static_cast<float>(mFrameHeight * mDar);
     } else {
-        realWidth = static_cast<int>(mFrameHeight * mDar);
+        realWidth = static_cast<float>(mFrameHeight * mDar);
         realHeight = mFrameHeight;
     }
 
-    float scale_w = windowWidth * 1.0f / realWidth;
-    float scale_h = windowHeight * 1.0f / realHeight;
+    float scale_w = windowWidth / realWidth;
+    float scale_h = windowHeight / realHeight;
 
     if (mScale == IVideoRender::Scale::Scale_AspectFit) {
         if (scale_w >= scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w) / 2;
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h) / 2;
         }
     } else if (mScale == IVideoRender::Scale::Scale_AspectFill) {
         if (scale_w < scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w) / 2;
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h) / 2;
         }
     }

diff --git a/framework/render/video/glRender/OESProgramContext.cpp b/framework/render/video/glRender/OESProgramContext.cpp
@@ -153,47 +153,47 @@ void OESProgramContext::updateDrawRegion() {
         return;
     }
 
-    int windowWidth = mWindowWidth;
-    int windowHeight = mWindowHeight;
-    int off_x = 0;
-    int off_y = 0;
-    int w = mWindowWidth;
-    int h = mWindowHeight;
-    int realWidth = 0;
-    int realHeight = 0;
+    float windowWidth = mWindowWidth;
+    float windowHeight = mWindowHeight;
+    float off_x = 0;
+    float off_y = 0;
+    float w = mWindowWidth;
+    float h = mWindowHeight;
+    float realWidth = 0;
+    float realHeight = 0;
 
     if (mRotate == IVideoRender::Rotate::Rotate_90 ||
         mRotate == IVideoRender::Rotate::Rotate_270) {
         realWidth = mFrameHeight;
-        realHeight = static_cast<int>(mFrameHeight * mDar);
+        realHeight = static_cast<float>(mFrameHeight * mDar);
     } else {
-        realWidth = static_cast<int>(mFrameHeight * mDar);
+        realWidth = static_cast<float>(mFrameHeight * mDar);
         realHeight = mFrameHeight;
     }
 
-    float scale_w = windowWidth * 1.0f / realWidth;
-    float scale_h = windowHeight * 1.0f / realHeight;
+    float scale_w = windowWidth / realWidth;
+    float scale_h = windowHeight / realHeight;
 
     if (mScale == IVideoRender::Scale::Scale_AspectFit) {
         if (scale_w >= scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w);
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h);
         }
     } else if (mScale == IVideoRender::Scale::Scale_AspectFill) {
         if (scale_w < scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w);
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h);
         }
     }
 
-    float offX = off_x * 1.0f / windowWidth;
-    float offY = off_y * 1.0f / windowHeight;
+    float offX = off_x / windowWidth;
+    float offY = off_y / windowHeight;
 
     if (mRotate == IVideoRender::Rotate::Rotate_None) {
         mDrawRegion[0] = (GLfloat) 1.0f - offX;

diff --git a/framework/render/video/glRender/YUVProgramContext.cpp b/framework/render/video/glRender/YUVProgramContext.cpp
@@ -268,41 +268,41 @@ void YUVProgramContext::updateDrawRegion() {
         return;
     }
 
-    int windowWidth = mWindowWidth;
-    int windowHeight = mWindowHeight;
-    int off_x = 0;
-    int off_y = 0;
-    int w = mWindowWidth;
-    int h = mWindowHeight;
-    int realWidth = 0;
-    int realHeight = 0;
+    float windowWidth = mWindowWidth;
+    float windowHeight = mWindowHeight;
+    float off_x = 0;
+    float off_y = 0;
+    float w = mWindowWidth;
+    float h = mWindowHeight;
+    float realWidth = 0;
+    float realHeight = 0;
 
     if (mRotate == IVideoRender::Rotate::Rotate_90 ||
         mRotate == IVideoRender::Rotate::Rotate_270) {
         realWidth = mFrameHeight;
-        realHeight = static_cast<int>(mFrameHeight * mDar);
+        realHeight = static_cast<float>(mFrameHeight * mDar);
     } else {
-        realWidth = static_cast<int>(mFrameHeight * mDar);
+        realWidth = static_cast<float>(mFrameHeight * mDar);
         realHeight = mFrameHeight;
     }
 
-    float scale_w = windowWidth * 1.0f / realWidth;
-    float scale_h = windowHeight * 1.0f / realHeight;
+    float scale_w = windowWidth / realWidth;
+    float scale_h = windowHeight  / realHeight;
 
     if (mScale == IVideoRender::Scale::Scale_AspectFit) {
         if (scale_w >= scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w) / 2;
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h) / 2;
         }
     } else if (mScale == IVideoRender::Scale::Scale_AspectFill) {
         if (scale_w < scale_h) {
-            w = static_cast<int>(scale_h * realWidth);
+            w = scale_h * realWidth;
             off_x = (windowWidth - w) / 2;
         } else {
-            h = static_cast<int>(scale_w * realHeight);
+            h = scale_w * realHeight;
             off_y = (windowHeight - h) / 2;
         }
     }