Add clamping to local warp parameters

Change-Id: I1665b3812726fd40326a4ac3da79dc316529e17e

av1/common/mv.h

@@ -43,6 +43,11 @@ typedef struct mv32 {
 #define WARPEDMODEL_PREC_BITS 16
 #define WARPEDMODEL_ROW3HOMO_PREC_BITS 16
 
+#define WARPEDMODEL_TRANS_CLAMP (128 << WARPEDMODEL_PREC_BITS)
+#define WARPEDMODEL_DIAGAFFINE_CLAMP (1 << (WARPEDMODEL_PREC_BITS + 1))
+#define WARPEDMODEL_NONDIAGAFFINE_CLAMP (1 << (WARPEDMODEL_PREC_BITS - 1))
+#define WARPEDMODEL_ROW3HOMO_CLAMP (1 << (WARPEDMODEL_PREC_BITS - 1))
+
 // Bits of subpel precision for warped interpolation
 #define WARPEDPIXEL_PREC_BITS 6
 #define WARPEDPIXEL_PREC_SHIFTS (1 << WARPEDPIXEL_PREC_BITS)

av1/common/warped_motion.c

@@ -1445,6 +1445,20 @@ static int find_affine_int(const int np, int *pts1, int *pts2, BLOCK_SIZE bsize,
   wm->wmmat[1] = ROUND_POWER_OF_TWO_SIGNED(v, 3);
 
   wm->wmmat[6] = wm->wmmat[7] = 0;
+
+  // Clamp values
+  wm->wmmat[0] = clamp(wm->wmmat[0], -WARPEDMODEL_TRANS_CLAMP,
+                       WARPEDMODEL_TRANS_CLAMP - 1);
+  wm->wmmat[1] = clamp(wm->wmmat[1], -WARPEDMODEL_TRANS_CLAMP,
+                       WARPEDMODEL_TRANS_CLAMP - 1);
+  wm->wmmat[2] = clamp(wm->wmmat[2], -WARPEDMODEL_DIAGAFFINE_CLAMP,
+                       WARPEDMODEL_DIAGAFFINE_CLAMP - 1);
+  wm->wmmat[5] = clamp(wm->wmmat[5], -WARPEDMODEL_DIAGAFFINE_CLAMP,
+                       WARPEDMODEL_DIAGAFFINE_CLAMP - 1);
+  wm->wmmat[3] = clamp(wm->wmmat[3], -WARPEDMODEL_NONDIAGAFFINE_CLAMP,
+                       WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1);
+  wm->wmmat[4] = clamp(wm->wmmat[4], -WARPEDMODEL_NONDIAGAFFINE_CLAMP,
+                       WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1);
   return 0;
 }
 
@@ -1588,6 +1602,20 @@ static int find_affine_int(const int np, int *pts1, int *pts2, BLOCK_SIZE bsize,
   wm->wmmat[1] += ROUND_POWER_OF_TWO_SIGNED(off, 3);
   wm->wmmat[6] = wm->wmmat[7] = 0;
 
+  // Clamp values
+  wm->wmmat[0] = clamp(wm->wmmat[0], -WARPEDMODEL_TRANS_CLAMP,
+                       WARPEDMODEL_TRANS_CLAMP - 1);
+  wm->wmmat[1] = clamp(wm->wmmat[1], -WARPEDMODEL_TRANS_CLAMP,
+                       WARPEDMODEL_TRANS_CLAMP - 1);
+  wm->wmmat[2] = clamp(wm->wmmat[2], -WARPEDMODEL_DIAGAFFINE_CLAMP,
+                       WARPEDMODEL_DIAGAFFINE_CLAMP - 1);
+  wm->wmmat[5] = clamp(wm->wmmat[5], -WARPEDMODEL_DIAGAFFINE_CLAMP,
+                       WARPEDMODEL_DIAGAFFINE_CLAMP - 1);
+  wm->wmmat[3] = clamp(wm->wmmat[3], -WARPEDMODEL_NONDIAGAFFINE_CLAMP,
+                       WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1);
+  wm->wmmat[4] = clamp(wm->wmmat[4], -WARPEDMODEL_NONDIAGAFFINE_CLAMP,
+                       WARPEDMODEL_NONDIAGAFFINE_CLAMP - 1);
+
   return 0;
 }
 #endif  // LEAST_SQUARES_ORDER == 2