Prevent illegal signaling of gm types above AFFINE

BUG=aomedia:693

Change-Id: I09b34abc41ee6f85619f5a05f668e06491e542f0

av1/common/mv.h

@@ -88,10 +88,12 @@ typedef enum {
 // GLOBAL_TRANS_TYPES 7 - up to full homography
 #define GLOBAL_TRANS_TYPES 4
 
+#if GLOBAL_TRANS_TYPES > 4
 // First bit indicates whether using identity or not
 // GLOBAL_TYPE_BITS=ceiling(log2(GLOBAL_TRANS_TYPES-1)) is the
 // number of bits needed to cover the remaining possibilities
 #define GLOBAL_TYPE_BITS (get_msb(2 * GLOBAL_TRANS_TYPES - 3))
+#endif  // GLOBAL_TRANS_TYPES > 4
 
 typedef struct {
 #if CONFIG_GLOBAL_MOTION

av1/decoder/decodeframe.c

@@ -4750,7 +4750,17 @@ static void read_global_motion_params(WarpedMotionParams *params,
                                       WarpedMotionParams *ref_params,
                                       aom_reader *r, int allow_hp) {
   TransformationType type = aom_read_bit(r, ACCT_STR);
-  if (type != IDENTITY) type += aom_read_literal(r, GLOBAL_TYPE_BITS, ACCT_STR);
+  if (type != IDENTITY) {
+#if GLOBAL_TRANS_TYPES > 4
+    type += aom_read_literal(r, GLOBAL_TYPE_BITS, ACCT_STR);
+#else
+    if (aom_read_bit(r, ACCT_STR))
+      type = ROTZOOM;
+    else
+      aom_read_bit(r, ACCT_STR) ? TRANSLATION : AFFINE;
+#endif  // GLOBAL_TRANS_TYPES > 4
+  }
+
   int trans_bits;
   int trans_dec_factor;
   int trans_prec_diff;

av1/encoder/bitstream.c

@@ -4535,7 +4535,14 @@ static void write_global_motion_params(WarpedMotionParams *params,
   int trans_bits;
   int trans_prec_diff;
   aom_write_bit(w, type != IDENTITY);
-  if (type != IDENTITY) aom_write_literal(w, type - 1, GLOBAL_TYPE_BITS);
+  if (type != IDENTITY) {
+#if GLOBAL_TRANS_TYPES > 4
+    aom_write_literal(w, type - 1, GLOBAL_TYPE_BITS);
+#else
+    aom_write_bit(w, type == ROTZOOM);
+    if (type != ROTZOOM) aom_write_bit(w, type == TRANSLATION);
+#endif  // GLOBAL_TRANS_TYPES > 4
+  }
 
   switch (type) {
     case HOMOGRAPHY:

av1/encoder/rd.c

@@ -569,8 +569,17 @@ void av1_initialize_rd_consts(AV1_COMP *cpi) {
 #if CONFIG_GLOBAL_MOTION
   if (cpi->oxcf.pass != 1) {
     for (int i = 0; i < TRANS_TYPES; ++i)
+#if GLOBAL_TRANS_TYPES > 4
       cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                             << AV1_PROB_COST_SHIFT;
+#else
+      // IDENTITY: 1 bit
+      // TRANSLATION: 3 bits
+      // ROTZOOM: 2 bits
+      // AFFINE: 3 bits
+      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
+                            << AV1_PROB_COST_SHIFT;
+#endif  // GLOBAL_TRANS_TYPES > 4
   }
 #endif  // CONFIG_GLOBAL_MOTION
 }