Fix integer overflow.

Found using the undefined behaviour sanitizer (-fsanitize=undefined).

Change-Id: I5534a97c0e38f76b38c6a53b42759b0b676e360a

av1/encoder/aq_complexity.c

@@ -127,8 +127,9 @@ void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs,
   } else {
     // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
     // It is converted to bits * 256 units.
-    const int target_rate = (cpi->rc.sb64_target_rate * xmis * ymis * 256) /
-                            (cm->mib_size * cm->mib_size);
+    const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256;
+    const int denom = cm->mib_size * cm->mib_size;
+    const int target_rate = (int)(num / denom);
     double logvar;
     double low_var_thresh;
     const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth);