spatial_segmentation: improve edge case handling

Change-Id: If47ac212044a977b12c8df1322759d173dd29ed5

av1/common/pred_common.h

@@ -21,8 +21,10 @@ extern "C" {
 #endif
 
 #if CONFIG_SPATIAL_SEGMENTATION
-/* Picks CDFs based on number of matching segment IDs */
+/* Picks CDFs based on number of matching/out-of-bounds segment IDs */
 static INLINE int pick_spatial_seg_cdf(int prev_ul, int prev_u, int prev_l) {
+  if (prev_ul < 0 || prev_u < 0 || prev_l < 0) /* Edge case */
+    return 0;
   if ((prev_ul == prev_u) && (prev_ul == prev_l))
     return 2;
   else if ((prev_ul == prev_u) || (prev_ul == prev_l) || (prev_u == prev_l))
@@ -31,8 +33,12 @@ static INLINE int pick_spatial_seg_cdf(int prev_ul, int prev_u, int prev_l) {
     return 0;
 }
 
+/* If 2 or more are identical returns that as predictor, otherwise prev_l */
 static INLINE int pick_spatial_seg_pred(int prev_ul, int prev_u, int prev_l) {
-  /* If 2 or more are identical returns that as predictor, otherwise prev_l */
+  if (prev_u == -1) /* Edge case */
+    return prev_l == -1 ? 0 : prev_l;
+  if (prev_l == -1) /* Edge case */
+    return prev_u;
   return (prev_ul == prev_u) ? prev_u : prev_l;
 }
 

av1/decoder/decodemv.c

@@ -350,9 +350,9 @@ static int read_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                            int mi_row, int mi_col, aom_reader *r, int skip) {
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
   struct segmentation_probs *const segp = &ec_ctx->seg;
-  int prev_ul = 0; /* Top left segment_id */
-  int prev_l = 0;  /* Current left segment_id */
-  int prev_u = 0;  /* Current top segment_id */
+  int prev_ul = -1; /* Top left segment_id */
+  int prev_l = -1;  /* Current left segment_id */
+  int prev_u = -1;  /* Current top segment_id */
 
   if ((xd->up_available) && (xd->left_available))
     prev_ul = get_segment_id(cm, cm->current_frame_seg_map, BLOCK_4X4,

av1/encoder/bitstream.c

@@ -593,9 +593,9 @@ static void write_segment_id(AV1_COMP *cpi, const MB_MODE_INFO *const mbmi,
                              int mi_col, int skip) {
   AV1_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
-  int prev_ul = 0; /* Top left segment_id */
-  int prev_l = 0;  /* Current left segment_id */
-  int prev_u = 0;  /* Current top segment_id */
+  int prev_ul = -1; /* Top left segment_id */
+  int prev_l = -1;  /* Current left segment_id */
+  int prev_u = -1;  /* Current top segment_id */
 
   if (!seg->enabled || !seg->update_map) return;