leb128: Add failure test.

Add tests of expected failure conditions, and handle invalid pad
sizes in the fixed length encode function.

BUG=aomedia:1125

Change-Id: I8b9d0fbc44bb776bca3b4c3f0467a2125f13be2e

aom/src/aom_integer.c

@@ -57,6 +57,7 @@ int aom_uleb_encode_fixed_size(uint32_t value, size_t available,
                                size_t pad_to_size, uint8_t *coded_value,
                                size_t *coded_size) {
   if (!coded_value || !coded_size || available < pad_to_size ||
+      pad_to_size > kMaximumLeb128Size ||
       aom_uleb_encode(value, available, coded_value, coded_size) != 0) {
     return -1;
   }

test/aom_integer_test.cc

@@ -103,3 +103,27 @@ TEST(AomLeb128, SizeTest) {
               aom_uleb_size_in_bytes(kSizeTestInputs[i]));
   }
 }
+
+TEST(AomLeb128, FailTest) {
+  const size_t kWriteBufferSize = 4;
+  const uint32_t kValidTestValue = 1;
+  uint8_t write_buffer[kWriteBufferSize] = { 0 };
+  size_t coded_size = 0;
+  ASSERT_EQ(
+      aom_uleb_encode(kValidTestValue, kWriteBufferSize, NULL, &coded_size),
+      -1);
+  ASSERT_EQ(aom_uleb_encode(kValidTestValue, kWriteBufferSize, &write_buffer[0],
+                            NULL),
+            -1);
+
+  const uint32_t kValueOutOfRange = 0xFFFFFFFF;
+  ASSERT_EQ(aom_uleb_encode(kValueOutOfRange, kWriteBufferSize,
+                            &write_buffer[0], &coded_size),
+            -1);
+
+  const size_t kPadSizeOutOfRange = 5;
+  ASSERT_EQ(aom_uleb_encode_fixed_size(kValidTestValue, kWriteBufferSize,
+                                       kPadSizeOutOfRange, &write_buffer[0],
+                                       &coded_size),
+            -1);
+}