Port r15532 and r16552 from libvorbis.

Fix for bug #1456-- the 'bulletproofing' from CVE-2008-1420 inadvertantly 
 rejects a harmless/legal (if suboptimal) codebook arrangement that was 
 apparently used in 1.0b1.
Modify fix for Trac #1572; some files from the earliest beta
 accidentally used an oversized phrasebook in res decode; allow these.


git-svn-id: https://svn.xiph.org/trunk/Tremor@17520 0101bb08-14d6-0310-b084-bc0e0c8e3800

res012.c

@@ -115,6 +115,10 @@ vorbis_info_residue *res0_unpack(vorbis_info *vi,oggpack_buffer *opb){
 
   /* verify the phrasebook is not specifying an impossible or
      inconsistent partitioning scheme. */
+  /* modify the phrasebook ranging check from r16327; an early beta
+     encoder had a bug where it used an oversized phrasebook by
+     accident.  These files should continue to be playable, but don't
+     allow an exploit */
   {
     int entries = ci->book_param[info->groupbook]->entries;
     int dim = ci->book_param[info->groupbook]->dim;
@@ -124,7 +128,7 @@ vorbis_info_residue *res0_unpack(vorbis_info *vi,oggpack_buffer *opb){
       if(partvals > entries) goto errout;
       dim--;
     }
-    if(partvals != entries) goto errout;
+    info->partvals = partvals;
   }
 
   return(info);
@@ -222,7 +226,7 @@ static int _01inverse(vorbis_block *vb,vorbis_look_residue *vl,
 	  /* fetch the partition word for each channel */
 	  for(j=0;j<ch;j++){
 	    int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-	    if(temp==-1)goto eopbreak;
+	    if(temp==-1 || temp>=info->partvals)goto eopbreak;
 	    partword[j][l]=look->decodemap[temp];
 	    if(partword[j][l]==NULL)goto errout;
 	  }
@@ -304,7 +308,7 @@ int res2_inverse(vorbis_block *vb,vorbis_look_residue *vl,
 	if(s==0){
 	  /* fetch the partition word */
 	  int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-	  if(temp==-1)goto eopbreak;
+	  if(temp==-1 || temp>info->partvals)goto eopbreak;
 	  partword[l]=look->decodemap[temp];
 	  if(partword[l]==NULL)goto errout;
 	}