Limiting intra-frame prediction codebook to 32 entries (plus sign)

libcelt/celt.c

@@ -340,6 +340,8 @@ int celt_encode(CELTEncoder *st, celt_int16_t *pcm, unsigned char *compressed, i
       }
    }
    
+   if (ec_enc_tell(&st->enc, 0) < nbCompressedBytes*8 - 8)
+      celt_warning_int ("too make unused bits", nbCompressedBytes*8-ec_enc_tell(&st->enc, 0));
    //printf ("%d\n", ec_enc_tell(&st->enc, 0)-8*nbCompressedBytes);
    /* Finishing the stream with a 0101... pattern so that the decoder can check is everything's right */
    {

libcelt/rate.c

@@ -124,7 +124,12 @@ void alloc_init(struct alloc_data *alloc, const CELTMode *m)
                done = 1;
             /* Add the intra-frame prediction bits */
             if (eBands[i] >= m->pitchEnd)
-               alloc->bits[i][j] += (1<<BITRES) + log2_frac64(2*eBands[i]-eBands[i+1],BITRES);
+            {
+               int max_pos = 2*eBands[i]-eBands[i+1];
+               if (max_pos > 32)
+                  max_pos = 32;
+               alloc->bits[i][j] += (1<<BITRES) + log2_frac(max_pos,BITRES);
+            }
             /* We could just update rev_bits here */
             if (done)
                break;

libcelt/vq.c

@@ -271,7 +271,11 @@ void intra_prediction(float *x, float *W, int N, int K, float *Y, float *P, int
    float s = 1;
    int sign;
    float E;
-   for (i=0;i<N0*B-N;i+=B)
+   int max_pos = N0-N/B;
+   if (max_pos > 32)
+      max_pos = 32;
+
+   for (i=0;i<max_pos*B;i+=B)
    {
       int j;
       float xy=0, yy=0;
@@ -298,7 +302,7 @@ void intra_prediction(float *x, float *W, int N, int K, float *Y, float *P, int
       sign = 0;
    //printf ("%d %d ", sign, best);
    ec_enc_uint(enc,sign,2);
-   ec_enc_uint(enc,best/B,N0-N/B);
+   ec_enc_uint(enc,best/B,max_pos);
    //printf ("%d %f\n", best, best_score);
    
    float pred_gain;
@@ -335,13 +339,17 @@ void intra_unquant(float *x, int N, int K, float *Y, float *P, int B, int N0, ec
    float s;
    int best;
    float E;
+   int max_pos = N0-N/B;
+   if (max_pos > 32)
+      max_pos = 32;
+   
    sign = ec_dec_uint(dec, 2);
    if (sign == 0)
       s = 1;
    else
       s = -1;
    
-   best = B*ec_dec_uint(dec, N0-N/B);
+   best = B*ec_dec_uint(dec, max_pos);
    //printf ("%d %d ", sign, best);
 
    float pred_gain;