Fix stack corruption with high rate SILK encoding added by the recent CBR...

Fix stack corruption with high rate SILK encoding added by the recent CBR changes and enable -fstack-protector-all on gcc autotools builds.

configure.ac

@@ -172,7 +172,7 @@ AC_SUBST(SYMBOL_VISIBILITY)
 fi
 
 if test $ac_cv_c_compiler_gnu = yes ; then
-        CFLAGS="$CFLAGS -W -Wstrict-prototypes -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow"
+        CFLAGS="$CFLAGS -fstack-protector-all -W -Wstrict-prototypes -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow"
 fi
 
 AC_CHECK_FUNCS([lrintf])

silk/fixed/encode_frame_FIX.c

@@ -96,7 +96,7 @@ opus_int silk_encode_frame_FIX(
     opus_int16   ec_prevLagIndex_copy;
     opus_int     ec_prevSignalType_copy;
     opus_int8    LastGainIndex_copy2;
-    opus_uint8   ec_buf_copy[ 512 ];
+    opus_uint8   ec_buf_copy[ 1275 ];
 
 TIC(ENCODE_FRAME)
 
@@ -231,6 +231,7 @@ TOC(ENCODE_PULSES)
                 if( nBits > maxBits && found_lower ) {
                     /* Restore output state from earlier iteration that did meet the bitrate budget */
                     silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) );
+                    silk_assert( sRangeEnc_copy2.offs<=1275 );
                     silk_memcpy( psRangeEnc->buf, ec_buf_copy, sRangeEnc_copy2.offs );
                     silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy2, sizeof( silk_nsq_state ) );
                     psEnc->sShape.LastGainIndex = LastGainIndex_copy2;
@@ -252,6 +253,7 @@ TOC(ENCODE_PULSES)
                 gainMult_lower = gainMult_Q8;
                 /* Copy part of the output state */
                 silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) );
+                silk_assert( psRangeEnc->offs<=1275 );
                 silk_memcpy( ec_buf_copy, psRangeEnc->buf, psRangeEnc->offs );
                 silk_memcpy( &sNSQ_copy2, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) );
                 LastGainIndex_copy2 = psEnc->sShape.LastGainIndex;
@@ -262,12 +264,12 @@ TOC(ENCODE_PULSES)
 
             if( ( found_lower & found_upper ) == 0 ) {
                 /* Adjust gain according to high-rate rate/distortion curve */
-                opus_int32 gain_factor_Q16;
-                gain_factor_Q16 = silk_log2lin( silk_LSHIFT( nBits - maxBits, 7 ) / psEnc->sCmn.frame_length + SILK_FIX_CONST( 16, 7 ) );
+                opus_int32 gain_factor_Q16;
+                gain_factor_Q16 = silk_log2lin( silk_LSHIFT( nBits - maxBits, 7 ) / psEnc->sCmn.frame_length + SILK_FIX_CONST( 16, 7 ) );
                 if( nBits > maxBits ) {
                     gain_factor_Q16 = silk_max_32( gain_factor_Q16, SILK_FIX_CONST( 1.3, 16 ) );
                 }
-                gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 );
+                gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 );
             } else {
                 /* Adjust gain by interpolating */
                 gainMult_Q8 = gainMult_lower + silk_DIV32_16( silk_MUL( gainMult_upper - gainMult_lower, maxBits - nBits_lower ), nBits_upper - nBits_lower );

silk/float/encode_frame_FLP.c

@@ -100,7 +100,7 @@ opus_int silk_encode_frame_FLP(
     opus_int     ec_prevSignalType_copy;
     opus_int8    LastGainIndex_copy2;
     opus_int32   pGains_Q16[ MAX_NB_SUBFR ];
-    opus_uint8   ec_buf_copy[ 512 ];
+    opus_uint8   ec_buf_copy[ 1275 ];
 
 TIC(ENCODE_FRAME)
 
@@ -225,6 +225,7 @@ TOC(ENCODE_PULSES)
                 if( nBits > maxBits && found_lower ) {
                     /* Restore output state from earlier iteration that did meet the bitrate budget */
                    silk_memcpy( psRangeEnc, &sRangeEnc_copy2, sizeof( ec_enc ) );
+                   silk_assert( sRangeEnc_copy2.offs<=1275 );
                    silk_memcpy( psRangeEnc->buf, ec_buf_copy, sRangeEnc_copy2.offs );
                    silk_memcpy( &psEnc->sCmn.sNSQ, &sNSQ_copy2, sizeof( silk_nsq_state ) );
                    psEnc->sShape.LastGainIndex = LastGainIndex_copy2;
@@ -246,6 +247,7 @@ TOC(ENCODE_PULSES)
                 gainMult_lower = gainMult_Q8;
                 /* Copy part of the output state */
                 silk_memcpy( &sRangeEnc_copy2, psRangeEnc, sizeof( ec_enc ) );
+                silk_assert( psRangeEnc->offs<=1275 );
                 silk_memcpy( ec_buf_copy, psRangeEnc->buf, psRangeEnc->offs );
                 silk_memcpy( &sNSQ_copy2, &psEnc->sCmn.sNSQ, sizeof( silk_nsq_state ) );
                 LastGainIndex_copy2 = psEnc->sShape.LastGainIndex;
@@ -256,13 +258,13 @@ TOC(ENCODE_PULSES)
 
             if( ( found_lower & found_upper ) == 0 ) {
                 /* Adjust gain according to high-rate rate/distortion curve */
-                opus_int32 gain_factor_Q16;
+                opus_int32 gain_factor_Q16;
                 gain_factor_Q16 = silk_log2lin( silk_LSHIFT( nBits - maxBits, 7 ) / psEnc->sCmn.frame_length + SILK_FIX_CONST( 16, 7 ) );
-                gain_factor_Q16 = silk_min_32(gain_factor_Q16, SILK_FIX_CONST( 2, 16 ) );
+                gain_factor_Q16 = silk_min_32(gain_factor_Q16, SILK_FIX_CONST( 2, 16 ) );
                 if( nBits > maxBits ) {
                     gain_factor_Q16 = silk_max_32( gain_factor_Q16, SILK_FIX_CONST( 1.3, 16 ) );
                 }
-                gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 );
+                gainMult_Q8 = silk_SMULWB( gain_factor_Q16, gainMult_Q8 );
             } else {
                 /* Adjust gain by interpolating */
                 gainMult_Q8 = gainMult_lower + ( ( gainMult_upper - gainMult_lower ) * ( maxBits - nBits_lower ) ) / ( nBits_upper - nBits_lower );