WIP: Using LPCNet for PLC

Makefile.am

@@ -10,9 +10,11 @@ lib_LTLIBRARIES = libopus.la
 DIST_SUBDIRS = doc
 
 AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/celt -I$(top_srcdir)/silk \
-              -I$(top_srcdir)/silk/float -I$(top_srcdir)/silk/fixed $(NE10_CFLAGS)
+              -I$(top_srcdir)/silk/float -I$(top_srcdir)/silk/fixed $(NE10_CFLAGS) \
+              -I$(top_srcdir)/lpcnet/include
 
 include celt_sources.mk
+include lpcnet_sources.mk
 include silk_sources.mk
 include opus_sources.mk
 
@@ -83,7 +85,7 @@ include celt_headers.mk
 include silk_headers.mk
 include opus_headers.mk
 
-libopus_la_SOURCES = $(CELT_SOURCES) $(SILK_SOURCES) $(OPUS_SOURCES)
+libopus_la_SOURCES = $(CELT_SOURCES) $(SILK_SOURCES) $(LPCNET_SOURCES) $(OPUS_SOURCES)
 libopus_la_LDFLAGS = -no-undefined -version-info @OPUS_LT_CURRENT@:@OPUS_LT_REVISION@:@OPUS_LT_AGE@
 libopus_la_LIBADD = $(NE10_LIBS) $(LIBM)
 if OPUS_ARM_EXTERNAL_ASM
@@ -157,16 +159,17 @@ tests_test_opus_padding_LDADD = libopus.la $(NE10_LIBS) $(LIBM)
 
 CELT_OBJ = $(CELT_SOURCES:.c=.lo)
 SILK_OBJ = $(SILK_SOURCES:.c=.lo)
+LPCNET_OBJ = $(LPCNET_SOURCES:.c=.lo)
 OPUS_OBJ = $(OPUS_SOURCES:.c=.lo)
 
 tests_test_opus_projection_SOURCES = tests/test_opus_projection.c tests/test_opus_common.h
-tests_test_opus_projection_LDADD = $(OPUS_OBJ) $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
+tests_test_opus_projection_LDADD = $(OPUS_OBJ) $(SILK_OBJ) $(LPCNET_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
 if OPUS_ARM_EXTERNAL_ASM
 tests_test_opus_projection_LDADD += libarmasm.la
 endif
 
 silk_tests_test_unit_LPC_inv_pred_gain_SOURCES = silk/tests/test_unit_LPC_inv_pred_gain.c
-silk_tests_test_unit_LPC_inv_pred_gain_LDADD = $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
+silk_tests_test_unit_LPC_inv_pred_gain_LDADD = $(SILK_OBJ) $(LPCNET_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
 if OPUS_ARM_EXTERNAL_ASM
 silk_tests_test_unit_LPC_inv_pred_gain_LDADD += libarmasm.la
 endif

lpcnet_sources.mk

+LPCNET_SOURCES = \
+lpcnet/src/ceps_codebooks.c \
+lpcnet/src/common.c \
+lpcnet/src/freq.c \
+lpcnet/src/kiss99.c \
+lpcnet/src/lpcnet.c \
+lpcnet/src/lpcnet_dec.c \
+lpcnet/src/lpcnet_enc.c \
+lpcnet/src/lpcnet_plc.c \
+lpcnet/src/nnet.c \
+lpcnet/src/nnet_data.c

silk/PLC.c

@@ -33,6 +33,10 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "stack_alloc.h"
 #include "PLC.h"
 
+#ifdef NEURAL_PLC
+#include "lpcnet.h"
+#endif
+
 #define NB_ATT 2
 static const opus_int16 HARM_ATT_Q15[NB_ATT]              = { 32440, 31130 }; /* 0.99, 0.95 */
 static const opus_int16 PLC_RAND_ATTENUATE_V_Q15[NB_ATT]  = { 31130, 26214 }; /* 0.95, 0.8 */
@@ -60,6 +64,14 @@ void silk_PLC_Reset(
     psDec->sPLC.prevGain_Q16[ 1 ] = SILK_FIX_CONST( 1, 16 );
     psDec->sPLC.subfr_length = 20;
     psDec->sPLC.nb_subfr = 2;
+#ifdef NEURAL_PLC
+    if( psDec->sPLC.lpcnet != NULL ) {
+        lpcnet_plc_init( psDec->sPLC.lpcnet );
+    } else {
+        /* FIXME: This is leaking memory. The right fix is for the LPCNet state to be part of the PLC struct itself. */
+        psDec->sPLC.lpcnet = lpcnet_plc_create();
+    }
+#endif
 }
 
 void silk_PLC(
@@ -88,6 +100,14 @@ void silk_PLC(
         /* Update state             */
         /****************************/
         silk_PLC_update( psDec, psDecCtrl );
+#ifdef NEURAL_PLC
+        if ( psDec->sPLC.fs_kHz == 16 ) {
+            int k;
+            for( k = 0; k < psDec->nb_subfr; k += 2 ) {
+                lpcnet_plc_update( psDec->sPLC.lpcnet, frame + k * psDec->subfr_length );
+            }
+        }
+#endif
     }
 }
 
@@ -371,6 +391,17 @@ static OPUS_INLINE void silk_PLC_conceal(
         /* Scale with Gain */
         frame[ i ] = (opus_int16)silk_SAT16( silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( sLPC_Q14_ptr[ MAX_LPC_ORDER + i ], prevGain_Q10[ 1 ] ), 8 ) ) );
     }
+#ifdef NEURAL_PLC
+    if ( psDec->sPLC.fs_kHz == 16 ) {
+        for( k = 0; k < psDec->nb_subfr; k += 2 ) {
+            lpcnet_plc_conceal(psDec->sPLC.lpcnet, frame + k * psDec->subfr_length );
+        }
+    }
+    /* We *should* be able to copy only from psDec->frame_length-MAX_LPC_ORDER, i.e. the last MAX_LPC_ORDER samples. */
+    for( i = 0; i < psDec->frame_length; i++ ) {
+        sLPC_Q14_ptr[ MAX_LPC_ORDER + i ] = (int)floor(.5 + frame[ i ] * (float)(1 << 24) / prevGain_Q10[ 1 ] );
+    }
+#endif
 
     /* Save LPC state */
     silk_memcpy( psDec->sLPC_Q14_buf, &sLPC_Q14_ptr[ psDec->frame_length ], MAX_LPC_ORDER * sizeof( opus_int32 ) );

silk/structs.h

@@ -34,6 +34,10 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "entenc.h"
 #include "entdec.h"
 
+#ifdef NEURAL_PLC
+#include "lpcnet.h"
+#endif
+
 #ifdef __cplusplus
 extern "C"
 {
@@ -243,6 +247,10 @@ typedef struct {
     opus_int                    fs_kHz;
     opus_int                    nb_subfr;
     opus_int                    subfr_length;
+#ifdef NEURAL_PLC
+    /* FIXME: We should include the state struct directly to preserve the state shadow copy property. */
+    LPCNetPLCState              *lpcnet;
+#endif
 } silk_PLC_struct;
 
 /* Struct for CNG */