From d0f6df59498a56de653298964f333043332675cf Mon Sep 17 00:00:00 2001
From: Jean-Marc Valin <jmvalin@amazon.com>
Date: Tue, 25 Jan 2022 02:48:51 -0500
Subject: [PATCH] WIP: Using LPCNet for PLC
---
Makefile.am | 11 +++++++----
lpcnet_sources.mk | 11 +++++++++++
silk/PLC.c | 31 +++++++++++++++++++++++++++++++
silk/structs.h | 8 ++++++++
4 files changed, 57 insertions(+), 4 deletions(-)
create mode 100644 lpcnet_sources.mk
diff --git a/Makefile.am b/Makefile.am
index 492fc09da..3d1d5ed28 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,9 +10,11 @@ lib_LTLIBRARIES = libopus.la
DIST_SUBDIRS = doc
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/celt -I$(top_srcdir)/silk \
- -I$(top_srcdir)/silk/float -I$(top_srcdir)/silk/fixed $(NE10_CFLAGS)
+ -I$(top_srcdir)/silk/float -I$(top_srcdir)/silk/fixed $(NE10_CFLAGS) \
+ -I$(top_srcdir)/lpcnet/include
include celt_sources.mk
+include lpcnet_sources.mk
include silk_sources.mk
include opus_sources.mk
@@ -83,7 +85,7 @@ include celt_headers.mk
include silk_headers.mk
include opus_headers.mk
-libopus_la_SOURCES = $(CELT_SOURCES) $(SILK_SOURCES) $(OPUS_SOURCES)
+libopus_la_SOURCES = $(CELT_SOURCES) $(SILK_SOURCES) $(LPCNET_SOURCES) $(OPUS_SOURCES)
libopus_la_LDFLAGS = -no-undefined -version-info @OPUS_LT_CURRENT@:@OPUS_LT_REVISION@:@OPUS_LT_AGE@
libopus_la_LIBADD = $(NE10_LIBS) $(LIBM)
if OPUS_ARM_EXTERNAL_ASM
@@ -157,16 +159,17 @@ tests_test_opus_padding_LDADD = libopus.la $(NE10_LIBS) $(LIBM)
CELT_OBJ = $(CELT_SOURCES:.c=.lo)
SILK_OBJ = $(SILK_SOURCES:.c=.lo)
+LPCNET_OBJ = $(LPCNET_SOURCES:.c=.lo)
OPUS_OBJ = $(OPUS_SOURCES:.c=.lo)
tests_test_opus_projection_SOURCES = tests/test_opus_projection.c tests/test_opus_common.h
-tests_test_opus_projection_LDADD = $(OPUS_OBJ) $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
+tests_test_opus_projection_LDADD = $(OPUS_OBJ) $(SILK_OBJ) $(LPCNET_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
if OPUS_ARM_EXTERNAL_ASM
tests_test_opus_projection_LDADD += libarmasm.la
endif
silk_tests_test_unit_LPC_inv_pred_gain_SOURCES = silk/tests/test_unit_LPC_inv_pred_gain.c
-silk_tests_test_unit_LPC_inv_pred_gain_LDADD = $(SILK_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
+silk_tests_test_unit_LPC_inv_pred_gain_LDADD = $(SILK_OBJ) $(LPCNET_OBJ) $(CELT_OBJ) $(NE10_LIBS) $(LIBM)
if OPUS_ARM_EXTERNAL_ASM
silk_tests_test_unit_LPC_inv_pred_gain_LDADD += libarmasm.la
endif
diff --git a/lpcnet_sources.mk b/lpcnet_sources.mk
new file mode 100644
index 000000000..5e6674d6e
--- /dev/null
+++ b/lpcnet_sources.mk
@@ -0,0 +1,11 @@
+LPCNET_SOURCES = \
+lpcnet/src/ceps_codebooks.c \
+lpcnet/src/common.c \
+lpcnet/src/freq.c \
+lpcnet/src/kiss99.c \
+lpcnet/src/lpcnet.c \
+lpcnet/src/lpcnet_dec.c \
+lpcnet/src/lpcnet_enc.c \
+lpcnet/src/lpcnet_plc.c \
+lpcnet/src/nnet.c \
+lpcnet/src/nnet_data.c
diff --git a/silk/PLC.c b/silk/PLC.c
index 4667440db..431c7214a 100644
--- a/silk/PLC.c
+++ b/silk/PLC.c
@@ -33,6 +33,10 @@ POSSIBILITY OF SUCH DAMAGE.
#include "stack_alloc.h"
#include "PLC.h"
+#ifdef NEURAL_PLC
+#include "lpcnet.h"
+#endif
+
#define NB_ATT 2
static const opus_int16 HARM_ATT_Q15[NB_ATT] = { 32440, 31130 }; /* 0.99, 0.95 */
static const opus_int16 PLC_RAND_ATTENUATE_V_Q15[NB_ATT] = { 31130, 26214 }; /* 0.95, 0.8 */
@@ -60,6 +64,14 @@ void silk_PLC_Reset(
psDec->sPLC.prevGain_Q16[ 1 ] = SILK_FIX_CONST( 1, 16 );
psDec->sPLC.subfr_length = 20;
psDec->sPLC.nb_subfr = 2;
+#ifdef NEURAL_PLC
+ if( psDec->sPLC.lpcnet != NULL ) {
+ lpcnet_plc_init( psDec->sPLC.lpcnet );
+ } else {
+ /* FIXME: This is leaking memory. The right fix is for the LPCNet state to be part of the PLC struct itself. */
+ psDec->sPLC.lpcnet = lpcnet_plc_create();
+ }
+#endif
}
void silk_PLC(
@@ -88,6 +100,14 @@ void silk_PLC(
/* Update state */
/****************************/
silk_PLC_update( psDec, psDecCtrl );
+#ifdef NEURAL_PLC
+ if ( psDec->sPLC.fs_kHz == 16 ) {
+ int k;
+ for( k = 0; k < psDec->nb_subfr; k += 2 ) {
+ lpcnet_plc_update( psDec->sPLC.lpcnet, frame + k * psDec->subfr_length );
+ }
+ }
+#endif
}
}
@@ -371,6 +391,17 @@ static OPUS_INLINE void silk_PLC_conceal(
/* Scale with Gain */
frame[ i ] = (opus_int16)silk_SAT16( silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( sLPC_Q14_ptr[ MAX_LPC_ORDER + i ], prevGain_Q10[ 1 ] ), 8 ) ) );
}
+#ifdef NEURAL_PLC
+ if ( psDec->sPLC.fs_kHz == 16 ) {
+ for( k = 0; k < psDec->nb_subfr; k += 2 ) {
+ lpcnet_plc_conceal(psDec->sPLC.lpcnet, frame + k * psDec->subfr_length );
+ }
+ }
+ /* We *should* be able to copy only from psDec->frame_length-MAX_LPC_ORDER, i.e. the last MAX_LPC_ORDER samples. */
+ for( i = 0; i < psDec->frame_length; i++ ) {
+ sLPC_Q14_ptr[ MAX_LPC_ORDER + i ] = (int)floor(.5 + frame[ i ] * (float)(1 << 24) / prevGain_Q10[ 1 ] );
+ }
+#endif
/* Save LPC state */
silk_memcpy( psDec->sLPC_Q14_buf, &sLPC_Q14_ptr[ psDec->frame_length ], MAX_LPC_ORDER * sizeof( opus_int32 ) );
diff --git a/silk/structs.h b/silk/structs.h
index 3380c757b..44ba06e19 100644
--- a/silk/structs.h
+++ b/silk/structs.h
@@ -34,6 +34,10 @@ POSSIBILITY OF SUCH DAMAGE.
#include "entenc.h"
#include "entdec.h"
+#ifdef NEURAL_PLC
+#include "lpcnet.h"
+#endif
+
#ifdef __cplusplus
extern "C"
{
@@ -243,6 +247,10 @@ typedef struct {
opus_int fs_kHz;
opus_int nb_subfr;
opus_int subfr_length;
+#ifdef NEURAL_PLC
+ /* FIXME: We should include the state struct directly to preserve the state shadow copy property. */
+ LPCNetPLCState *lpcnet;
+#endif
} silk_PLC_struct;
/* Struct for CNG */
--
GitLab