From c74330e85035c2d47f101fa33797e93ecaaebcd3 Mon Sep 17 00:00:00 2001
From: Jean-Marc Valin <jmvalin@amazon.com>
Date: Thu, 15 Jul 2021 16:06:56 -0400
Subject: [PATCH] Pre-compute GRU B conditioning
Adapted from PR: https://github.com/mozilla/LPCNet/pull/134
by zhuxiaoxu <zhuxiaoxu@ainirobot.com>
but had to be reworked due to previous weight quantization changes.
---
dnn/lpcnet.c | 17 +++++++------
dnn/nnet.c | 44 +++++++++++++++++++++++++++++++++
dnn/nnet.h | 2 ++
dnn/training_tf2/dump_lpcnet.py | 27 ++++++++++++++------
4 files changed, 76 insertions(+), 14 deletions(-)
diff --git a/dnn/lpcnet.c b/dnn/lpcnet.c
index a59cfbcab..101d27205 100644
--- a/dnn/lpcnet.c
+++ b/dnn/lpcnet.c
@@ -54,9 +54,10 @@ static void print_vector(float *x, int N)
}
#endif
-void run_frame_network(LPCNetState *lpcnet, float *condition, float *gru_a_condition, const float *features, int pitch)
+void run_frame_network(LPCNetState *lpcnet, float *gru_a_condition, float *gru_b_condition, const float *features, int pitch)
{
NNetState *net;
+ float condition[FEATURE_DENSE2_OUT_SIZE];
float in[FRAME_INPUT_SIZE];
float conv1_out[FEATURE_CONV1_OUT_SIZE];
float conv2_out[FEATURE_CONV2_OUT_SIZE];
@@ -74,13 +75,15 @@ void run_frame_network(LPCNetState *lpcnet, float *condition, float *gru_a_condi
compute_dense(&feature_dense1, dense1_out, conv2_out);
compute_dense(&feature_dense2, condition, dense1_out);
compute_dense(&gru_a_dense_feature, gru_a_condition, condition);
+ compute_dense(&gru_b_dense_feature, gru_b_condition, condition);
if (lpcnet->frame_count < 1000) lpcnet->frame_count++;
}
-int run_sample_network(NNetState *net, const float *condition, const float *gru_a_condition, int last_exc, int last_sig, int pred, const float *sampling_logit_table)
+int run_sample_network(NNetState *net, const float *gru_a_condition, const float *gru_b_condition, int last_exc, int last_sig, int pred, const float *sampling_logit_table)
{
float gru_a_input[3*GRU_A_STATE_SIZE];
float in_b[GRU_A_STATE_SIZE+FEATURE_DENSE2_OUT_SIZE];
+ float gru_b_input[3*GRU_B_STATE_SIZE];
#if 1
compute_gru_a_input(gru_a_input, gru_a_condition, GRU_A_STATE_SIZE, &gru_a_embed_sig, last_sig, &gru_a_embed_pred, pred, &gru_a_embed_exc, last_exc);
#else
@@ -92,8 +95,8 @@ int run_sample_network(NNetState *net, const float *condition, const float *gru_
/*compute_gru3(&gru_a, net->gru_a_state, gru_a_input);*/
compute_sparse_gru(&sparse_gru_a, net->gru_a_state, gru_a_input);
RNN_COPY(in_b, net->gru_a_state, GRU_A_STATE_SIZE);
- RNN_COPY(&in_b[GRU_A_STATE_SIZE], condition, FEATURE_DENSE2_OUT_SIZE);
- compute_gru2(&gru_b, net->gru_b_state, in_b);
+ RNN_COPY(gru_b_input, gru_b_condition, 3*GRU_B_STATE_SIZE);
+ compute_gruB(&gru_b, gru_b_input, net->gru_b_state, in_b);
return sample_mdense(&dual_fc, net->gru_b_state, sampling_logit_table);
}
@@ -131,16 +134,16 @@ LPCNET_EXPORT void lpcnet_destroy(LPCNetState *lpcnet)
LPCNET_EXPORT void lpcnet_synthesize(LPCNetState *lpcnet, const float *features, short *output, int N)
{
int i;
- float condition[FEATURE_DENSE2_OUT_SIZE];
float lpc[LPC_ORDER];
float gru_a_condition[3*GRU_A_STATE_SIZE];
+ float gru_b_condition[3*GRU_B_STATE_SIZE];
int pitch;
/* Matches the Python code -- the 0.1 avoids rounding issues. */
pitch = (int)floor(.1 + 50*features[36]+100);
pitch = IMIN(255, IMAX(33, pitch));
memmove(&lpcnet->old_gain[1], &lpcnet->old_gain[0], (FEATURES_DELAY-1)*sizeof(lpcnet->old_gain[0]));
lpcnet->old_gain[0] = features[PITCH_GAIN_FEATURE];
- run_frame_network(lpcnet, condition, gru_a_condition, features, pitch);
+ run_frame_network(lpcnet, gru_a_condition, gru_b_condition, features, pitch);
memcpy(lpc, lpcnet->old_lpc[FEATURES_DELAY-1], LPC_ORDER*sizeof(lpc[0]));
memmove(lpcnet->old_lpc[1], lpcnet->old_lpc[0], (FEATURES_DELAY-1)*LPC_ORDER*sizeof(lpc[0]));
lpc_from_cepstrum(lpcnet->old_lpc[0], features);
@@ -160,7 +163,7 @@ LPCNET_EXPORT void lpcnet_synthesize(LPCNetState *lpcnet, const float *features,
for (j=0;j<LPC_ORDER;j++) pred -= lpcnet->last_sig[j]*lpc[j];
last_sig_ulaw = lin2ulaw(lpcnet->last_sig[0]);
pred_ulaw = lin2ulaw(pred);
- exc = run_sample_network(&lpcnet->nnet, condition, gru_a_condition, lpcnet->last_exc, last_sig_ulaw, pred_ulaw, lpcnet->sampling_logit_table);
+ exc = run_sample_network(&lpcnet->nnet, gru_a_condition, gru_b_condition, lpcnet->last_exc, last_sig_ulaw, pred_ulaw, lpcnet->sampling_logit_table);
pcm = pred + ulaw2lin(exc);
RNN_MOVE(&lpcnet->last_sig[1], &lpcnet->last_sig[0], LPC_ORDER-1);
lpcnet->last_sig[0] = pcm;
diff --git a/dnn/nnet.c b/dnn/nnet.c
index 9cd0d13fb..566268ec1 100644
--- a/dnn/nnet.c
+++ b/dnn/nnet.c
@@ -296,6 +296,50 @@ void compute_gru2(const GRULayer *gru, float *state, const float *input)
state[i] = h[i];
}
+void compute_gruB(const GRULayer *gru, const float* gru_b_condition, float *state, const float *input)
+{
+ int i;
+ int N, M;
+ int stride;
+ float zrh[3*MAX_RNN_NEURONS];
+ float recur[3*MAX_RNN_NEURONS];
+ float *z;
+ float *r;
+ float *h;
+ M = gru->nb_inputs;
+ N = gru->nb_neurons;
+ z = zrh;
+ r = &zrh[N];
+ h = &zrh[2*N];
+ celt_assert(gru->nb_neurons <= MAX_RNN_NEURONS);
+ celt_assert(input != state);
+ celt_assert(gru->reset_after);
+ stride = 3*N;
+ /* Compute update gate. */
+#ifdef USE_SU_BIAS
+ for (i=0;i<3*N;i++)
+ zrh[i] = gru->subias[i] + gru_b_condition[i];
+#else
+ for (i=0;i<3*N;i++)
+ zrh[i] = gru->bias[i] + gru_b_condition[i];
+#endif
+ sgemv_accum8x4(zrh, gru->input_weights, 3*N, M, stride, input);
+ for (i=0;i<3*N;i++)
+ recur[i] = gru->bias[3*N + i];
+ sgemv_accum(recur, gru->recurrent_weights, 3*N, N, stride, state);
+ for (i=0;i<2*N;i++)
+ zrh[i] += recur[i];
+ compute_activation(zrh, zrh, 2*N, ACTIVATION_SIGMOID);
+ for (i=0;i<N;i++)
+ h[i] += recur[2*N+i]*r[i];
+ compute_activation(h, h, N, gru->activation);
+ for (i=0;i<N;i++)
+ h[i] = z[i]*state[i] + (1-z[i])*h[i];
+ for (i=0;i<N;i++)
+ state[i] = h[i];
+}
+
+
void compute_gru3(const GRULayer *gru, float *state, const float *input)
{
int i;
diff --git a/dnn/nnet.h b/dnn/nnet.h
index 12648cbbd..b7e9b6905 100644
--- a/dnn/nnet.h
+++ b/dnn/nnet.h
@@ -103,6 +103,8 @@ void compute_gru(const GRULayer *gru, float *state, const float *input);
void compute_gru2(const GRULayer *gru, float *state, const float *input);
+void compute_gruB(const GRULayer *gru, const float* gru_b_condition, float *state, const float *input);
+
void compute_gru3(const GRULayer *gru, float *state, const float *input);
void compute_sparse_gru(const SparseGRULayer *gru, float *state, const float *input);
diff --git a/dnn/training_tf2/dump_lpcnet.py b/dnn/training_tf2/dump_lpcnet.py
index a5edefd69..6980d28b9 100755
--- a/dnn/training_tf2/dump_lpcnet.py
+++ b/dnn/training_tf2/dump_lpcnet.py
@@ -126,16 +126,16 @@ def dump_sparse_gru(self, f, hf):
hf.write('extern const SparseGRULayer {};\n\n'.format(name));
return True
-def dump_gru_layer(self, f, hf):
+def dump_grub(self, f, hf, gru_a_size):
global max_rnn_neurons
name = self.name
print("printing layer " + name + " of type " + self.__class__.__name__)
weights = self.get_weights()
f.write('#ifdef DOT_PROD\n')
- qweight = np.clip(np.round(128.*weights[0]).astype('int'), -128, 127)
+ qweight = np.clip(np.round(128.*weights[0][:gru_a_size, :]).astype('int'), -128, 127)
printVector(f, qweight, name + '_weights', dotp=True, dtype='qweight')
f.write('#else /*DOT_PROD*/\n')
- printVector(f, weights[0], name + '_weights')
+ printVector(f, weights[0][:gru_a_size, :], name + '_weights')
f.write('#endif /*DOT_PROD*/\n')
printVector(f, weights[1], name + '_recurrent_weights')
printVector(f, weights[-1], name + '_bias')
@@ -153,12 +153,18 @@ def dump_gru_layer(self, f, hf):
neurons = weights[0].shape[1]//3
max_rnn_neurons = max(max_rnn_neurons, neurons)
f.write('const GRULayer {} = {{\n {}_bias,\n {}_subias,\n {}_weights,\n {}_recurrent_weights,\n {}, {}, ACTIVATION_{}, {}\n}};\n\n'
- .format(name, name, name, name, name, weights[0].shape[0], weights[0].shape[1]//3, activation, reset_after))
- hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
- hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+ .format(name, name, name, name, name, gru_a_size, weights[0].shape[1]//3, activation, reset_after))
hf.write('extern const GRULayer {};\n\n'.format(name));
return True
-GRU.dump_layer = dump_gru_layer
+
+def dump_gru_layer_dummy(self, f, hf):
+ name = self.name
+ weights = self.get_weights()
+ hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+ hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+ return True;
+
+GRU.dump_layer = dump_gru_layer_dummy
def dump_dense_layer_impl(name, weights, bias, activation, f, hf):
printVector(f, weights, name + '_weights')
@@ -272,6 +278,13 @@ W = model.get_layer('gru_a').get_weights()[0][3*embed_size:,:]
b = model.get_layer('gru_a').get_weights()[2]
dump_dense_layer_impl('gru_a_dense_feature', W, b, 'LINEAR', f, hf)
+W = model.get_layer('gru_b').get_weights()[0][model.rnn_units1:,:]
+b = model.get_layer('gru_b').get_weights()[2]
+# Set biases to zero because they'll be included in the GRU input part
+# (we need regular and SU biases)
+dump_dense_layer_impl('gru_b_dense_feature', W, 0*b, 'LINEAR', f, hf)
+dump_grub(model.get_layer('gru_b'), f, hf, model.rnn_units1)
+
layer_list = []
for i, layer in enumerate(model.layers):
if layer.dump_layer(f, hf):
--
GitLab