diff --git a/dnn/Makefile.am b/dnn/Makefile.am index 3d87ee0289ea69fcd00812dc4f430719661e4724..0b86f2304456f535ae70bbc6b90a4ab59559cb98 100644 --- a/dnn/Makefile.am +++ b/dnn/Makefile.am @@ -38,6 +38,7 @@ liblpcnet_la_SOURCES = \ plc_data.c \ ceps_codebooks.c \ pitch.c \ + parse_lpcnet_weights.c \ freq.c \ kiss_fft.c \ lpcnet_plc.c @@ -46,7 +47,7 @@ liblpcnet_la_LIBADD = $(DEPS_LIBS) $(lrintf_lib) $(LIBM) liblpcnet_la_LDFLAGS = -no-undefined \ -version-info @OP_LT_CURRENT@:@OP_LT_REVISION@:@OP_LT_AGE@ -noinst_PROGRAMS = lpcnet_demo dump_data +noinst_PROGRAMS = lpcnet_demo dump_data dump_weights_blob lpcnet_demo_SOURCES = lpcnet_demo.c lpcnet_demo_LDADD = liblpcnet.la @@ -61,6 +62,10 @@ dump_data_SOURCES = common.c dump_data.c burg.c freq.c kiss_fft.c pitch.c lpcnet dump_data_LDADD = $(LIBM) dump_data_CFLAGS = $(AM_CFLAGS) +dump_weights_blob_SOURCES = nnet_data.c write_lpcnet_weights.c +dump_weights_blob_LDADD = $(LIBM) +dump_weights_blob_CFLAGS = $(AM_CFLAGS) -DDUMP_BINARY_WEIGHTS + pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = lpcnet.pc diff --git a/dnn/lpcnet.c b/dnn/lpcnet.c index 8b90b16df89d76990b51b1dfa47ba194ac0afe0d..4f6a93a461602de9581e3c4b4dca54763b594dcf 100644 --- a/dnn/lpcnet.c +++ b/dnn/lpcnet.c @@ -96,16 +96,16 @@ void run_frame_network(LPCNetState *lpcnet, float *gru_a_condition, float *gru_b pitch = IMIN(255, IMAX(33, pitch)); net = &lpcnet->nnet; RNN_COPY(in, features, NB_FEATURES); - compute_embedding(&embed_pitch, &in[NB_FEATURES], pitch); - compute_conv1d(&feature_conv1, conv1_out, net->feature_conv1_state, in); + compute_embedding(&lpcnet->model.embed_pitch, &in[NB_FEATURES], pitch); + compute_conv1d(&lpcnet->model.feature_conv1, conv1_out, net->feature_conv1_state, in); if (lpcnet->frame_count < FEATURE_CONV1_DELAY) RNN_CLEAR(conv1_out, FEATURE_CONV1_OUT_SIZE); - compute_conv1d(&feature_conv2, conv2_out, net->feature_conv2_state, conv1_out); + compute_conv1d(&lpcnet->model.feature_conv2, conv2_out, net->feature_conv2_state, conv1_out); if (lpcnet->frame_count < FEATURES_DELAY) RNN_CLEAR(conv2_out, FEATURE_CONV2_OUT_SIZE); - _lpcnet_compute_dense(&feature_dense1, dense1_out, conv2_out); - _lpcnet_compute_dense(&feature_dense2, condition, dense1_out); + _lpcnet_compute_dense(&lpcnet->model.feature_dense1, dense1_out, conv2_out); + _lpcnet_compute_dense(&lpcnet->model.feature_dense2, condition, dense1_out); RNN_COPY(rc, condition, LPC_ORDER); - _lpcnet_compute_dense(&gru_a_dense_feature, gru_a_condition, condition); - _lpcnet_compute_dense(&gru_b_dense_feature, gru_b_condition, condition); + _lpcnet_compute_dense(&lpcnet->model.gru_a_dense_feature, gru_a_condition, condition); + _lpcnet_compute_dense(&lpcnet->model.gru_b_dense_feature, gru_b_condition, condition); #ifdef END2END rc2lpc(lpc, rc); #elif FEATURES_DELAY>0 @@ -122,25 +122,27 @@ void run_frame_network(LPCNetState *lpcnet, float *gru_a_condition, float *gru_b if (lpcnet->frame_count < 1000) lpcnet->frame_count++; } -int run_sample_network(NNetState *net, const float *gru_a_condition, const float *gru_b_condition, int last_exc, int last_sig, int pred, const float *sampling_logit_table, kiss99_ctx *rng) +int run_sample_network(LPCNetState *lpcnet, const float *gru_a_condition, const float *gru_b_condition, int last_exc, int last_sig, int pred, const float *sampling_logit_table, kiss99_ctx *rng) { + NNetState *net; float gru_a_input[3*GRU_A_STATE_SIZE]; float in_b[GRU_A_STATE_SIZE+FEATURE_DENSE2_OUT_SIZE]; float gru_b_input[3*GRU_B_STATE_SIZE]; + net = &lpcnet->nnet; #if 1 - compute_gru_a_input(gru_a_input, gru_a_condition, GRU_A_STATE_SIZE, &gru_a_embed_sig, last_sig, &gru_a_embed_pred, pred, &gru_a_embed_exc, last_exc); + compute_gru_a_input(gru_a_input, gru_a_condition, GRU_A_STATE_SIZE, &lpcnet->model.gru_a_embed_sig, last_sig, &lpcnet->model.gru_a_embed_pred, pred, &lpcnet->model.gru_a_embed_exc, last_exc); #else RNN_COPY(gru_a_input, gru_a_condition, 3*GRU_A_STATE_SIZE); - accum_embedding(&gru_a_embed_sig, gru_a_input, last_sig); - accum_embedding(&gru_a_embed_pred, gru_a_input, pred); - accum_embedding(&gru_a_embed_exc, gru_a_input, last_exc); + accum_embedding(&lpcnet->model.gru_a_embed_sig, gru_a_input, last_sig); + accum_embedding(&lpcnet->model.gru_a_embed_pred, gru_a_input, pred); + accum_embedding(&lpcnet->model.gru_a_embed_exc, gru_a_input, last_exc); #endif /*compute_gru3(&gru_a, net->gru_a_state, gru_a_input);*/ - compute_sparse_gru(&sparse_gru_a, net->gru_a_state, gru_a_input); + compute_sparse_gru(&lpcnet->model.sparse_gru_a, net->gru_a_state, gru_a_input); RNN_COPY(in_b, net->gru_a_state, GRU_A_STATE_SIZE); RNN_COPY(gru_b_input, gru_b_condition, 3*GRU_B_STATE_SIZE); - compute_gruB(&gru_b, gru_b_input, net->gru_b_state, in_b); - return sample_mdense(&dual_fc, net->gru_b_state, sampling_logit_table, rng); + compute_gruB(&lpcnet->model.gru_b, gru_b_input, net->gru_b_state, in_b); + return sample_mdense(&lpcnet->model.dual_fc, net->gru_b_state, sampling_logit_table, rng); } LPCNET_EXPORT int lpcnet_get_size() @@ -159,6 +161,7 @@ LPCNET_EXPORT int lpcnet_init(LPCNetState *lpcnet) lpcnet->sampling_logit_table[i] = -log((1-prob)/prob); } kiss99_srand(&lpcnet->rng, (const unsigned char *)rng_string, strlen(rng_string)); + init_lpcnet_model(&lpcnet->model, lpcnet_arrays); return 0; } @@ -205,7 +208,7 @@ void lpcnet_synthesize_tail_impl(LPCNetState *lpcnet, short *output, int N, int for (j=0;j<LPC_ORDER;j++) pred -= lpcnet->last_sig[j]*lpcnet->lpc[j]; last_sig_ulaw = lin2ulaw(lpcnet->last_sig[0]); pred_ulaw = lin2ulaw(pred); - exc = run_sample_network(&lpcnet->nnet, lpcnet->gru_a_condition, lpcnet->gru_b_condition, lpcnet->last_exc, last_sig_ulaw, pred_ulaw, lpcnet->sampling_logit_table, &lpcnet->rng); + exc = run_sample_network(lpcnet, lpcnet->gru_a_condition, lpcnet->gru_b_condition, lpcnet->last_exc, last_sig_ulaw, pred_ulaw, lpcnet->sampling_logit_table, &lpcnet->rng); if (i < preload) { exc = lin2ulaw(output[i]-PREEMPH*lpcnet->deemph_mem - pred); pcm = output[i]-PREEMPH*lpcnet->deemph_mem; diff --git a/dnn/lpcnet_private.h b/dnn/lpcnet_private.h index 1d2936d69f389de8eaae5c3b7ed7d702887d75f1..07f32ac19fb6d2b0edade9764a7da0be885e5181 100644 --- a/dnn/lpcnet_private.h +++ b/dnn/lpcnet_private.h @@ -39,6 +39,7 @@ struct LPCNetState { float deemph_mem; float lpc[LPC_ORDER]; kiss99_ctx rng; + LPCNetModel model; }; struct LPCNetDecState { diff --git a/dnn/nnet.h b/dnn/nnet.h index 5b5bd128e5afff81d97e3aba19a418a4261c57fb..406113e694a82eb6dc419ef415d799b83600863b 100644 --- a/dnn/nnet.h +++ b/dnn/nnet.h @@ -146,6 +146,8 @@ void compute_gru_a_input(float *output, const float *input, int N, const Embeddi int sample_from_pdf(const float *pdf, int N, float exp_boost, float pdf_floor); +extern const WeightArray lpcnet_arrays[]; + int mdense_init(MDenseLayer *layer, const WeightArray *arrays, const char *bias, diff --git a/dnn/training_tf2/dump_lpcnet.py b/dnn/training_tf2/dump_lpcnet.py index 4205fb7c6a5dcd1a03906be994067ad467146372..c207dae8fed7247d81da5d1ad9886492e6ea7eca 100755 --- a/dnn/training_tf2/dump_lpcnet.py +++ b/dnn/training_tf2/dump_lpcnet.py @@ -61,9 +61,7 @@ def printVector(f, vector, name, dtype='float', dotp=False): #print('static const float ', name, '[', len(v), '] = \n', file=f) if name not in array_list: array_list.append(name) - f.write('#ifdef USE_WEIGHTS_FILE\n') - f.write('static const {} *{} = NULL;\n'.format(dtype, name, len(v))) - f.write('#else\n') + f.write('#ifndef USE_WEIGHTS_FILE\n') f.write('#define WEIGHTS_{}_DEFINED\n'.format(name)) f.write('#define WEIGHTS_{}_TYPE WEIGHT_TYPE_{}\n'.format(name, dtype)) f.write('static const {} {}[{}] = {{\n '.format(dtype, name, len(v))) @@ -143,11 +141,8 @@ def dump_sparse_gru(self, f, hf): reset_after = 1 neurons = weights[0].shape[1]//3 max_rnn_neurons = max(max_rnn_neurons, neurons) - f.write('const SparseGRULayer {} = {{\n {}_bias,\n {}_subias,\n {}_recurrent_weights_diag,\n {}_recurrent_weights,\n {}_recurrent_weights_idx,\n {}, ACTIVATION_{}, {}\n}};\n\n' - .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after)) hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3)) hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3)) - hf.write('extern const SparseGRULayer {};\n\n'.format(name)); model_struct.write(' SparseGRULayer {};\n'.format(name)); model_init.write(' if (sparse_gru_init(&model->{}, arrays, "{}_bias", "{}_subias", "{}_recurrent_weights_diag", "{}_recurrent_weights", "{}_recurrent_weights_idx", {}, ACTIVATION_{}, {})) return 1;\n' .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after)) @@ -182,9 +177,6 @@ def dump_grub(self, f, hf, gru_a_size): reset_after = 1 neurons = weights[0].shape[1]//3 max_rnn_neurons = max(max_rnn_neurons, neurons) - f.write('const GRULayer {} = {{\n {}_bias,\n {}_subias,\n {}_weights,\n {}_weights_idx,\n {}_recurrent_weights,\n {}, {}, ACTIVATION_{}, {}\n}};\n\n' - .format(name, name, name, name, name, name, gru_a_size, weights[0].shape[1]//3, activation, reset_after)) - hf.write('extern const GRULayer {};\n\n'.format(name)); model_struct.write(' GRULayer {};\n'.format(name)); model_init.write(' if (gru_init(&model->{}, arrays, "{}_bias", "{}_subias", "{}_weights", "{}_weights_idx", "{}_recurrent_weights", {}, {}, ACTIVATION_{}, {})) return 1;\n' .format(name, name, name, name, name, name, gru_a_size, weights[0].shape[1]//3, activation, reset_after)) @@ -202,10 +194,7 @@ GRU.dump_layer = dump_gru_layer_dummy def dump_dense_layer_impl(name, weights, bias, activation, f, hf): printVector(f, weights, name + '_weights') printVector(f, bias, name + '_bias') - f.write('const DenseLayer {} = {{\n {}_bias,\n {}_weights,\n {}, {}, ACTIVATION_{}\n}};\n\n' - .format(name, name, name, weights.shape[0], weights.shape[1], activation)) hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights.shape[1])) - hf.write('extern const DenseLayer {};\n\n'.format(name)); model_struct.write(' DenseLayer {};\n'.format(name)); model_init.write(' if (dense_init(&model->{}, arrays, "{}_bias", "{}_weights", {}, {}, ACTIVATION_{})) return 1;\n' .format(name, name, name, weights.shape[0], weights.shape[1], activation)) @@ -230,10 +219,7 @@ def dump_mdense_layer(self, f, hf): printVector(f, np.transpose(weights[2], (1, 0)), name + '_factor') activation = self.activation.__name__.upper() max_mdense_tmp = max(max_mdense_tmp, weights[0].shape[0]*weights[0].shape[2]) - f.write('const MDenseLayer {} = {{\n {}_bias,\n {}_weights,\n {}_factor,\n {}, {}, {}, ACTIVATION_{}\n}};\n\n' - .format(name, name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation)) hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[0])) - hf.write('extern const MDenseLayer {};\n\n'.format(name)); model_struct.write(' MDenseLayer {};\n'.format(name)); model_init.write(' if (mdense_init(&model->{}, arrays, "{}_bias", "{}_weights", "{}_factor", {}, {}, {}, ACTIVATION_{})) return 1;\n' .format(name, name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation)) @@ -249,12 +235,9 @@ def dump_conv1d_layer(self, f, hf): printVector(f, weights[-1], name + '_bias') activation = self.activation.__name__.upper() max_conv_inputs = max(max_conv_inputs, weights[0].shape[1]*weights[0].shape[0]) - f.write('const Conv1DLayer {} = {{\n {}_bias,\n {}_weights,\n {}, {}, {}, ACTIVATION_{}\n}};\n\n' - .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation)) hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[2])) hf.write('#define {}_STATE_SIZE ({}*{})\n'.format(name.upper(), weights[0].shape[1], (weights[0].shape[0]-1))) hf.write('#define {}_DELAY {}\n'.format(name.upper(), (weights[0].shape[0]-1)//2)) - hf.write('extern const Conv1DLayer {};\n\n'.format(name)); model_struct.write(' Conv1DLayer {};\n'.format(name)); model_init.write(' if (conv1d_init(&model->{}, arrays, "{}_bias", "{}_weights", {}, {}, {}, ACTIVATION_{})) return 1;\n' .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation)) @@ -264,10 +247,7 @@ Conv1D.dump_layer = dump_conv1d_layer def dump_embedding_layer_impl(name, weights, f, hf): printVector(f, weights, name + '_weights') - f.write('const EmbeddingLayer {} = {{\n {}_weights,\n {}, {}\n}};\n\n' - .format(name, name, weights.shape[0], weights.shape[1])) hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights.shape[1])) - hf.write('extern const EmbeddingLayer {};\n\n'.format(name)); model_struct.write(' EmbeddingLayer {};\n'.format(name)); model_init.write(' if (embedding_init(&model->{}, arrays, "{}_weights", {}, {})) return 1;\n' .format(name, name, weights.shape[0], weights.shape[1])) @@ -312,6 +292,7 @@ if __name__ == "__main__": model_struct = io.StringIO() model_init = io.StringIO() model_struct.write('typedef struct {\n') + model_init.write('#ifndef DUMP_BINARY_WEIGHTS\n') model_init.write('int init_lpcnet_model(LPCNetModel *model, const WeightArray *arrays) {\n') array_list = [] @@ -375,16 +356,17 @@ if __name__ == "__main__": dump_sparse_gru(model.get_layer('gru_a'), f, hf) - f.write('#ifdef DUMP_BINARY_WEIGHTS\n') + f.write('#ifndef USE_WEIGHTS_FILE\n') f.write('const WeightArray lpcnet_arrays[] = {\n') for name in array_list: f.write('#ifdef WEIGHTS_{}_DEFINED\n'.format(name)) f.write(' {{"{}", WEIGHTS_{}_TYPE, sizeof({}), {}}},\n'.format(name, name, name, name)) f.write('#endif\n') - f.write(' {NULL, 0, 0}\n};\n') - f.write('#endif\n\n') + f.write(' {NULL, 0, 0, NULL}\n};\n') + f.write('#endif\n') model_init.write(' return 0;\n}\n') + model_init.write('#endif\n') f.write(model_init.getvalue()) hf.write('#define MAX_RNN_NEURONS {}\n\n'.format(max_rnn_neurons)) @@ -399,6 +381,7 @@ if __name__ == "__main__": model_struct.write('} LPCNetModel;\n\n') hf.write(model_struct.getvalue()) + hf.write('int init_lpcnet_model(LPCNetModel *model, const WeightArray *arrays);\n\n') hf.write('\n\n#endif\n') f.close()