Commit 9229b7bf authored by Monty's avatar Monty
Browse files

Update the VQ tools to pared down codebook abstraction.

Eliminate long unused tools; they're still in SVN.


svn path=/trunk/vorbis/; revision=16959
parent 8269e97d
......@@ -2,8 +2,7 @@
INCLUDES = -I../lib -I$(top_srcdir)/include @OGG_CFLAGS@
EXTRA_PROGRAMS = latticebuild latticepare latticehint\
latticetune huffbuild distribution
EXTRA_PROGRAMS = latticebuild latticetune huffbuild distribution
CLEANFILES = $(EXTRA_PROGRAMS)
AM_LDFLAGS = -static
......@@ -11,10 +10,6 @@ LDADD = ../lib/libvorbis.la
latticebuild_SOURCES = latticebuild.c vqgen.c bookutil.c\
vqgen.h bookutil.h localcodebook.h
latticepare_SOURCES = latticepare.c vqgen.c bookutil.c vqsplit.c\
vqgen.h vqsplit.h bookutil.h localcodebook.h
latticehint_SOURCES = latticehint.c bookutil.c\
vqsplit.h bookutil.h localcodebook.h
latticetune_SOURCES = latticetune.c vqgen.c bookutil.c\
vqgen.h bookutil.h localcodebook.h
huffbuild_SOURCES = huffbuild.c vqgen.c bookutil.c\
......
......@@ -5,7 +5,7 @@
* GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
* IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
* *
* THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
* THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2010 *
* by the Xiph.Org Foundation http://www.xiph.org/ *
* *
********************************************************************
......@@ -22,69 +22,61 @@
#include <errno.h>
#include "bookutil.h"
int _best(codebook *book, float *a, int step){
/* as of current encoder, only centered, integer val, maptype 1 is in
use */
int _best(codebook *book, int *a, int step){
int dim=book->dim;
int k,o;
int i,j,o;
int minval=book->minval;
int del=book->delta;
int qv=book->quantvals;
int ze=(qv>>1);
int index=0;
/* assumes integer/centered encoder codebook maptype 1 no more than dim 8 */
if(delta!=1){
for(k=0,o=step*(dim-1);k<dim;k++,o-=step){
int v = (a[o]-minval+(delta>>1))/delta;
if(del!=1){
for(i=0,o=step*(dim-1);i<dim;i++,o-=step){
int v = ((int)rint(a[o])-minval+(del>>1))/del;
int m = (v<ze ? ((ze-v)<<1)-1 : ((v-ze)<<1));
index = index*qv+ (m<0?0:(m>=qv?qv-1:m));
}
}else{
for(k=0,o=step*(dim-1);k<dim;k++,o-=step){
int v = a[o]-minval;
for(i=0,o=step*(dim-1);i<dim;i++,o-=step){
int v = (int)rint(a[o])-minval;
int m = (v<ze ? ((ze-v)<<1)-1 : ((v-ze)<<1));
index = index*qv+ (m<0?0:(m>=qv?qv-1:m));
}
}
/* did the direct lookup find a used entry? */
if(book->c->lengthlist[index]>0)
return(index);
/* brute force it */
{
if(book->c->lengthlist[index]<=0){
const static_codebook *c=book->c;
int i,besti=-1;
float best=0.f;
float *e=book->valuelist;
int best=-1;
/* assumes integer/centered encoder codebook maptype 1 no more than dim 8 */
int e[8]={0,0,0,0,0,0,0,0};
int maxval = book->minval + book->delta*(book->quantvals-1);
for(i=0;i<book->entries;i++){
if(c->lengthlist[i]>0){
float this=_dist(dim,e,a,step);
if(besti==-1 || this<best){
float this=0;
for(j=0;j<dim;j++){
float val=(e[j]-a[j*step]);
this+=val*val;
}
if(best==-1 || this<best){
best=this;
besti=i;
index=i;
}
}
e+=dim;
/* assumes the value patterning created by the tools in vq/ */
j=0;
while(e[j]>=maxval)
e[j++]=0;
if(e[j]>=0)
e[j]+=book->delta;
e[j]= -e[j];
}
/*if(savebest!=-1 && savebest!=besti){
fprintf(stderr,"brute force/pigeonhole disagreement:\n"
"original:");
for(i=0;i<dim*step;i+=step)fprintf(stderr,"%g,",a[i]);
fprintf(stderr,"\n"
"pigeonhole (entry %d, err %g):",savebest,saverr);
for(i=0;i<dim;i++)fprintf(stderr,"%g,",
(book->valuelist+savebest*dim)[i]);
fprintf(stderr,"\n"
"bruteforce (entry %d, err %g):",besti,best);
for(i=0;i<dim;i++)fprintf(stderr,"%g,",
(book->valuelist+besti*dim)[i]);
fprintf(stderr,"\n");
}*/
return(besti);
}
}
return index;
}
/* A few little utils for reading files */
/* read a line. Use global, persistent buffering */
......@@ -241,9 +233,6 @@ char *find_seek_to(FILE *in,char *s){
codebook *codebook_load(char *filename){
codebook *b=_ogg_calloc(1,sizeof(codebook));
static_codebook *c=(static_codebook *)(b->c=_ogg_calloc(1,sizeof(static_codebook)));
encode_aux_nearestmatch *a=NULL;
encode_aux_threshmatch *t=NULL;
encode_aux_pigeonhole *p=NULL;
int quant_to_read=0;
FILE *in=fopen(filename,"r");
char *line;
......@@ -273,153 +262,6 @@ codebook *codebook_load(char *filename){
exit(1);
}
/* find the auxiliary encode struct[s] (if any) */
if(find_seek_to(in,"static const encode_aux_nearestmatch _vq_aux")){
/* how big? */
c->nearest_tree=a=_ogg_calloc(1,sizeof(encode_aux_nearestmatch));
line=get_line(in);
line=get_line(in);
line=get_line(in);
line=get_line(in);
line=get_line(in);
if(sscanf(line,"%ld, %ld",&(a->aux),&(a->alloc))!=2){
fprintf(stderr,"2: syntax in %s in line:\t %s",filename,line);
exit(1);
}
/* load ptr0 */
find_seek_to(in,"static const long _vq_ptr0");
reset_next_value();
a->ptr0=_ogg_malloc(sizeof(long)*a->aux);
for(i=0;i<a->aux;i++)
if(get_next_ivalue(in,a->ptr0+i)){
fprintf(stderr,"out of data while reading codebook %s\n",filename);
exit(1);
}
/* load ptr1 */
find_seek_to(in,"static const long _vq_ptr1");
reset_next_value();
a->ptr1=_ogg_malloc(sizeof(long)*a->aux);
for(i=0;i<a->aux;i++)
if(get_next_ivalue(in,a->ptr1+i)){
fprintf(stderr,"out of data while reading codebook %s\n",filename);
exit(1);
}
/* load p */
find_seek_to(in,"static const long _vq_p_");
reset_next_value();
a->p=_ogg_malloc(sizeof(long)*a->aux);
for(i=0;i<a->aux;i++)
if(get_next_ivalue(in,a->p+i)){
fprintf(stderr,"out of data while reading codebook %s\n",filename);
exit(1);
}
/* load q */
find_seek_to(in,"static const long _vq_q_");
reset_next_value();
a->q=_ogg_malloc(sizeof(long)*a->aux);
for(i=0;i<a->aux;i++)
if(get_next_ivalue(in,a->q+i)){
fprintf(stderr,"out of data while reading codebook %s\n",filename);
exit(1);
}
}
if(find_seek_to(in,"static const encode_aux_threshmatch _vq_aux")){
/* how big? */
c->thresh_tree=t=_ogg_calloc(1,sizeof(encode_aux_threshmatch));
line=get_line(in);
line=get_line(in);
line=get_line(in);
if(sscanf(line,"%d",&(t->quantvals))!=1){
fprintf(stderr,"3: syntax in %s in line:\t %s",filename,line);
exit(1);
}
line=get_line(in);
if(sscanf(line,"%d",&(t->threshvals))!=1){
fprintf(stderr,"4: syntax in %s in line:\t %s",filename,line);
exit(1);
}
/* load quantthresh */
find_seek_to(in,"static const float _vq_quantthresh_");
reset_next_value();
t->quantthresh=_ogg_malloc(sizeof(float)*t->threshvals);
for(i=0;i<t->threshvals-1;i++)
if(get_next_value(in,t->quantthresh+i)){
fprintf(stderr,"out of data 1 while reading codebook %s\n",filename);
exit(1);
}
/* load quantmap */
find_seek_to(in,"static const long _vq_quantmap_");
reset_next_value();
t->quantmap=_ogg_malloc(sizeof(long)*t->threshvals);
for(i=0;i<t->threshvals;i++)
if(get_next_ivalue(in,t->quantmap+i)){
fprintf(stderr,"out of data 2 while reading codebook %s\n",filename);
exit(1);
}
}
if(find_seek_to(in,"static const encode_aux_pigeonhole _vq_aux")){
int pigeons=1,i;
/* how big? */
c->pigeon_tree=p=_ogg_calloc(1,sizeof(encode_aux_pigeonhole));
line=get_line(in);
if(sscanf(line,"%f, %f, %d, %d",&(p->min),&(p->del),
&(p->mapentries),&(p->quantvals))!=4){
fprintf(stderr,"5: syntax in %s in line:\t %s",filename,line);
exit(1);
}
line=get_line(in);
line=get_line(in);
if(sscanf(line,"%ld",&(p->fittotal))!=1){
fprintf(stderr,"6: syntax in %s in line:\t %s",filename,line);
exit(1);
}
/* load pigeonmap */
find_seek_to(in,"static const long _vq_pigeonmap_");
reset_next_value();
p->pigeonmap=_ogg_malloc(sizeof(long)*p->mapentries);
for(i=0;i<p->mapentries;i++)
if(get_next_ivalue(in,p->pigeonmap+i)){
fprintf(stderr,"out of data (pigeonmap) while reading codebook %s\n",filename);
exit(1);
}
/* load fitlist */
find_seek_to(in,"static const long _vq_fitlist_");
reset_next_value();
p->fitlist=_ogg_malloc(sizeof(long)*p->fittotal);
for(i=0;i<p->fittotal;i++)
if(get_next_ivalue(in,p->fitlist+i)){
fprintf(stderr,"out of data (fitlist) while reading codebook %s\n",filename);
exit(1);
}
/* load fitmap */
find_seek_to(in,"static const long _vq_fitmap_");
reset_next_value();
for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
p->fitmap=_ogg_malloc(sizeof(long)*pigeons);
for(i=0;i<pigeons;i++)
if(get_next_ivalue(in,p->fitmap+i)){
fprintf(stderr,"out of data (fitmap) while reading codebook %s\n",filename);
exit(1);
}
/* load fitlength */
find_seek_to(in,"static const long _vq_fitlength_");
reset_next_value();
p->fitlength=_ogg_malloc(sizeof(long)*pigeons);
for(i=0;i<pigeons;i++)
if(get_next_ivalue(in,p->fitlength+i)){
fprintf(stderr,"out of data (fitlength) while reading codebook %s\n",filename);
exit(1);
}
}
switch(c->maptype){
case 0:
quant_to_read=0;
......@@ -456,6 +298,7 @@ codebook *codebook_load(char *filename){
fclose(in);
vorbis_book_init_encode(b,c);
b->valuelist=_book_unquantize(c,c->entries,NULL);
return(b);
}
......@@ -592,9 +435,6 @@ void build_tree_from_lengths0(int vals, long *hist, long *lengths){
}
void write_codebook(FILE *out,char *name,const static_codebook *c){
encode_aux_pigeonhole *p=c->pigeon_tree;
encode_aux_threshmatch *t=c->thresh_tree;
encode_aux_nearestmatch *n=c->nearest_tree;
int i,j,k;
/* save the book in C header form */
......@@ -620,137 +460,6 @@ void write_codebook(FILE *out,char *name,const static_codebook *c){
}
fprintf(out,"};\n\n");
if(t){
/* quantthresh */
fprintf(out,"static const float _vq_quantthresh_%s[] = {\n",name);
for(j=0;j<t->threshvals-1;){
fprintf(out,"\t");
for(k=0;k<8 && j<t->threshvals-1;k++,j++)
fprintf(out,"%.5g, ",t->quantthresh[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* quantmap */
fprintf(out,"static const long _vq_quantmap_%s[] = {\n",name);
for(j=0;j<t->threshvals;){
fprintf(out,"\t");
for(k=0;k<8 && j<t->threshvals;k++,j++)
fprintf(out,"%5ld,",t->quantmap[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
fprintf(out,"static const encode_aux_threshmatch _vq_auxt_%s = {\n",name);
fprintf(out,"\t(float *)_vq_quantthresh_%s,\n",name);
fprintf(out,"\t(long *)_vq_quantmap_%s,\n",name);
fprintf(out,"\t%d,\n",t->quantvals);
fprintf(out,"\t%d\n};\n\n",t->threshvals);
}
if(p){
int pigeons=1;
for(i=0;i<c->dim;i++)pigeons*=p->quantvals;
/* pigeonmap */
fprintf(out,"static const long _vq_pigeonmap_%s[] = {\n",name);
for(j=0;j<p->mapentries;){
fprintf(out,"\t");
for(k=0;k<8 && j<p->mapentries;k++,j++)
fprintf(out,"%5ld, ",p->pigeonmap[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* fitlist */
fprintf(out,"static const long _vq_fitlist_%s[] = {\n",name);
for(j=0;j<p->fittotal;){
fprintf(out,"\t");
for(k=0;k<8 && j<p->fittotal;k++,j++)
fprintf(out,"%5ld, ",p->fitlist[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* fitmap */
fprintf(out,"static const long _vq_fitmap_%s[] = {\n",name);
for(j=0;j<pigeons;){
fprintf(out,"\t");
for(k=0;k<8 && j<pigeons;k++,j++)
fprintf(out,"%5ld, ",p->fitmap[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* fitlength */
fprintf(out,"static const long _vq_fitlength_%s[] = {\n",name);
for(j=0;j<pigeons;){
fprintf(out,"\t");
for(k=0;k<8 && j<pigeons;k++,j++)
fprintf(out,"%5ld, ",p->fitlength[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
fprintf(out,"static const encode_aux_pigeonhole _vq_auxp_%s = {\n",name);
fprintf(out,"\t%g, %g, %d, %d,\n",
p->min,p->del,p->mapentries,p->quantvals);
fprintf(out,"\t_vq_pigeonmap_%s,\n",name);
fprintf(out,"\t%ld,\n",p->fittotal);
fprintf(out,"\t(long *)_vq_fitlist_%s,\n",name);
fprintf(out,"\t(long *)_vq_fitmap_%s,\n",name);
fprintf(out,"\t(long *)_vq_fitlength_%s\n};\n\n",name);
}
if(n){
/* ptr0 */
fprintf(out,"static const long _vq_ptr0_%s[] = {\n",name);
for(j=0;j<n->aux;){
fprintf(out,"\t");
for(k=0;k<8 && j<n->aux;k++,j++)
fprintf(out,"%6ld,",n->ptr0[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* ptr1 */
fprintf(out,"static const long _vq_ptr1_%s[] = {\n",name);
for(j=0;j<n->aux;){
fprintf(out,"\t");
for(k=0;k<8 && j<n->aux;k++,j++)
fprintf(out,"%6ld,",n->ptr1[j]);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* p */
fprintf(out,"static const long _vq_p_%s[] = {\n",name);
for(j=0;j<n->aux;){
fprintf(out,"\t");
for(k=0;k<8 && j<n->aux;k++,j++)
fprintf(out,"%6ld,",n->p[j]*c->dim);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
/* q */
fprintf(out,"static const long _vq_q_%s[] = {\n",name);
for(j=0;j<n->aux;){
fprintf(out,"\t");
for(k=0;k<8 && j<n->aux;k++,j++)
fprintf(out,"%6ld,",n->q[j]*c->dim);
fprintf(out,"\n");
}
fprintf(out,"};\n\n");
fprintf(out,"static const encode_aux_nearestmatch _vq_auxn_%s = {\n",name);
fprintf(out,"\t(long *)_vq_ptr0_%s,\n",name);
fprintf(out,"\t(long *)_vq_ptr1_%s,\n",name);
fprintf(out,"\t(long *)_vq_p_%s,\n",name);
fprintf(out,"\t(long *)_vq_q_%s,\n",name);
fprintf(out,"\t%ld, %ld\n};\n\n",n->aux,n->aux);
}
/* tie it all together */
fprintf(out,"static const static_codebook %s = {\n",name);
......@@ -764,18 +473,5 @@ void write_codebook(FILE *out,char *name,const static_codebook *c){
else
fprintf(out,"\tNULL,\n");
if(n)
fprintf(out,"\t(encode_aux_nearestmatch *)&_vq_auxn_%s,\n",name);
else
fprintf(out,"\tNULL,\n");
if(t)
fprintf(out,"\t(encode_aux_threshmatch *)&_vq_auxt_%s,\n",name);
else
fprintf(out,"\tNULL,\n");
if(p)
fprintf(out,"\t(encode_aux_pigeonhole *)&_vq_auxp_%s,\n",name);
else
fprintf(out,"\tNULL,\n");
fprintf(out,"\t0\n};\n\n");
}
......@@ -178,9 +178,6 @@ int main(int argc, char *argv[]){
fprintf(file,"\t0, 0, 0, 0, 0,\n");
fprintf(file,"\tNULL,\n");
fprintf(file,"\tNULL,\n");
fprintf(file,"\tNULL,\n");
fprintf(file,"\tNULL,\n");
fprintf(file,"\t0\n};\n\n");
fclose(file);
......
/********************************************************************
* *
* THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
* USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
* GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
* IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
* *
* THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
* by the Xiph.Org Foundation http://www.xiph.org/ *
* *
********************************************************************
function: utility main for building thresh/pigeonhole encode hints
last mod: $Id$
********************************************************************/
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <errno.h>
#include "../lib/scales.h"
#include "bookutil.h"
#include "vqgen.h"
#include "vqsplit.h"
/* The purpose of this util is to build encode hints for lattice
codebooks so that brute forcing each codebook entry isn't needed.
Threshhold hints are for books in which each scalar in the vector
is independant (eg, residue) and pigeonhole lookups provide a
minimum error fit for words where the scalars are interdependant
(each affecting the fit of the next in sequence) as in an LSP
sequential book (or can be used along with a sparse threshhold map,
like a splitting tree that need not be trained)
If the input book is non-sequential, a threshhold hint is built.
If the input book is sequential, a pigeonholing hist is built.
If the book is sparse, a pigeonholing hint is built, possibly in addition
to the threshhold hint
command line:
latticehint book.vqh [threshlist]
latticehint produces book.vqh on stdout */
static int longsort(const void *a, const void *b){
return(**((long **)a)-**((long **)b));
}
static int addtosearch(int entry,long **tempstack,long *tempcount,int add){
long *ptr=tempstack[entry];
long i=tempcount[entry];
if(ptr){
while(i--)
if(*ptr++==add)return(0);
tempstack[entry]=_ogg_realloc(tempstack[entry],
(tempcount[entry]+1)*sizeof(long));
}else{
tempstack[entry]=_ogg_malloc(sizeof(long));
}
tempstack[entry][tempcount[entry]++]=add;
return(1);
}
static void setvals(int dim,encode_aux_pigeonhole *p,
long *temptrack,float *tempmin,float *tempmax,
int seqp){
int i;
float last=0.f;
for(i=0;i<dim;i++){
tempmin[i]=(temptrack[i])*p->del+p->min+last;
tempmax[i]=tempmin[i]+p->del;
if(seqp)last=tempmin[i];
}
}
/* note that things are currently set up such that input fits that
quantize outside the pigeonmap are dropped and brute-forced. So we
can ignore the <0 and >=n boundary cases in min/max error */
static float minerror(int dim,float *a,encode_aux_pigeonhole *p,
long *temptrack,float *tempmin,float *tempmax){
int i;
float err=0.f;
for(i=0;i<dim;i++){
float eval=0.f;
if(a[i]<tempmin[i]){
eval=tempmin[i]-a[i];
}else if(a[i]>tempmax[i]){
eval=a[i]-tempmax[i];
}
err+=eval*eval;
}
return(err);
}
static float maxerror(int dim,float *a,encode_aux_pigeonhole *p,
long *temptrack,float *tempmin,float *tempmax){
int i;
float err=0.f,eval;
for(i=0;i<dim;i++){
if(a[i]<tempmin[i]){
eval=tempmax[i]-a[i];
}else if(a[i]>tempmax[i]){
eval=a[i]-tempmin[i];
}else{
float t1=a[i]-tempmin[i];
eval=tempmax[i]-a[i];
if(t1>eval)eval=t1;
}
err+=eval*eval;
}
return(err);
}
int main(int argc,char *argv[]){
codebook *b;
static_codebook *c;
int entries=-1,dim=-1;
float min,del;
char *name;
long i,j;
float *suggestions;
int suggcount=0;