Commit 6bc6a4ad authored by conrad's avatar conrad
Browse files

libfishsound flac: chase malloc/realloc failures, and check

libflac encode/decode status for memory allocation errors
For Mozilla bug 468293

git-svn-id: http://svn.annodex.net/libfishsound/trunk@3854 8158c8cd-e7e1-0310-9fa4-c5954c97daef
parent 5bf59dc8
...@@ -141,6 +141,7 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder, ...@@ -141,6 +141,7 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder,
FishSound* fsound = (FishSound*)client_data; FishSound* fsound = (FishSound*)client_data;
FishSoundFlacInfo* fi = (FishSoundFlacInfo *)fsound->codec_data; FishSoundFlacInfo* fi = (FishSoundFlacInfo *)fsound->codec_data;
int i, j, channels, blocksize, offset; int i, j, channels, blocksize, offset;
float * ipcm;
channels = frame->header.channels; channels = frame->header.channels;
blocksize = frame->header.blocksize; blocksize = frame->header.blocksize;
...@@ -158,7 +159,10 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder, ...@@ -158,7 +159,10 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder,
FishSoundDecoded_FloatIlv dfi; FishSoundDecoded_FloatIlv dfi;
float* retpcm; float* retpcm;
fi->ipcm = realloc(fi->ipcm, sizeof(float) * channels * blocksize); if ((ipcm = realloc(fi->ipcm, sizeof(float) * channels * blocksize)) == NULL)
return FLAC__STREAM_DECODER_WRITE_STATUS_ABORT;
fi->ipcm = ipcm;
retpcm = (float*) fi->ipcm; retpcm = (float*) fi->ipcm;
for (i = 0; i < blocksize; i++) { for (i = 0; i < blocksize; i++) {
offset = i * channels; offset = i * channels;
...@@ -173,7 +177,9 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder, ...@@ -173,7 +177,9 @@ fs_flac_write_callback(const FLAC__StreamDecoder *decoder,
float *d; /* de-interleave dest */ float *d; /* de-interleave dest */
for (j = 0; j < channels; j++) { for (j = 0; j < channels; j++) {
fi->pcm_out[j] = realloc(fi->pcm_out[j], sizeof(float) * blocksize); if ((ipcm = realloc(fi->pcm_out[j], sizeof(float) * blocksize)) == NULL)
return FLAC__STREAM_DECODER_WRITE_STATUS_ABORT;
fi->pcm_out[j] = ipcm;
} }
for (i = 0; i < blocksize; i++) for (i = 0; i < blocksize; i++)
for (j = 0; j < channels; j++) { for (j = 0; j < channels; j++) {
...@@ -296,12 +302,14 @@ fs_flac_decode (FishSound * fsound, unsigned char * buf, long bytes) ...@@ -296,12 +302,14 @@ fs_flac_decode (FishSound * fsound, unsigned char * buf, long bytes)
#endif #endif
return -1; return -1;
} }
fi->buffer = fs_malloc(sizeof(unsigned char)*bytes); if ((fi->buffer = fs_malloc(sizeof(unsigned char)*bytes)) == NULL)
return FISH_SOUND_ERR_OUT_OF_MEMORY;
memcpy(fi->buffer, buf+9, bytes-9); memcpy(fi->buffer, buf+9, bytes-9);
fi->bufferlength = bytes-9; fi->bufferlength = bytes-9;
} }
else if (fi->packetno <= fi->header_packets){ else if (fi->packetno <= fi->header_packets){
unsigned char* tmp = fs_malloc(sizeof(unsigned char)*(fi->bufferlength+bytes)); unsigned char* tmp;
#ifdef DEBUG #ifdef DEBUG
printf("fs_flac_decode: handling header (fi->header_packets = %d)\n", printf("fs_flac_decode: handling header (fi->header_packets = %d)\n",
fi->header_packets); fi->header_packets);
...@@ -322,23 +330,38 @@ fs_flac_decode (FishSound * fsound, unsigned char * buf, long bytes) ...@@ -322,23 +330,38 @@ fs_flac_decode (FishSound * fsound, unsigned char * buf, long bytes)
} }
} }
if ((tmp = fs_malloc(sizeof(unsigned char)*(fi->bufferlength+bytes))) == NULL)
return FISH_SOUND_ERR_OUT_OF_MEMORY;
memcpy(tmp, fi->buffer, fi->bufferlength); memcpy(tmp, fi->buffer, fi->bufferlength);
memcpy(tmp+fi->bufferlength, buf, bytes); memcpy(tmp+fi->bufferlength, buf, bytes);
fi->bufferlength += bytes; fi->bufferlength += bytes;
fs_free(fi->buffer); fs_free(fi->buffer);
fi->buffer = tmp; fi->buffer = tmp;
if (fi->packetno == fi->header_packets) { if (fi->packetno == fi->header_packets) {
FLAC__stream_decoder_process_until_end_of_metadata(fi->fsd); if (FLAC__stream_decoder_process_until_end_of_metadata(fi->fsd) == false) {
goto dec_err;
}
fs_free(fi->buffer); fs_free(fi->buffer);
} }
} else { } else {
fi->buffer = buf; fi->buffer = buf;
fi->bufferlength = bytes; fi->bufferlength = bytes;
FLAC__stream_decoder_process_single(fi->fsd); if (FLAC__stream_decoder_process_single(fi->fsd) == false) {
goto dec_err;
}
} }
fi->packetno++; fi->packetno++;
return 0; return 0;
dec_err:
switch (FLAC__stream_decoder_get_state(fi->fsd)) {
case FLAC__STREAM_DECODER_MEMORY_ALLOCATION_ERROR:
return FISH_SOUND_ERR_OUT_OF_MEMORY;
default:
return FISH_SOUND_ERR_GENERIC;
}
} }
#else /* !FS_DECODE */ #else /* !FS_DECODE */
...@@ -372,7 +395,9 @@ fs_flac_enc_write_callback(const FLAC__StreamEncoder *encoder, ...@@ -372,7 +395,9 @@ fs_flac_enc_write_callback(const FLAC__StreamEncoder *encoder,
printf("fs_flac_enc_write_callback: generating FLAC header packet: " printf("fs_flac_enc_write_callback: generating FLAC header packet: "
"%c%c%c%c\n", buffer[0], buffer[1], buffer[2], buffer[3]); "%c%c%c%c\n", buffer[0], buffer[1], buffer[2], buffer[3]);
#endif #endif
fi->buffer = (unsigned char*)fs_malloc(sizeof(unsigned char)*(bytes+9)); if ((fi->buffer = (unsigned char*)fs_malloc(sizeof(unsigned char)*(bytes+9))) == NULL)
return FLAC__STREAM_ENCODER_WRITE_STATUS_FATAL_ERROR;
fi->buffer[0] = 0x7f; fi->buffer[0] = 0x7f;
fi->buffer[1] = 0x46; /* 'F' */ fi->buffer[1] = 0x46; /* 'F' */
fi->buffer[2] = 0x4c; /* 'L' */ fi->buffer[2] = 0x4c; /* 'L' */
...@@ -390,7 +415,11 @@ fs_flac_enc_write_callback(const FLAC__StreamEncoder *encoder, ...@@ -390,7 +415,11 @@ fs_flac_enc_write_callback(const FLAC__StreamEncoder *encoder,
/* Make a temporary copy of the metadata header to pass to the user /* Make a temporary copy of the metadata header to pass to the user
* callback. * callback.
*/ */
unsigned char* tmp = (unsigned char*)fs_malloc(sizeof(unsigned char)*(bytes+fi->bufferlength)); unsigned char* tmp;
if ((tmp = (unsigned char*)fs_malloc(sizeof(unsigned char)*(bytes+fi->bufferlength))) == NULL)
return FLAC__STREAM_ENCODER_WRITE_STATUS_FATAL_ERROR;
memcpy (tmp, fi->buffer, fi->bufferlength); memcpy (tmp, fi->buffer, fi->bufferlength);
memcpy (tmp+fi->bufferlength, buffer, bytes); memcpy (tmp+fi->bufferlength, buffer, bytes);
fs_free(fi->buffer); fs_free(fi->buffer);
...@@ -481,7 +510,8 @@ fs_flac_encode_vcentry (const FishSoundComment * comment) ...@@ -481,7 +510,8 @@ fs_flac_encode_vcentry (const FishSoundComment * comment)
length += value_len + 1; length += value_len + 1;
} }
entry = fs_malloc (length); if ((entry = fs_malloc (length)) == NULL)
return NULL;
/* We assume that comment->name, value are NUL terminated, as they were /* We assume that comment->name, value are NUL terminated, as they were
* produced by our own comments.c */ * produced by our own comments.c */
...@@ -501,7 +531,7 @@ static FLAC__StreamMetadata * ...@@ -501,7 +531,7 @@ static FLAC__StreamMetadata *
fs_flac_encode_vorbiscomments (FishSound * fsound) fs_flac_encode_vorbiscomments (FishSound * fsound)
{ {
FishSoundFlacInfo * fi = fsound->codec_data; FishSoundFlacInfo * fi = fsound->codec_data;
FLAC__StreamMetadata * metadata; FLAC__StreamMetadata * metadata = NULL;
const FishSoundComment * comment; const FishSoundComment * comment;
unsigned int i=0, length=0, total_length; unsigned int i=0, length=0, total_length;
FLAC__VCEntry * comments; FLAC__VCEntry * comments;
...@@ -521,11 +551,13 @@ fs_flac_encode_vorbiscomments (FishSound * fsound) ...@@ -521,11 +551,13 @@ fs_flac_encode_vorbiscomments (FishSound * fsound)
if (length == 0) return NULL; if (length == 0) return NULL;
comments = (FLAC__VCEntry *)fs_malloc (sizeof(FLAC__VCEntry) * length); if ((comments = (FLAC__VCEntry *)fs_malloc (sizeof(FLAC__VCEntry) * length)) == NULL)
goto encode_vc_oom;
for (comment = fish_sound_comment_first (fsound); comment; for (comment = fish_sound_comment_first (fsound); comment;
comment = fish_sound_comment_next (fsound, comment)) { comment = fish_sound_comment_next (fsound, comment)) {
comments[i].entry = fs_flac_encode_vcentry (comment); if ((comments[i].entry = fs_flac_encode_vcentry (comment)) == NULL) {
}
comments[i].length = strlen((char *)comments[i].entry); comments[i].length = strlen((char *)comments[i].entry);
/* In the generated vorbiscomment data, each entry is preceded by a /* In the generated vorbiscomment data, each entry is preceded by a
...@@ -534,7 +566,9 @@ fs_flac_encode_vorbiscomments (FishSound * fsound) ...@@ -534,7 +566,9 @@ fs_flac_encode_vorbiscomments (FishSound * fsound)
i++; i++;
} }
metadata = (FLAC__StreamMetadata *) fs_malloc (sizeof (*metadata)); if ((metadata = (FLAC__StreamMetadata *) fs_malloc (sizeof (*metadata))) == NULL)
goto encode_vc_oom;
metadata->type = FLAC__METADATA_TYPE_VORBIS_COMMENT; metadata->type = FLAC__METADATA_TYPE_VORBIS_COMMENT;
metadata->is_last = true; metadata->is_last = true;
metadata->length = total_length; metadata->length = total_length;
...@@ -546,6 +580,21 @@ fs_flac_encode_vorbiscomments (FishSound * fsound) ...@@ -546,6 +580,21 @@ fs_flac_encode_vorbiscomments (FishSound * fsound)
fi->enc_vc_metadata = metadata; fi->enc_vc_metadata = metadata;
return metadata; return metadata;
encode_vc_oom:
if (metadata != NULL)
fs_free (metadata);
/* Unwind allocated comment entries */
for (i--; i >= 0; i--) {
if (comments[i].entry != NULL)
fs_free (comments[i].entry);
}
if (comments != NULL)
fs_free (comments);
return NULL;
} }
static FishSound * static FishSound *
...@@ -590,11 +639,19 @@ fs_flac_enc_headers (FishSound * fsound) ...@@ -590,11 +639,19 @@ fs_flac_enc_headers (FishSound * fsound)
return fsound; return fsound;
} }
static long
fs_flac_encode_fatal (FishSoundFlacInfo *fi, long err)
{
FLAC__stream_encoder_delete (fi->fse);
fi->fse = NULL;
return err;
}
static long static long
fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames) fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames)
{ {
FishSoundFlacInfo *fi = fsound->codec_data; FishSoundFlacInfo *fi = fsound->codec_data;
FLAC__int32 *buffer; FLAC__int32 *buffer, *ipcm;
float * p, norm = (1 << (BITS_PER_SAMPLE - 1)); float * p, norm = (1 << (BITS_PER_SAMPLE - 1));
long i; long i;
int j, channels = fsound->info.channels; int j, channels = fsound->info.channels;
...@@ -603,7 +660,10 @@ fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames) ...@@ -603,7 +660,10 @@ fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames)
printf("fs_flac_encode_f: IN, frames = %ld\n", frames); printf("fs_flac_encode_f: IN, frames = %ld\n", frames);
#endif #endif
fi->ipcm = realloc(fi->ipcm, sizeof(FLAC__int32) * channels * frames); if ((ipcm = realloc(fi->ipcm, sizeof(FLAC__int32) * channels * frames)) == NULL)
return FISH_SOUND_ERR_OUT_OF_MEMORY;
fi->ipcm = ipcm;
buffer = (FLAC__int32*) fi->ipcm; buffer = (FLAC__int32*) fi->ipcm;
for (i = 0; i < frames; i++) { for (i = 0; i < frames; i++) {
for (j = 0; j < channels; j++) { for (j = 0; j < channels; j++) {
...@@ -617,7 +677,17 @@ fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames) ...@@ -617,7 +677,17 @@ fs_flac_encode_f (FishSound * fsound, float * pcm[], long frames)
/* We could have used FLAC__stream_encoder_process() and a more direct /* We could have used FLAC__stream_encoder_process() and a more direct
* conversion loop above, rather than converting and interleaving. */ * conversion loop above, rather than converting and interleaving. */
FLAC__stream_encoder_process_interleaved(fi->fse, buffer, frames); if (FLAC__stream_encoder_process_interleaved(fi->fse, buffer, frames) == false) {
switch (FLAC__stream_encoder_get_state (fi->fse)) {
case FLAC__STREAM_ENCODER_OK:
case FLAC__STREAM_ENCODER_UNINITIALIZED:
break;
case FLAC__STREAM_ENCODER_MEMORY_ALLOCATION_ERROR:
return fs_flac_encode_fatal (fi, FISH_SOUND_ERR_OUT_OF_MEMORY);
default:
return fs_flac_encode_fatal (fi, FISH_SOUND_ERR_GENERIC);
}
}
fi->packetno++; fi->packetno++;
...@@ -628,7 +698,7 @@ static long ...@@ -628,7 +698,7 @@ static long
fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames) fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames)
{ {
FishSoundFlacInfo *fi = fsound->codec_data; FishSoundFlacInfo *fi = fsound->codec_data;
FLAC__int32 *buffer; FLAC__int32 *buffer, *ipcm;
float * p = (float*)pcm, norm = (1 << (BITS_PER_SAMPLE - 1)); float * p = (float*)pcm, norm = (1 << (BITS_PER_SAMPLE - 1));
long i, length = frames * fsound->info.channels; long i, length = frames * fsound->info.channels;
...@@ -636,7 +706,10 @@ fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames) ...@@ -636,7 +706,10 @@ fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames)
printf("fs_flac_encode_f_ilv: IN, frames = %ld\n", frames); printf("fs_flac_encode_f_ilv: IN, frames = %ld\n", frames);
#endif #endif
fi->ipcm = realloc(fi->ipcm, sizeof(FLAC__int32)*fsound->info.channels*frames); if ((ipcm = realloc(fi->ipcm, sizeof(FLAC__int32)*fsound->info.channels*frames)) == NULL)
return FISH_SOUND_ERR_OUT_OF_MEMORY;
fi->ipcm = ipcm;
buffer = (FLAC__int32*) fi->ipcm; buffer = (FLAC__int32*) fi->ipcm;
for (i=0; i<length; i++) for (i=0; i<length; i++)
buffer[i] = p[i] * norm; buffer[i] = p[i] * norm;
...@@ -644,7 +717,17 @@ fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames) ...@@ -644,7 +717,17 @@ fs_flac_encode_f_ilv (FishSound * fsound, float ** pcm, long frames)
if (fi->packetno == 0) if (fi->packetno == 0)
fs_flac_enc_headers (fsound); fs_flac_enc_headers (fsound);
FLAC__stream_encoder_process_interleaved(fi->fse, buffer, frames); if (FLAC__stream_encoder_process_interleaved(fi->fse, buffer, frames) == false) {
switch (FLAC__stream_encoder_get_state (fi->fse)) {
case FLAC__STREAM_ENCODER_OK:
case FLAC__STREAM_ENCODER_UNINITIALIZED:
break;
case FLAC__STREAM_ENCODER_MEMORY_ALLOCATION_ERROR:
return fs_flac_encode_fatal (fi, FISH_SOUND_ERR_OUT_OF_MEMORY);
default:
return fs_flac_encode_fatal (fi, FISH_SOUND_ERR_GENERIC);
}
}
fi->packetno++; fi->packetno++;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment