format_ebml.c 30.4 KB
Newer Older
1 2
/* Icecast
 *
3
 * This program is distributed under the GNU General Public License,
4 5 6
 * version 2. A copy of this license is included with this source.
 * At your option, this specific source file can also be distributed
 * under the GNU GPL version 3.
7
 *
8
 * Copyright 2012,      David Richards, Mozilla Foundation,
9
 *                      and others (see AUTHORS for details).
10
 * Copyright 2014,      Philipp "ph3-der-loewe" Schafft <lion@lion.leolix.org>.
11 12 13 14
 */

/* format_ebml.c
 *
giles's avatar
giles committed
15
 * format plugin for WebM/EBML
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "refbuf.h"
#include "source.h"
#include "client.h"

#include "stats.h"
#include "format.h"
#include "format_ebml.h"

#define CATMODULE "format-ebml"

#include "logging.h"

#define EBML_HEADER_MAX_SIZE 131072
#define EBML_SLICE_SIZE 4096

42 43 44
/* A value that no EBML var-int is allowed to take. */
#define EBML_UNKNOWN ((unsigned long long) -1)

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
/* The magic numbers for each element we are interested in.
 * Defined here:
 * http://www.matroska.org/technical/specs/index.html
 * http://www.webmproject.org/docs/container/
 *
 * Some of the higher-level elements have 4-byte identifiers;
 * The lower-level elements have 1-byte identifiers.
 */
#define UNCOMMON_MAGIC_LEN 4

#define SEGMENT_MAGIC "\x18\x53\x80\x67"
#define CLUSTER_MAGIC "\x1F\x43\xB6\x75"
#define TRACKS_MAGIC "\x16\x54\xAE\x6B"

#define COMMON_MAGIC_LEN 1

#define TRACK_ENTRY_MAGIC "\xAE"
#define TRACK_NUMBER_MAGIC "\xD7"
#define TRACK_TYPE_MAGIC "\x83"
#define SIMPLE_BLOCK_MAGIC "\xA3"

66 67 68 69
typedef enum ebml_read_mode {
    EBML_STATE_READING_HEADER = 0,
    EBML_STATE_READING_CLUSTERS
} ebml_read_mode;
70

71 72 73 74 75 76 77 78
typedef enum ebml_parsing_state {
    EBML_STATE_PARSING_HEADER = 0,
    EBML_STATE_COPYING_TO_HEADER,
    EBML_STATE_START_CLUSTER,
    EBML_STATE_PARSING_CLUSTERS,
    EBML_STATE_COPYING_TO_DATA
} ebml_parsing_state;

79 80 81 82 83 84
typedef enum ebml_chunk_type {
    EBML_CHUNK_HEADER = 0,
    EBML_CHUNK_CLUSTER_START,
    EBML_CHUNK_CLUSTER_CONTINUE
} ebml_chunk_type;

85 86 87 88 89 90
typedef enum ebml_keyframe_status {
    EBML_KEYFRAME_UNKNOWN = -1,
    EBML_KEYFRAME_DOES_NOT_START_CLUSTER = 0,
    EBML_KEYFRAME_STARTS_CLUSTER = 1
} ebml_keyframe_status;

91 92 93 94 95 96 97 98 99 100 101
typedef struct ebml_client_data_st ebml_client_data_t;

struct ebml_client_data_st {

    refbuf_t *header;
    int header_pos;

};

struct ebml_st {

102
    ebml_read_mode output_state;
103 104
    ebml_parsing_state parse_state;
    unsigned long long copy_len;
105

106
    int cluster_start;
107
    ebml_keyframe_status cluster_starts_with_keyframe;
108
    int flush_cluster;
109

110 111
    int position;
    unsigned char *buffer;
112 113 114

    int input_position;
    unsigned char *input_buffer;
115

116 117 118 119
    int header_size;
    int header_position;
    int header_read_position;
    unsigned char *header;
120

121 122 123
    unsigned long long keyframe_track_number;
    unsigned long long parsing_track_number;
    int parsing_track_is_video;
124 125
};

126 127 128 129 130 131
static void ebml_free_plugin(format_plugin_t *plugin);
static refbuf_t *ebml_get_buffer(source_t *source);
static int ebml_write_buf_to_client(client_t *client);
static void ebml_write_buf_to_file(source_t *source, refbuf_t *refbuf);
static int ebml_create_client_data(source_t *source, client_t *client);
static void ebml_free_client_data(client_t *client);
132 133 134 135

static ebml_t *ebml_create();
static void ebml_destroy(ebml_t *ebml);
static int ebml_read_space(ebml_t *ebml);
136
static int ebml_read(ebml_t *ebml, char *buffer, int len, ebml_chunk_type *chunk_type);
137
static unsigned char *ebml_get_write_buffer(ebml_t *ebml, int *bytes);
138
static int ebml_wrote(ebml_t *ebml, int len);
139 140 141 142 143 144
static int ebml_parse_tag(unsigned char      *buffer,
                          unsigned char      *buffer_end,
                          unsigned long long *payload_length);
static int ebml_parse_var_int(unsigned char      *buffer,
                              unsigned char      *buffer_end,
                              unsigned long long *out_value);
145 146 147 148 149
static int ebml_parse_sized_int(unsigned char      *buffer,
                                unsigned char      *buffer_end,
                                int                len,
                                int                is_signed,
                                unsigned long long *out_value);
150
static inline void ebml_check_track(ebml_t *ebml);
151

152
int format_ebml_get_plugin(source_t *source)
153 154 155 156 157 158 159 160 161 162 163 164 165
{

    ebml_source_state_t *ebml_source_state = calloc(1, sizeof(ebml_source_state_t));
    format_plugin_t *plugin = calloc(1, sizeof(format_plugin_t));

    plugin->get_buffer = ebml_get_buffer;
    plugin->write_buf_to_client = ebml_write_buf_to_client;
    plugin->create_client_data = ebml_create_client_data;
    plugin->free_plugin = ebml_free_plugin;
    plugin->write_buf_to_file = ebml_write_buf_to_file;
    plugin->set_tag = NULL;
    plugin->apply_settings = NULL;

166
    plugin->contenttype = httpp_getvar(source->parser, "content-type");
167 168

    plugin->_state = ebml_source_state;
169
    vorbis_comment_init(&plugin->vc);
170 171 172
    source->format = plugin;

    ebml_source_state->ebml = ebml_create();
173

174 175 176
    return 0;
}

177
static void ebml_free_plugin(format_plugin_t *plugin)
178 179 180 181
{

    ebml_source_state_t *ebml_source_state = plugin->_state;

182
    refbuf_release(ebml_source_state->header);
183
    ebml_destroy(ebml_source_state->ebml);
184
    free(ebml_source_state);
185
    vorbis_comment_clear(&plugin->vc);
186
    free(plugin);
187 188
}

189 190
/* Write to a client from the header buffer.
 */
191
static int send_ebml_header(client_t *client)
192 193 194 195 196 197
{

    ebml_client_data_t *ebml_client_data = client->format_data;
    int len = EBML_SLICE_SIZE;
    int ret;

198
    if (ebml_client_data->header->len - ebml_client_data->header_pos < len)
199 200 201
    {
        len = ebml_client_data->header->len - ebml_client_data->header_pos;
    }
202
    ret = client_send_bytes (client,
203 204 205 206 207 208 209 210 211 212 213 214
                             ebml_client_data->header->data + ebml_client_data->header_pos,
                             len);

    if (ret > 0)
    {
        ebml_client_data->header_pos += ret;
    }

    return ret;

}

215 216
/* Initial write-to-client function.
 */
217 218 219 220 221 222 223 224 225 226 227
static int ebml_write_buf_to_client (client_t *client)
{

    ebml_client_data_t *ebml_client_data = client->format_data;

    if (ebml_client_data->header_pos != ebml_client_data->header->len)
    {
        return send_ebml_header (client);
    }
    else
    {
228 229
        /* Now that the header's sent, short-circuit to the generic
         * write-refbufs function. */
230 231 232 233 234 235
        client->write_to_client = format_generic_write_to_client;
        return client->write_to_client(client);
    }

}

236 237
/* Return a refbuf to add to the queue.
 */
238
static refbuf_t *ebml_get_buffer(source_t *source)
239 240 241 242
{

    ebml_source_state_t *ebml_source_state = source->format->_state;
    format_plugin_t *format = source->format;
243 244 245
    unsigned char *write_buffer = NULL;
    int read_bytes = 0;
    int write_bytes = 0;
246
    ebml_chunk_type chunk_type;
247 248 249 250 251
    refbuf_t *refbuf;
    int ret;

    while (1)
    {
252 253
        read_bytes = ebml_read_space(ebml_source_state->ebml);
        if (read_bytes > 0) {
254
            /* A chunk is available for reading */
255 256
            refbuf = refbuf_new(read_bytes);
            ebml_read(ebml_source_state->ebml, refbuf->data, read_bytes, &chunk_type);
257 258 259

            if (ebml_source_state->header == NULL)
            {
260
                /* Capture header before adding clusters to the queue */
261 262 263 264
                ebml_source_state->header = refbuf;
                continue;
            }

265
/*            ICECAST_LOG_DEBUG("EBML: generated refbuf, size %i : %hhi %hhi %hhi",
266
 *                            read_bytes, refbuf->data[0], refbuf->data[1], refbuf->data[2]);
267
 */
268

269
            if (chunk_type == EBML_CHUNK_CLUSTER_START)
270 271
            {
                refbuf->sync_point = 1;
272
/*                ICECAST_LOG_DEBUG("EBML: ^ was sync point"); */
273 274 275
            }
            return refbuf;

276
        } else if(read_bytes == 0) {
277
            /* Feed more bytes into the parser */
278 279 280
            write_buffer = ebml_get_write_buffer(ebml_source_state->ebml, &write_bytes);
            read_bytes = client_read_bytes (source->client, write_buffer, write_bytes);
            if (read_bytes <= 0) {
281 282 283
                ebml_wrote (ebml_source_state->ebml, 0);
                return NULL;
            }
284 285 286
            format->read_bytes += read_bytes;
            ret = ebml_wrote (ebml_source_state->ebml, read_bytes);
            if (ret != read_bytes) {
287
                ICECAST_LOG_ERROR("Problem processing stream");
288 289 290
                source->running = 0;
                return NULL;
            }
291 292 293 294
        } else {
            ICECAST_LOG_ERROR("Problem processing stream");
            source->running = 0;
            return NULL;
295 296 297 298
        }
    }
}

299 300
/* Initialize client state.
 */
301
static int ebml_create_client_data(source_t *source, client_t *client)
302
{
303
    ebml_client_data_t *ebml_client_data;
304 305
    ebml_source_state_t *ebml_source_state = source->format->_state;

306 307
    if (!ebml_source_state->header)
        return -1;
308

309 310 311
    ebml_client_data = calloc(1, sizeof(ebml_client_data_t));
    if (!ebml_client_data)
        return -1;
312

313 314 315 316 317
    ebml_client_data->header = ebml_source_state->header;
    refbuf_addref(ebml_client_data->header);
    client->format_data = ebml_client_data;
    client->free_client_data = ebml_free_client_data;
    return 0;
318 319 320 321 322 323 324 325 326 327 328 329 330 331
}

static void ebml_free_client_data (client_t *client)
{

    ebml_client_data_t *ebml_client_data = client->format_data;

    refbuf_release (ebml_client_data->header);
    free (client->format_data);
    client->format_data = NULL;
}

static void ebml_write_buf_to_file_fail (source_t *source)
{
332
    ICECAST_LOG_WARN("Write to dump file failed, disabling");
333 334 335 336 337 338 339 340 341 342 343 344
    fclose (source->dumpfile);
    source->dumpfile = NULL;
}

static void ebml_write_buf_to_file (source_t *source, refbuf_t *refbuf)
{

    ebml_source_state_t *ebml_source_state = source->format->_state;

    if (ebml_source_state->file_headers_written == 0)
    {
        if (fwrite (ebml_source_state->header->data, 1,
345
                    ebml_source_state->header->len,
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
                    source->dumpfile) != ebml_source_state->header->len)
            ebml_write_buf_to_file_fail(source);
        else
            ebml_source_state->file_headers_written = 1;
    }

    if (fwrite (refbuf->data, 1, refbuf->len, source->dumpfile) != refbuf->len)
    {
        ebml_write_buf_to_file_fail(source);
    }

}

/* internal ebml parsing */

static void ebml_destroy(ebml_t *ebml)
{

    free(ebml->header);
    free(ebml->input_buffer);
    free(ebml->buffer);
    free(ebml);

}

static ebml_t *ebml_create()
{

    ebml_t *ebml = calloc(1, sizeof(ebml_t));

376 377
    ebml->output_state = EBML_STATE_READING_HEADER;

378
    ebml->header = calloc(1, EBML_HEADER_MAX_SIZE);
379
    ebml->buffer = calloc(1, EBML_SLICE_SIZE);
380 381
    ebml->input_buffer = calloc(1, EBML_SLICE_SIZE);

382
    ebml->cluster_start = -1;
383

384 385 386
    ebml->keyframe_track_number = EBML_UNKNOWN;
    ebml->parsing_track_number = EBML_UNKNOWN;
    ebml->parsing_track_is_video = 0;
387 388 389 390 391

    return ebml;

}

392 393 394
/* Return the size of a buffer needed to store the next
 * chunk that ebml_read can yield.
 */
395 396 397 398 399
static int ebml_read_space(ebml_t *ebml)
{

    int read_space;

400 401
    switch (ebml->output_state) {
        case EBML_STATE_READING_HEADER:
402

403 404 405 406 407 408 409 410
            if (ebml->header_size != 0) {
                /* The header can be read */
                return ebml->header_size;
            } else {
                /* The header's not ready yet */
                return 0;
            }
            break;
411

412
        case EBML_STATE_READING_CLUSTERS:
413

414 415 416 417
            if (ebml->cluster_start > 0) {
                /* return up until just before a new cluster starts */
                read_space = ebml->cluster_start;
            } else {
418

419 420 421 422
                if (ebml->position == EBML_SLICE_SIZE) {
                    /* The current cluster fills the buffer,
                     * we have no choice but to start flushing it.
                     */
423

424 425
                    ebml->flush_cluster = 1;
                }
426

427 428 429 430 431 432 433 434 435
                if (ebml->flush_cluster) {
                    /* return what we have */
                    read_space = ebml->position;
                } else {
                    /* wait until we've read more, so the parser has
                     * time to gather metadata
                     */
                    read_space = 0;
                }
436
            }
437

438
            return read_space;
439
    }
440

441 442
    ICECAST_LOG_ERROR("EBML: Invalid parser read state");
    return 0;
443 444
}

445
/* Return a chunk of the EBML/MKV/WebM stream.
446
 * The header will be buffered until it can be returned as one chunk.
447
 * A cluster element's opening tag will always start a new chunk.
448 449 450
 * 
 * chunk_type will be set to indicate if the chunk is the header,
 * the start of a cluster, or continuing the current cluster.
451
 */
452
static int ebml_read(ebml_t *ebml, char *buffer, int len, ebml_chunk_type *chunk_type)
453 454 455 456
{

    int read_space;
    int to_read;
457

458
    *chunk_type = EBML_CHUNK_HEADER;
459

460
    if (len < 1) {
461
        return 0;
462
    }
463

464 465
    switch (ebml->output_state) {
        case EBML_STATE_READING_HEADER:
466

467 468 469 470
            if (ebml->header_size != 0)
            {
                /* Can read a chunk of the header */
                read_space = ebml->header_size - ebml->header_read_position;
471

472 473 474 475 476
                if (read_space >= len) {
                    to_read = len;
                } else {
                    to_read = read_space;
                }
477

478 479
                memcpy(buffer, ebml->header, to_read);
                ebml->header_read_position += to_read;
480

481
                *chunk_type = EBML_CHUNK_HEADER;
482

483 484 485 486 487 488 489
                if (ebml->header_read_position == ebml->header_size) {
                    ebml->output_state = EBML_STATE_READING_CLUSTERS;
                }
            } else {
                /* The header's not ready yet */
                return 0;
            }
490

491
            break;
492

493
        case EBML_STATE_READING_CLUSTERS:
494

495 496
            *chunk_type = EBML_CHUNK_CLUSTER_CONTINUE;
            read_space = ebml->position;
497

498 499
            if (ebml->cluster_start == 0) {
                /* new cluster is starting now */
500

501 502 503 504 505 506 507
                if (ebml->cluster_starts_with_keyframe != EBML_KEYFRAME_DOES_NOT_START_CLUSTER) {
                    /* If we positively identified the first video frame as a non-keyframe,
                     * don't use this cluster as a sync point. Since some files lack
                     * video tracks completely, or we may have failed to probe
                     * the first video frame, it's better to be pass through
                     * ambiguous cases to avoid blocking the stream forever.
                     */
508 509
                    *chunk_type = EBML_CHUNK_CLUSTER_START;
                }
510

511 512 513
                /* mark end of cluster */
                ebml->cluster_start = -1;
            } else if (ebml->cluster_start > 0) {
514 515 516
                /* return up until just before a new cluster starts */
                read_space = ebml->cluster_start;
            }
517

518 519 520
            if (read_space < 1) {
                return 0;
            }
521

522
            if (read_space >= len ) {
523
                to_read = len;
524
            } else {
525
                to_read = read_space;
526
            }
527

528
            memcpy(buffer, ebml->buffer, to_read);
529

530 531 532
            /* Shift unread data down to the start of the buffer */
            memmove(ebml->buffer, ebml->buffer + to_read, ebml->position - to_read);
            ebml->position -= to_read;
533

534 535
            if (ebml->cluster_start > 0) {
                ebml->cluster_start -= to_read;
536
            }
537

538
            break;
539 540 541 542 543 544
    }

    return to_read;

}

545 546 547 548 549 550
/* Get pointer & length of the buffer able to accept input.
 * 
 * Returns the start of the writable space;
 * Sets bytes to the amount of space available.
 */
static unsigned char *ebml_get_write_buffer(ebml_t *ebml, int *bytes)
551
{
552 553
    *bytes = EBML_SLICE_SIZE - ebml->input_position;
    return ebml->input_buffer + ebml->input_position;
554 555
}

556 557
/* Process data that has been written to the EBML parser's input buffer.
 */
558 559
static int ebml_wrote(ebml_t *ebml, int len)
{
560 561 562 563
    int processing = 1;
    int cursor = 0;
    int to_copy;
    unsigned char *end_of_buffer;
564

565
    int tag_length;
566
    int value_length;
567
    int track_number_length;
568
    unsigned long long payload_length;
569
    unsigned long long data_value;
570 571
    unsigned long long track_number;
    unsigned char flags;
572
    int copy_state;
573

574 575
    ebml->input_position += len;
    end_of_buffer = ebml->input_buffer + ebml->input_position;
576

577
    while (processing) {
578

579
        /*ICECAST_LOG_DEBUG("Parse State: %i", ebml->parse_state);*/
580

581
        switch (ebml->parse_state) {
582

583 584
            case EBML_STATE_PARSING_HEADER:
            case EBML_STATE_PARSING_CLUSTERS:
585

586 587 588 589 590
                if (ebml->parse_state == EBML_STATE_PARSING_HEADER) {
                    copy_state = EBML_STATE_COPYING_TO_HEADER;
                } else {
                    copy_state = EBML_STATE_COPYING_TO_DATA;
                }
591

592 593
                tag_length = ebml_parse_tag(ebml->input_buffer + cursor,
                                            end_of_buffer, &payload_length);
594

595
                if (tag_length > 0) {
596

597 598 599 600
                    if (payload_length == EBML_UNKNOWN) {
                        /* Parse all children for tags we can't skip */
                        payload_length = 0;
                    }
601

602
                    /* Recognize tags of interest */
603 604
                    if (tag_length > UNCOMMON_MAGIC_LEN) {
                        if (!memcmp(ebml->input_buffer + cursor, CLUSTER_MAGIC, UNCOMMON_MAGIC_LEN)) {
605 606 607
                            /* Found a Cluster */
                            ebml->parse_state = EBML_STATE_START_CLUSTER;
                            break;
608
                        } else if (!memcmp(ebml->input_buffer + cursor, SEGMENT_MAGIC, UNCOMMON_MAGIC_LEN)) {
609 610
                            /* Parse all Segment children */
                            payload_length = 0;
611

612
                        } else if (!memcmp(ebml->input_buffer + cursor, TRACKS_MAGIC, UNCOMMON_MAGIC_LEN)) {
613 614
                            /* Parse all Tracks children */
                            payload_length = 0;
615

616
                        }
617

618
                    }
619

620 621
                    if (tag_length > COMMON_MAGIC_LEN) {
                        if (!memcmp(ebml->input_buffer + cursor, SIMPLE_BLOCK_MAGIC, COMMON_MAGIC_LEN)) {
622 623 624 625
                            /* Probe SimpleBlock header for the keyframe status */
                            if (ebml->cluster_starts_with_keyframe == EBML_KEYFRAME_UNKNOWN) {
                                track_number_length = ebml_parse_var_int(ebml->input_buffer + cursor + tag_length,
                                                                  end_of_buffer, &track_number);
626

627 628 629 630 631 632 633
                                if (track_number_length == 0) {
                                    /* Wait for more data */
                                    processing = 0;
                                } else if (track_number_length < 0) {
                                    return -1;
                                } else if (track_number == ebml->keyframe_track_number) {
                                    /* this block belongs to the video track */
634

635 636 637 638 639 640
                                    /* skip the 16-bit timecode for now, read the flags byte */
                                    if (cursor + tag_length + track_number_length + 2 >= ebml->input_position) {
                                        /* Wait for more data */
                                        processing = 0;
                                    } else {
                                        flags = ebml->input_buffer[cursor + tag_length + track_number_length + 2];
641

642 643 644 645 646 647 648 649 650
                                        if (flags & 0x80) {
                                            /* "keyframe" flag is set */
                                            ebml->cluster_starts_with_keyframe = EBML_KEYFRAME_STARTS_CLUSTER;
                                            /* ICECAST_LOG_DEBUG("Found keyframe in track %hhu", track_number); */
                                        } else {
                                            ebml->cluster_starts_with_keyframe = EBML_KEYFRAME_DOES_NOT_START_CLUSTER;
                                            /* ICECAST_LOG_DEBUG("Found non-keyframe in track %hhu", track_number); */
                                        }
                                    }
651

652
                                }
653

654
                            }
655

656
                        } else if (!memcmp(ebml->input_buffer + cursor, TRACK_ENTRY_MAGIC, COMMON_MAGIC_LEN)) {
657 658 659 660
                            /* Parse all TrackEntry children; reset the state */
                            payload_length = 0;
                            ebml->parsing_track_number = EBML_UNKNOWN;
                            ebml->parsing_track_is_video = 0;
661

662
                        } else if (!memcmp(ebml->input_buffer + cursor, TRACK_NUMBER_MAGIC, COMMON_MAGIC_LEN)) {
663 664 665
                            /* Probe TrackNumber for value */
                            value_length = ebml_parse_sized_int(ebml->input_buffer + cursor + tag_length,
                                                                end_of_buffer, payload_length, 0, &data_value);
666

667 668 669 670 671 672 673 674 675
                            if (value_length == 0) {
                                /* Wait for more data */
                                processing = 0;
                            } else if (value_length < 0) {
                                return -1;
                            } else {
                                ebml->parsing_track_number = data_value;
                                ebml_check_track(ebml);
                            }
676

677
                        } else if (!memcmp(ebml->input_buffer + cursor, TRACK_TYPE_MAGIC, COMMON_MAGIC_LEN)) {
678 679 680
                            /* Probe TrackType for a video flag */
                            value_length = ebml_parse_sized_int(ebml->input_buffer + cursor + tag_length,
                                                                end_of_buffer, payload_length, 0, &data_value);
681

682 683 684 685 686 687 688 689 690 691 692 693
                            if (value_length == 0) {
                                /* Wait for more data */
                                processing = 0;
                            } else if (value_length < 0) {
                                return -1;
                            } else {
                                if (data_value & 0x01) {
                                    /* This is a video track (0x01 flag = video) */
                                    ebml->parsing_track_is_video = 1;
                                    ebml_check_track(ebml);
                                }
                            }
694

695 696
                        }
                    }
697

698 699
                    /* Copy any data we don't need to probe any more */
                    if (processing) {
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
                        /* Non-cluster tag, copy it & children into buffer */
                        ebml->copy_len = tag_length + payload_length;
                        ebml->parse_state = copy_state;
                    }

                } else if (tag_length == 0) {
                    /* Wait for more data */
                    /* ICECAST_LOG_DEBUG("Wait"); */
                    processing = 0;
                } else if (tag_length < 0) {
                    /* Parse error */
                    /* ICECAST_LOG_DEBUG("Stop"); */
                    return -1;
                }
                break;
715

716 717 718 719 720 721 722
            case EBML_STATE_START_CLUSTER:
                /* found a cluster; wait to process it until
                 * any previous cluster tag has been flushed
                 * from the read buffer, so as to not lose the
                 * sync point.
                 */
                if (ebml->cluster_start >= 0) {
723 724
                    /* Allow the cluster in the read buffer to flush. */
                    ebml->flush_cluster = 1;
725 726
                    processing = 0;
                } else {
727

728 729
                    tag_length = ebml_parse_tag(ebml->input_buffer + cursor,
                                                end_of_buffer, &payload_length);
730

731 732
                    /* The header has been fully read by now, publish its size. */
                    ebml->header_size = ebml->header_position;
733

734
                    /* Mark this potential sync point, prepare probe */
735
                    ebml->cluster_start = ebml->position;
736
                    ebml->cluster_starts_with_keyframe = EBML_KEYFRAME_UNKNOWN;
737

738 739
                    /* Buffer data to give us time to probe for keyframes, etc. */
                    ebml->flush_cluster = 0;
740

741 742 743 744 745
                    /* Copy cluster tag to read buffer */
                    ebml->copy_len = tag_length;
                    ebml->parse_state = EBML_STATE_COPYING_TO_DATA;
                }
                break;
746

747 748 749 750 751 752
            case EBML_STATE_COPYING_TO_HEADER:
            case EBML_STATE_COPYING_TO_DATA:
                to_copy = ebml->input_position - cursor;
                if (to_copy > ebml->copy_len) {
                    to_copy = ebml->copy_len;
                }
753

754 755 756 757 758
                if (ebml->parse_state == EBML_STATE_COPYING_TO_HEADER) {
                    if ((ebml->header_position + to_copy) > EBML_HEADER_MAX_SIZE) {
                        ICECAST_LOG_ERROR("EBML Header too large, failing");
                        return -1;
                    }
759

760 761
                    memcpy(ebml->header + ebml->header_position, ebml->input_buffer + cursor, to_copy);
                    ebml->header_position += to_copy;
762

763 764 765 766
                } else if (ebml->parse_state == EBML_STATE_COPYING_TO_DATA) {
                    if ((ebml->position + to_copy) > EBML_SLICE_SIZE) {
                        to_copy = EBML_SLICE_SIZE - ebml->position;
                    }
767

768 769 770 771
                    memcpy(ebml->buffer + ebml->position, ebml->input_buffer + cursor, to_copy);
                    ebml->position += to_copy;
                }
                /* ICECAST_LOG_DEBUG("Copied %i of %hhu", to_copy, ebml->copy_len); */
772

773 774
                cursor += to_copy;
                ebml->copy_len -= to_copy;
775

776 777 778 779 780 781 782 783 784 785 786
                if (ebml->copy_len == 0) {
                    /* resume parsing */
                    if (ebml->parse_state == EBML_STATE_COPYING_TO_HEADER) {
                        ebml->parse_state = EBML_STATE_PARSING_HEADER;
                    } else {
                        ebml->parse_state = EBML_STATE_PARSING_CLUSTERS;
                    }
                } else {
                    /* wait for more data */
                    processing = 0;
                }
787

788
                break;
789

790 791
            default:
                processing = 0;
792

793
        }
794

795
    }
796

797 798 799
    /* Shift unprocessed data down to the start of the buffer */
    memmove(ebml->input_buffer, ebml->input_buffer + cursor, ebml->input_position - cursor);
    ebml->input_position -= cursor;
800

801
    return len;
802

803
}
804

805 806 807 808 809
static inline void ebml_check_track(ebml_t *ebml)
{
    if (ebml->keyframe_track_number == EBML_UNKNOWN
        && ebml->parsing_track_is_video
        && ebml->parsing_track_number != EBML_UNKNOWN) {
810

811 812 813 814 815
        ebml->keyframe_track_number = ebml->parsing_track_number;
        ICECAST_LOG_DEBUG("Identified track #%ffu as the video track", ebml->keyframe_track_number);
    }
}

816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
/* Try to parse an EBML tag at the given location, returning the
 * length of the tag & the length of the associated payload.
 * 
 * Returns the length of the tag on success, and writes the payload
 * size to *payload_length.
 * 
 * Return 0 if it would be necessary to read past the
 * given end-of-buffer address to read a complete tag.
 * 
 * Returns -1 if the tag is corrupt.
 */

static int ebml_parse_tag(unsigned char *buffer,
                          unsigned char *buffer_end,
                          unsigned long long *payload_length)
{
    int type_length;
    int size_length;
    unsigned long long value;
835

836
    *payload_length = 0;
837

838 839
    /* read past the type tag */
    type_length = ebml_parse_var_int(buffer, buffer_end, &value);
840

841 842 843
    if (type_length <= 0) {
        return type_length;
    }
844

845 846
    /* read the length tag */
    size_length = ebml_parse_var_int(buffer + type_length, buffer_end, payload_length);
847

848 849 850
    if (size_length <= 0) {
        return size_length;
    }
851

852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
    return type_length + size_length;
}

/* Try to parse an EBML variable-length integer.
 * Returns 0 if there's not enough space to read the number;
 * Returns -1 if the number is malformed.
 * Else, returns the length of the number in bytes and writes the
 * value to *out_value.
 */
static int ebml_parse_var_int(unsigned char *buffer,
                              unsigned char *buffer_end,
                              unsigned long long *out_value)
{
    int size = 1;
    int i;
    unsigned char mask = 0x80;
    unsigned long long value;
    unsigned long long unknown_marker;
870

871 872 873
    if (buffer >= buffer_end) {
        return 0;
    }
874

875 876
    /* find the length marker bit in the first byte */
    value = buffer[0];
877

878 879 880 881 882 883 884 885 886
    while (mask) {
        if (value & mask) {
            value = value & ~mask;
            unknown_marker = mask - 1;
            break;
        }
        size++;
        mask = mask >> 1;
    }
887

888 889 890 891 892
    /* catch malformed number (no prefix) */
    if (mask == 0) {
        ICECAST_LOG_DEBUG("Corrupt var-int");
        return -1;
    }
893

894 895 896 897
    /* catch number bigger than parsing buffer */
    if (buffer + size - 1 >= buffer_end) {
        return 0;
    }
898

899 900 901 902 903
    /* read remaining bytes of (big-endian) number */
    for (i = 1; i < size; i++) {
        value = (value << 8) + buffer[i];
        unknown_marker = (unknown_marker << 8) + 0xFF;
    }
904

905
    /* catch special "unknown" length */
906

907 908 909 910 911 912 913 914 915
    if (value == unknown_marker) {
        *out_value = EBML_UNKNOWN;
    } else {
        *out_value = value;
    }

/*
    ICECAST_LOG_DEBUG("Varint: value %lli, unknown %llu, mask %hhu, size %i", value, unknown_marker, mask, size);
*/
916

917 918
    return size;
}
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933

/* Parse a normal int that may be from 1-8 bytes long.
 * Returns 0 if there's not enough space to read the number;
 * Returns -1 if the number is mis-sized.
 * Else, returns the length of the number in bytes and writes the
 * value to *out_value.
 */
static int ebml_parse_sized_int(unsigned char       *buffer,
                                unsigned char       *buffer_end,
                                int                 len,
                                int                 is_signed,
                                unsigned long long  *out_value)
{
    long long value;
    int i;
934

935 936 937 938
    if (len < 1 || len > 8) {
        ICECAST_LOG_DEBUG("Sized int of %i bytes", len);
        return -1;
    }
939

940 941 942
    if (buffer + len >= buffer_end) {
        return 0;
    }
943

944 945 946 947 948
    if (is_signed && ((signed char) buffer[0]) < 0) {
        value = -1;
    } else {
        value = 0;
    }
949

950 951 952
    for (i = 0; i < len; i++) {
        value = (value << 8) + ((unsigned char) buffer[i]);
    }
953

954
    *out_value = value;
955

956 957
    return len;
}