GitLab

There are multiple invalid shift operations in the tremor codec.

To reproduce build tremor with clang and -fsanitize=undefined (undefined behavior sanitizer or ubsan). Then compile ivorbisfile_example.c that comes with it and pass the attached ogg file. Setting UBSAN_OPTIONS="print_stacktrace=1" gives a better output with stack traces.

Output from ubsan:

sharedbook.c:420:21: runtime error: left shift of 16 by 27 places cannot be represented in type 'int'
    #0 0x53fdf3 in vorbis_book_init_decode /f/tremor/Tremor/sharedbook.c:420:21
    #1 0x520f00 in _vds_init /f/tremor/Tremor/block.c:169:10
    #2 0x520f00 in vorbis_synthesis_init /f/tremor/Tremor/block.c:229
    #3 0x5173be in _make_decode_ready /f/tremor/Tremor/vorbisfile.c:595:8
    #4 0x517925 in _fetch_and_process_packet /f/tremor/Tremor/vorbisfile.c:679:15
    #5 0x51ad9d in ov_read /f/tremor/Tremor/vorbisfile.c:1934:15
    #6 0x50aa11 in main (/f/tremor/Tremor/a.out+0x50aa11)
    #7 0x7fcfccf8478f in __libc_start_main (/lib64/libc.so.6+0x2078f)
    #8 0x419cc8 in _start (/f/tremor/Tremor/a.out+0x419cc8)

SUMMARY: AddressSanitizer: undefined-behavior sharedbook.c:420:21 in 
misc.h:242:6: runtime error: left shift of negative value -134217728
    #0 0x53d303 in VFLOAT_ADD /f/tremor/Tremor/./misc.h:242:6
    #1 0x53a8a3 in _book_unquantize /f/tremor/Tremor/sharedbook.c:230:10
    #2 0x53e493 in vorbis_book_init_decode /f/tremor/Tremor/sharedbook.c:383:18
    #3 0x520f00 in _vds_init /f/tremor/Tremor/block.c:169:10
    #4 0x520f00 in vorbis_synthesis_init /f/tremor/Tremor/block.c:229
    #5 0x5173be in _make_decode_ready /f/tremor/Tremor/vorbisfile.c:595:8
    #6 0x517925 in _fetch_and_process_packet /f/tremor/Tremor/vorbisfile.c:679:15
    #7 0x51ad9d in ov_read /f/tremor/Tremor/vorbisfile.c:1934:15
    #8 0x50aa11 in main (/f/tremor/Tremor/a.out+0x50aa11)
    #9 0x7fcfccf8478f in __libc_start_main (/lib64/libc.so.6+0x2078f)
    #10 0x419cc8 in _start (/f/tremor/Tremor/a.out+0x419cc8)

SUMMARY: AddressSanitizer: undefined-behavior misc.h:242:6 in 
misc.h:77:68: runtime error: left shift of negative value -1
    #0 0x546c70 in MULT31_SHIFT15 /f/tremor/Tremor/./misc.h:77:68
    #1 0x546c70 in render_line /f/tremor/Tremor/floor1.c:323
    #2 0x546c70 in floor1_inverse2 /f/tremor/Tremor/floor1.c:442
    #3 0x558f86 in mapping0_inverse /f/tremor/Tremor/mapping0.c:285:5
    #4 0x51799a in _fetch_and_process_packet /f/tremor/Tremor/vorbisfile.c:697:15
    #5 0x51ad9d in ov_read /f/tremor/Tremor/vorbisfile.c:1934:15
    #6 0x50aa11 in main (/f/tremor/Tremor/a.out+0x50aa11)
    #7 0x7fcfccf8478f in __libc_start_main (/lib64/libc.so.6+0x2078f)
    #8 0x419cc8 in _start (/f/tremor/Tremor/a.out+0x419cc8)

SUMMARY: AddressSanitizer: undefined-behavior misc.h:77:68 in 
misc.h:71:21: runtime error: left shift of negative value -42347
    #0 0x560d2f in MULT31 /f/tremor/Tremor/./misc.h:71:21
    #1 0x560d2f in XPROD31 /f/tremor/Tremor/./misc.h:153
    #2 0x55a3a5 in mdct_backward /f/tremor/Tremor/mdct.c:362:5
    #3 0x55930d in mapping0_inverse /f/tremor/Tremor/mapping0.c:296:5
    #4 0x51799a in _fetch_and_process_packet /f/tremor/Tremor/vorbisfile.c:697:15
    #5 0x51ad9d in ov_read /f/tremor/Tremor/vorbisfile.c:1934:15
    #6 0x50aa11 in main (/f/tremor/Tremor/a.out+0x50aa11)
    #7 0x7fcfccf8478f in __libc_start_main (/lib64/libc.so.6+0x2078f)
    #8 0x419cc8 in _start (/f/tremor/Tremor/a.out+0x419cc8)

SUMMARY: AddressSanitizer: undefined-behavior misc.h:71:21 in 
misc.h:71:21: runtime error: left shift of negative value -4
    #0 0x52a0c8 in MULT31 /f/tremor/Tremor/./misc.h:71:21
    #1 0x52a0c8 in _vorbis_apply_window /f/tremor/Tremor/window.c:76
    #2 0x55965f in mapping0_inverse /f/tremor/Tremor/mapping0.c:306:7
    #3 0x51799a in _fetch_and_process_packet /f/tremor/Tremor/vorbisfile.c:697:15
    #4 0x51ad9d in ov_read /f/tremor/Tremor/vorbisfile.c:1934:15
    #5 0x50aa11 in main (/f/tremor/Tremor/a.out+0x50aa11)
    #6 0x7fcfccf8478f in __libc_start_main (/lib64/libc.so.6+0x2078f)
    #7 0x419cc8 in _start (/f/tremor/Tremor/a.out+0x419cc8)