videolan
/
ffmpeg
mirror of https://git.videolan.org/git/ffmpeg.git
1
Fork 0
Code Releases Activity

avcodec/diracdec: Fix qfactor/offset tables 

It seems the previous tables where calculated with 32bit integers ignoring
overflows.
Also check for the max qindex, the value is choosen so that the qfactor/offset
fit in int32.

Fixes: 070b7914fd5dfe8f93248bea71363410/asan_static-oob_c8d034_2764_258e20f4a3c79158aecddb61a833d756.drc
Fixes out of array reads

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2016-01-13 23:51:47 +01:00
parent 69697be922
commit 5fbd97fc75
1 changed files with 63 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
libavcodec/diracdec.c
View File

@ -253,53 +253,61 @@ static const uint8_t default_qmat[][4][4] = {
{ { 3, 1, 1, 0}, { 0, 4, 4, 2}, { 0, 6, 6, 5}, { 0, 9, 9, 7} },
};
static const int32_t qscale_tab[128] = {
4, 5, 6, 7, 8, 10, 11, 13,
16, 19, 23, 27, 32, 38, 45, 54,
64, 76, 91, 108, 128, 152, 181, 215,
256, 304, 362, 431, 512, 609, 724, 861,
1024, 1218, 1448, 1722, 2048, 2435, 2896, 3444,
4096, 4871, 5793, 6889, 8192, 9742, 11585, 13777,
16384, 19484, -13317, 27554, 32768, -1581, 9853, -10518,
65536, -3164, -16782, -21037, 131072, -6328, 2922, 23552,
262144, -12658, 5844, -18524, 524288, 15232, 11689, 28578,
1048576, -10085, -13110, -8471, 2097152, -20170, 10267, -16943,
4194304, 208, -15954, 31741, 8388608, 416, 4579, -2146,
16777216, 832, 9158, -4293, 33554432, 1663, -18172, -8587,
67108864, 3326, 143, -17175, 134217728, 6653, 285, 31276,
268435456, 13306, 570, -3075, 536870912, -13938, 1140, -6152,
1073741824, 12672, 2281, -12304, -2147483648, -15205, 4561, -24610,
0, 10138, 9122, 16407, 0, -20274, -18243, -32813,
static const int32_t qscale_tab[116] = {
4, 5, 6, 7, 8, 10, 11, 13,
16, 19, 23, 27, 32, 38, 45, 54,
64, 76, 91, 108, 128, 152, 181, 215,
256, 304, 362, 431, 512, 609, 724, 861,
1024, 1218, 1448, 1722, 2048, 2435, 2896, 3444,
4096, 4871, 5793, 6889, 8192, 9742, 11585, 13777,
16384, 19484, 23170, 27554, 32768, 38968, 46341, 55109,
65536, 77936, 92682, 110218, 131072, 155872, 185364, 220436,
262144, 311744, 370728, 440872, 524288, 623487, 741455, 881744,
1048576, 1246974, 1482910, 1763488, 2097152, 2493948, 2965821, 3526975,
4194304, 4987896, 5931642, 7053950, 8388608, 9975792, 11863283, 14107901,
16777216, 19951585, 23726566, 28215802, 33554432, 39903169, 47453133, 56431603,
67108864, 79806339, 94906266, 112863206, 134217728, 159612677, 189812531, 225726413,
268435456, 319225354, 379625062, 451452825, 536870912, 638450708, 759250125, 902905651,
1073741824,1276901417,1518500250,1805811301,/*2147483648,2553802834,3037000500,3611622603,
4294967296*/
};
static const int32_t qoffset_intra_tab[128] = {
static const int32_t qoffset_intra_tab[120] = {
1, 2, 3, 4, 4, 5, 6, 7,
8, 10, 12, 14, 16, 19, 23, 27,
32, 38, 46, 54, 64, 76, 91, 108,
128, 152, 181, 216, 256, 305, 362, 431,
512, 609, 724, 861, 1024, 1218, 1448, 1722,
2048, 2436, 2897, 3445, 4096, 4871, 5793, 6889,
8192, 9742, -6658, 13777, 16384, -790, 4927, -5258,
32768, -1581, -8390, -10518, 65536, -3163, 1461, 11776,
131072, -6328, 2922, -9261, 262144, 7616, 5845, 14289,
524288, -5042, -6554, -4235, 1048576, -10084, 5134, -8471,
2097152, 104, -7976, 15871, 4194304, 208, 2290, -1072,
8388608, 416, 4579, -2146, 16777216, 832, -9085, -4293,
33554432, 1663, 72, -8587, 67108864, 3327, 143, 15638,
134217728, 6653, 285, -1537, 268435456, -6968, 570, -3075,
536870912, 6336, 1141, -6151, -1073741823, -7602, 2281, -12304,
0, 5069, 4561, 8204, 0, -10136, -9121, -16406,
8, 10, 12, 14, 16, 19, 23, 27,
32, 38, 46, 54, 64, 76, 91, 108,
128, 152, 181, 216, 256, 305, 362, 431,
512, 609, 724, 861, 1024, 1218, 1448, 1722,
2048, 2436, 2897, 3445, 4096, 4871, 5793, 6889,
8192, 9742, 11585, 13777, 16384, 19484, 23171, 27555,
32768, 38968, 46341, 55109, 65536, 77936, 92682, 110218,
131072, 155872, 185364, 220436, 262144, 311744, 370728, 440872,
524288, 623487, 741455, 881744, 1048576, 1246974, 1482911, 1763488,
2097152, 2493948, 2965821, 3526975, 4194304, 4987896, 5931642, 7053951,
8388608, 9975793, 11863283, 14107901, 16777216, 19951585, 23726567, 28215802,
33554432, 39903170, 47453133, 56431603, 67108864, 79806339, 94906266, 112863207,
134217728, 159612677, 189812531, 225726413, 268435456, 319225354, 379625063, 451452826,
536870912, 638450709, 759250125, 902905651,1073741824,1276901417,1518500250,1805811302,
/*2147483648, 2553802834, 3037000500, 3611622603, 4294967296,*/
};
static const int qoffset_inter_tab[MAX_QUANT+1] = {
1, 2, 2, 3, 3, 4, 4, 5,
6, 7, 9, 10, 12, 14, 17, 20,
24, 29, 34, 41, 48, 57, 68, 81,
96, 114, 136, 162, 192, 228, 272, 323,
384, 457, 543, 646, 768, 913, 1086, 1292,
1536, 1827, 2172, 2583, 3072, 3653, 4344, 5166,
6144, 7307, 8689, 10333, 12288, 14613, 17378, 20666,
24576, 29226
static const int qoffset_inter_tab[122] = {
1, 2, 2, 3, 3, 4, 4, 5,
6, 7, 9, 10, 12, 14, 17, 20,
24, 29, 34, 41, 48, 57, 68, 81,
96, 114, 136, 162, 192, 228, 272, 323,
384, 457, 543, 646, 768, 913, 1086, 1292,
1536, 1827, 2172, 2583, 3072, 3653, 4344, 5166,
6144, 7307, 8689, 10333, 12288, 14613, 17378, 20666,
24576, 29226, 34756, 41332, 49152, 58452, 69512, 82664,
98304, 116904, 139023, 165327, 196608, 233808, 278046, 330654,
393216, 467615, 556091, 661308, 786432, 935231, 1112183, 1322616,
1572864, 1870461, 2224366, 2645231, 3145728, 3740922, 4448731, 5290463,
6291456, 7481844, 8897462, 10580926, 12582912, 14963688, 17794925, 21161851,
25165824, 29927377, 35589850, 42323702, 50331648, 59854754, 71179699, 84647405,
100663296, 119709508, 142359398, 169294809, 201326592, 239419016, 284718797, 338589619,
402653184, 478838031, 569437594, 677179238, 805306368, 957676063,1138875188,1354358476,
1610612736, 1915352125, /*2277750375, 2708716952, 3221225472, 3830704250,*/
};
/* magic number division by 3 from schroedinger */
@ -601,7 +609,11 @@ static inline void codeblock(DiracContext *s, SubBand *b,
b->quant = quant;
}
b->quant = FFMIN(b->quant, MAX_QUANT);
if (b->quant > 115) {
av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", b->quant);
b->quant = 0;
return;
}
qfactor = qscale_tab[b->quant];
/* TODO: context pointer? */
@ -781,12 +793,18 @@ static void decode_subband(DiracContext *s, GetBitContext *gb, int quant,
int top = b1->height * slice_y / s->num_y;
int bottom = b1->height *(slice_y+1) / s->num_y;
int qfactor = qscale_tab[quant & 0x7f];
int qoffset = qoffset_intra_tab[quant & 0x7f] + 2;
int qfactor, qoffset;
uint8_t *buf1 = b1->ibuf + top * b1->stride;
uint8_t *buf2 = b2 ? b2->ibuf + top * b2->stride: NULL;
int x, y;
if (quant > 115) {
av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", quant);
return;
}
qfactor = qscale_tab[quant & 0x7f];
qoffset = qoffset_intra_tab[quant & 0x7f] + 2;
/* we have to constantly check for overread since the spec explicitly
requires this, with the meaning that all remaining coeffs are set to 0 */
if (get_bits_count(gb) >= bits_end)