videolan
/
ffmpeg
mirror of https://git.videolan.org/git/ffmpeg.git
1
Fork 0
Code Releases Activity

qtrle: Use bytestream2 functions to prevent buffer overreads. 

Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>
This commit is contained in:
Aneesh Dogra 2012-03-16 00:43:23 +05:30 committed by Ronald S. Bultje
parent 341404f753
commit de64d8cf17
1 changed files with 74 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
libavcodec/qtrle.c
View File

@ -35,27 +35,17 @@
#include <stdlib.h>
#include <string.h>
#include "libavutil/intreadwrite.h"
#include "avcodec.h"
#include "bytestream.h"
typedef struct QtrleContext {
AVCodecContext *avctx;
AVFrame frame;
const unsigned char *buf;
int size;
GetByteContext g;
uint32_t pal[256];
} QtrleContext;
#define CHECK_STREAM_PTR(n) \
if ((stream_ptr + n) > s->size) { \
av_log (s->avctx, AV_LOG_INFO, "Problem: stream_ptr out of bounds (%d >= %d)\n", \
stream_ptr + n, s->size); \
return; \
}
#define CHECK_PIXEL_PTR(n) \
if ((pixel_ptr + n > pixel_limit) || (pixel_ptr + n < 0)) { \
av_log (s->avctx, AV_LOG_INFO, "Problem: pixel_ptr = %d, pixel_limit = %d\n", \
@ -63,7 +53,7 @@ typedef struct QtrleContext {
return; \
} \
static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_1bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr = 0;
@ -74,9 +64,8 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int skip;
while (lines_to_change) {
CHECK_STREAM_PTR(2);
skip = s->buf[stream_ptr++];
rle_code = (signed char)s->buf[stream_ptr++];
skip = bytestream2_get_byte(&s->g);
rle_code = (signed char)bytestream2_get_byte(&s->g);
if (rle_code == 0)
break;
if(skip & 0x80) {
@ -92,9 +81,9 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
rle_code = -rle_code;
/* get the next 2 bytes from the stream, treat them as groups
* of 8 pixels, and output them rle_code times */
CHECK_STREAM_PTR(2);
pi0 = s->buf[stream_ptr++];
pi1 = s->buf[stream_ptr++];
pi0 = bytestream2_get_byte(&s->g);
pi1 = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 2);
while (rle_code--) {
@ -104,17 +93,16 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
} else {
/* copy the same pixel directly to output 2 times */
rle_code *= 2;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code);
while (rle_code--)
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
int row_ptr, int lines_to_change, int bpp)
static inline void qtrle_decode_2n4bpp(QtrleContext *s, int row_ptr,
int lines_to_change, int bpp)
{
int rle_code, i;
int pixel_ptr;
@ -125,24 +113,21 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
int num_pixels = (bpp == 4) ? 8 : 16;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (num_pixels * (s->buf[stream_ptr++] - 1));
pixel_ptr = row_ptr + (num_pixels * (bytestream2_get_byte(&s->g) - 1));
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (num_pixels * (s->buf[stream_ptr++] - 1));
pixel_ptr += (num_pixels * (bytestream2_get_byte(&s->g) - 1));
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
/* get the next 4 bytes from the stream, treat them as palette
* indexes, and output them rle_code times */
CHECK_STREAM_PTR(4);
for (i = num_pixels-1; i >= 0; i--) {
pi[num_pixels-1-i] = (s->buf[stream_ptr] >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
stream_ptr+= ((i & ((num_pixels>>2)-1)) == 0);
pi[num_pixels-1-i] = (bytestream2_peek_byte(&s->g) >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
bytestream2_skip(&s->g, ((i & ((num_pixels>>2)-1)) == 0));
}
CHECK_PIXEL_PTR(rle_code * num_pixels);
while (rle_code--) {
@ -152,17 +137,18 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
} else {
/* copy the same pixel directly to output 4 times */
rle_code *= 4;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code*(num_pixels>>2));
while (rle_code--) {
if(bpp == 4) {
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x0f;
rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x0f;
int x = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = (x >> 4) & 0x0f;
rgb[pixel_ptr++] = x & 0x0f;
} else {
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 6) & 0x03;
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x03;
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 2) & 0x03;
rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x03;
int x = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = (x >> 6) & 0x03;
rgb[pixel_ptr++] = (x >> 4) & 0x03;
rgb[pixel_ptr++] = (x >> 2) & 0x03;
rgb[pixel_ptr++] = x & 0x03;
}
}
}
@ -171,7 +157,7 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
}
}
static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_8bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -181,25 +167,22 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (4 * (s->buf[stream_ptr++] - 1));
pixel_ptr = row_ptr + (4 * (bytestream2_get_byte(&s->g) - 1));
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (4 * (s->buf[stream_ptr++] - 1));
pixel_ptr += (4 * (bytestream2_get_byte(&s->g) - 1));
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
/* get the next 4 bytes from the stream, treat them as palette
* indexes, and output them rle_code times */
CHECK_STREAM_PTR(4);
pi1 = s->buf[stream_ptr++];
pi2 = s->buf[stream_ptr++];
pi3 = s->buf[stream_ptr++];
pi4 = s->buf[stream_ptr++];
pi1 = bytestream2_get_byte(&s->g);
pi2 = bytestream2_get_byte(&s->g);
pi3 = bytestream2_get_byte(&s->g);
pi4 = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 4);
@ -212,11 +195,10 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
} else {
/* copy the same pixel directly to output 4 times */
rle_code *= 4;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code);
while (rle_code--) {
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
@ -224,7 +206,7 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_16bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -234,21 +216,17 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 2;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 2;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 2;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 2;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(2);
rgb16 = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
rgb16 = bytestream2_get_be16(&s->g);
CHECK_PIXEL_PTR(rle_code * 2);
@ -257,13 +235,11 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
pixel_ptr += 2;
}
} else {
CHECK_STREAM_PTR(rle_code * 2);
CHECK_PIXEL_PTR(rle_code * 2);
/* copy pixels directly to output */
while (rle_code--) {
rgb16 = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
rgb16 = bytestream2_get_be16(&s->g);
*(unsigned short *)(&rgb[pixel_ptr]) = rgb16;
pixel_ptr += 2;
}
@ -273,7 +249,7 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_24bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -283,22 +259,19 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 3;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 3;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 3;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 3;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(3);
r = s->buf[stream_ptr++];
g = s->buf[stream_ptr++];
b = s->buf[stream_ptr++];
r = bytestream2_get_byte(&s->g);
g = bytestream2_get_byte(&s->g);
b = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 3);
@ -308,14 +281,13 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
rgb[pixel_ptr++] = b;
}
} else {
CHECK_STREAM_PTR(rle_code * 3);
CHECK_PIXEL_PTR(rle_code * 3);
/* copy pixels directly to output */
while (rle_code--) {
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
@ -323,7 +295,7 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_32bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -333,21 +305,17 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 4;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 4;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 4;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 4;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(4);
argb = AV_RB32(s->buf + stream_ptr);
stream_ptr += 4;
argb = bytestream2_get_be32(&s->g);
CHECK_PIXEL_PTR(rle_code * 4);
@ -356,14 +324,12 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
pixel_ptr += 4;
}
} else {
CHECK_STREAM_PTR(rle_code * 4);
CHECK_PIXEL_PTR(rle_code * 4);
/* copy pixels directly to output */
while (rle_code--) {
argb = AV_RB32(s->buf + stream_ptr);
argb = bytestream2_get_be32(&s->g);
AV_WN32A(rgb + pixel_ptr, argb);
stream_ptr += 4;
pixel_ptr += 4;
}
}
@ -419,16 +385,12 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
void *data, int *data_size,
AVPacket *avpkt)
{
const uint8_t *buf = avpkt->data;
int buf_size = avpkt->size;
QtrleContext *s = avctx->priv_data;
int header, start_line;
int stream_ptr, height, row_ptr;
int height, row_ptr;
int has_palette = 0;
s->buf = buf;
s->size = buf_size;
bytestream2_init(&s->g, avpkt->data, avpkt->size);
s->frame.reference = 1;
s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
FF_BUFFER_HINTS_REUSABLE | FF_BUFFER_HINTS_READABLE;
@ -438,64 +400,63 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
}
/* check if this frame is even supposed to change */
if (s->size < 8)
if (avpkt->size < 8)
goto done;
/* start after the chunk size */
stream_ptr = 4;
bytestream2_seek(&s->g, 4, SEEK_SET);
/* fetch the header */
header = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
header = bytestream2_get_be16(&s->g);
/* if a header is present, fetch additional decoding parameters */
if (header & 0x0008) {
if(s->size < 14)
if (avpkt->size < 14)
goto done;
start_line = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 4;
height = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 4;
start_line = bytestream2_get_be16(&s->g);
bytestream2_skip(&s->g, 2);
height = bytestream2_get_be16(&s->g);
bytestream2_skip(&s->g, 2);
} else {
start_line = 0;
height = s->avctx->height;
height = s->avctx->height;
}
row_ptr = s->frame.linesize[0] * start_line;
switch (avctx->bits_per_coded_sample) {
case 1:
case 33:
qtrle_decode_1bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_1bpp(s, row_ptr, height);
break;
case 2:
case 34:
qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 2);
qtrle_decode_2n4bpp(s, row_ptr, height, 2);
has_palette = 1;
break;
case 4:
case 36:
qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 4);
qtrle_decode_2n4bpp(s, row_ptr, height, 4);
has_palette = 1;
break;
case 8:
case 40:
qtrle_decode_8bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_8bpp(s, row_ptr, height);
has_palette = 1;
break;
case 16:
qtrle_decode_16bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_16bpp(s, row_ptr, height);
break;
case 24:
qtrle_decode_24bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_24bpp(s, row_ptr, height);
break;
case 32:
qtrle_decode_32bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_32bpp(s, row_ptr, height);
break;
default:
@ -521,7 +482,7 @@ done:
*(AVFrame*)data = s->frame;
/* always report that the buffer was completely consumed */
return buf_size;
return avpkt->size;
}
static av_cold int qtrle_decode_end(AVCodecContext *avctx)