videolan
/
ffmpeg
mirror of https://git.videolan.org/git/ffmpeg.git
1
Fork 0
Code Releases Activity

vmd: refactor the inner decode loop 

Simplify a little, assume empty frames are acceptable and
do not pointlessly reinit the bytestream2 contexts using
possibly wrong size values.

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
This commit is contained in:
Luca Barbato 2013-05-28 22:09:59 +02:00
parent c8f3cb9119
commit 676da248ca
1 changed files with 84 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
libavcodec/vmdav.c
View File

@ -268,91 +268,95 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
}
s->size -= PALETTE_COUNT * 3 + 2;
}
if (s->size > 0) {
/* originally UnpackFrame in VAG's code */
bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer);
if (bytestream2_get_bytes_left(&gb) < 1)
return AVERROR_INVALIDDATA;
meth = bytestream2_get_byteu(&gb);
if (meth & 0x80) {
lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb),
s->unpack_buffer, s->unpack_buffer_size);
meth &= 0x7F;
bytestream2_init(&gb, s->unpack_buffer, s->unpack_buffer_size);
}
dp = &frame->data[0][frame_y * frame->linesize[0] + frame_x];
pp = &s->prev_frame.data[0][frame_y * s->prev_frame.linesize[0] + frame_x];
switch (meth) {
case 1:
for (i = 0; i < frame_height; i++) {
ofs = 0;
do {
len = bytestream2_get_byte(&gb);
if (len & 0x80) {
len = (len & 0x7F) + 1;
if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len)
return AVERROR_INVALIDDATA;
if (!s->size)
return 0;
/* originally UnpackFrame in VAG's code */
if (bytestream2_get_bytes_left(&gb) < 1)
return AVERROR_INVALIDDATA;
meth = bytestream2_get_byteu(&gb);
if (meth & 0x80) {
lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb),
s->unpack_buffer, s->unpack_buffer_size);
meth &= 0x7F;
bytestream2_init(&gb, s->unpack_buffer, s->unpack_buffer_size);
}
dp = &frame->data[0][frame_y * frame->linesize[0] + frame_x];
pp = &s->prev_frame.data[0][frame_y * s->prev_frame.linesize[0] + frame_x];
switch (meth) {
case 1:
for (i = 0; i < frame_height; i++) {
ofs = 0;
do {
len = bytestream2_get_byte(&gb);
if (len & 0x80) {
len = (len & 0x7F) + 1;
if (ofs + len > frame_width ||
bytestream2_get_bytes_left(&gb) < len)
return AVERROR_INVALIDDATA;
bytestream2_get_buffer(&gb, &dp[ofs], len);
ofs += len;
} else {
/* interframe pixel copy */
if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
return AVERROR_INVALIDDATA;
memcpy(&dp[ofs], &pp[ofs], len + 1);
ofs += len + 1;
}
} while (ofs < frame_width);
if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR,
"VMD video: offset > width (%d > %d)\n",
ofs, frame_width);
return AVERROR_INVALIDDATA;
}
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
}
break;
case 2:
for (i = 0; i < frame_height; i++) {
bytestream2_get_buffer(&gb, dp, frame_width);
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
}
break;
case 3:
for (i = 0; i < frame_height; i++) {
ofs = 0;
do {
len = bytestream2_get_byte(&gb);
if (len & 0x80) {
len = (len & 0x7F) + 1;
if (bytestream2_get_byte(&gb) == 0xFF)
len = rle_unpack(gb.buffer, &dp[ofs],
len, bytestream2_get_bytes_left(&gb),
frame_width - ofs);
else
bytestream2_get_buffer(&gb, &dp[ofs], len);
ofs += len;
} else {
/* interframe pixel copy */
if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
return AVERROR_INVALIDDATA;
memcpy(&dp[ofs], &pp[ofs], len + 1);
ofs += len + 1;
}
} while (ofs < frame_width);
if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n",
ofs, frame_width);
return AVERROR_INVALIDDATA;
bytestream2_skip(&gb, len);
} else {
/* interframe pixel copy */
if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
return AVERROR_INVALIDDATA;
memcpy(&dp[ofs], &pp[ofs], len + 1);
ofs += len + 1;
}
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
} while (ofs < frame_width);
if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR,
"VMD video: offset > width (%d > %d)\n",
ofs, frame_width);
return AVERROR_INVALIDDATA;
}
break;
case 2:
for (i = 0; i < frame_height; i++) {
bytestream2_get_buffer(&gb, dp, frame_width);
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
}
break;
case 3:
for (i = 0; i < frame_height; i++) {
ofs = 0;
do {
len = bytestream2_get_byte(&gb);
if (len & 0x80) {
len = (len & 0x7F) + 1;
if (bytestream2_get_byte(&gb) == 0xFF)
len = rle_unpack(gb.buffer, &dp[ofs],
len, bytestream2_get_bytes_left(&gb),
frame_width - ofs);
else
bytestream2_get_buffer(&gb, &dp[ofs], len);
bytestream2_skip(&gb, len);
} else {
/* interframe pixel copy */
if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
return AVERROR_INVALIDDATA;
memcpy(&dp[ofs], &pp[ofs], len + 1);
ofs += len + 1;
}
} while (ofs < frame_width);
if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n",
ofs, frame_width);
return AVERROR_INVALIDDATA;
}
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
}
break;
dp += frame->linesize[0];
pp += s->prev_frame.linesize[0];
}
break;
}
return 0;
}