diff --git a/libavcodec/hevc.c b/libavcodec/hevc.c index eba1ac041e..5c78c44844 100644 --- a/libavcodec/hevc.c +++ b/libavcodec/hevc.c @@ -797,10 +797,10 @@ static void hls_sao_param(HEVCContext *s, int rx, int ry) #undef SET_SAO #undef CTB -static void hls_transform_unit(HEVCContext *s, int x0, int y0, - int xBase, int yBase, int cb_xBase, int cb_yBase, - int log2_cb_size, int log2_trafo_size, - int trafo_depth, int blk_idx) +static int hls_transform_unit(HEVCContext *s, int x0, int y0, + int xBase, int yBase, int cb_xBase, int cb_yBase, + int log2_cb_size, int log2_trafo_size, + int trafo_depth, int blk_idx) { HEVCLocalContext *lc = s->HEVClc; @@ -835,6 +835,18 @@ static void hls_transform_unit(HEVCContext *s, int x0, int y0, if (ff_hevc_cu_qp_delta_sign_flag(s) == 1) lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta; lc->tu.is_cu_qp_delta_coded = 1; + + if (lc->tu.cu_qp_delta < -(26 + s->sps->qp_bd_offset / 2) || + lc->tu.cu_qp_delta > (25 + s->sps->qp_bd_offset / 2)) { + av_log(s->avctx, AV_LOG_ERROR, + "The cu_qp_delta %d is outside the valid range " + "[%d, %d].\n", + lc->tu.cu_qp_delta, + -(26 + s->sps->qp_bd_offset / 2), + (25 + s->sps->qp_bd_offset / 2)); + return AVERROR_INVALIDDATA; + } + ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size); } @@ -870,6 +882,7 @@ static void hls_transform_unit(HEVCContext *s, int x0, int y0, ff_hevc_hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2); } } + return 0; } static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size) @@ -887,13 +900,14 @@ static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_si s->is_pcm[i + j * min_pu_width] = 2; } -static void hls_transform_tree(HEVCContext *s, int x0, int y0, - int xBase, int yBase, int cb_xBase, int cb_yBase, - int log2_cb_size, int log2_trafo_size, - int trafo_depth, int blk_idx) +static int hls_transform_tree(HEVCContext *s, int x0, int y0, + int xBase, int yBase, int cb_xBase, int cb_yBase, + int log2_cb_size, int log2_trafo_size, + int trafo_depth, int blk_idx) { HEVCLocalContext *lc = s->HEVClc; uint8_t split_transform_flag; + int ret; if (trafo_depth > 0 && log2_trafo_size == 2) { SAMPLE_CBF(lc->tt.cbf_cb[trafo_depth], x0, y0) = @@ -948,14 +962,26 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, int x1 = x0 + ((1 << log2_trafo_size) >> 1); int y1 = y0 + ((1 << log2_trafo_size) >> 1); - hls_transform_tree(s, x0, y0, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 0); - hls_transform_tree(s, x1, y0, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 1); - hls_transform_tree(s, x0, y1, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 2); - hls_transform_tree(s, x1, y1, x0, y0, cb_xBase, cb_yBase, log2_cb_size, - log2_trafo_size - 1, trafo_depth + 1, 3); + ret = hls_transform_tree(s, x0, y0, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 0); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x1, y0, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 1); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x0, y1, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 2); + if (ret < 0) + return ret; + ret = hls_transform_tree(s, x1, y1, x0, y0, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size - 1, + trafo_depth + 1, 3); + if (ret < 0) + return ret; } else { int min_tu_size = 1 << s->sps->log2_min_tb_size; int log2_min_tu_size = s->sps->log2_min_tb_size; @@ -967,9 +993,11 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, lc->tt.cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth); } - hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase, - log2_cb_size, log2_trafo_size, trafo_depth, blk_idx); - + ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase, + log2_cb_size, log2_trafo_size, trafo_depth, + blk_idx); + if (ret < 0) + return ret; // TODO: store cbf_luma somewhere else if (lc->tt.cbf_luma) { int i, j; @@ -989,6 +1017,7 @@ static void hls_transform_tree(HEVCContext *s, int x0, int y0, set_deblocking_bypass(s, x0, y0, log2_trafo_size); } } + return 0; } static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size) @@ -1599,7 +1628,7 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) int min_cb_width = s->sps->min_cb_width; int x_cb = x0 >> log2_min_cb_size; int y_cb = y0 >> log2_min_cb_size; - int x, y; + int x, y, ret; lc->cu.x = x0; lc->cu.y = y0; @@ -1656,7 +1685,6 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) lc->cu.pcm_flag = ff_hevc_pcm_flag_decode(s); } if (lc->cu.pcm_flag) { - int ret; intra_prediction_unit_default_value(s, x0, y0, log2_cb_size); ret = hls_pcm_sample(s, x0, y0, log2_cb_size); if (s->sps->pcm.loop_filter_disable_flag) @@ -1715,8 +1743,11 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ? s->sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag : s->sps->max_transform_hierarchy_depth_inter; - hls_transform_tree(s, x0, y0, x0, y0, x0, y0, log2_cb_size, - log2_cb_size, 0, 0); + ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0, + log2_cb_size, + log2_cb_size, 0, 0); + if (ret < 0) + return ret; } else { if (!s->sh.disable_deblocking_filter_flag) ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size,