From 3de13d5212b6a0ff146dc0b50e34ee7dbc822eef Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Apr 2015 15:48:37 -0400 Subject: [PATCH] vp9: remove another optimization branch in iadst16 which causes overflows. See sample vp90-2-14-resize-fp-tiles-16-8.webm from the vp9 test vector set to reproduce the issue. --- libavcodec/x86/vp9itxfm.asm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavcodec/x86/vp9itxfm.asm b/libavcodec/x86/vp9itxfm.asm index a9d45a4b46..9cf0d78fab 100644 --- a/libavcodec/x86/vp9itxfm.asm +++ b/libavcodec/x86/vp9itxfm.asm @@ -1622,7 +1622,9 @@ VP9_IDCT_IDCT_16x16_ADD_XMM avx PSIGNW m3, [pw_m1] ; m3=out1[w], m7=t10[w] SUMSUB_BA w, 2, 6, 1 ; m2=out14[w], m6=t11[w] -%if cpuflag(ssse3) + ; unfortunately, the code below overflows in some cases, e.g. + ; http://downloads.webmproject.org/test_data/libvpx/vp90-2-14-resize-fp-tiles-16-8.webm +%if 0; cpuflag(ssse3) SUMSUB_BA w, 7, 6, 1 pmulhrsw m7, [pw_11585x2] ; m7=out6[w] pmulhrsw m6, [pw_11585x2] ; m6=out9[w]