videolan
/
ffmpeg
mirror of https://git.videolan.org/git/ffmpeg.git
1
Fork 0
Code Releases Activity

fate: Allow running multiple rounds of tests with differing settings 

This can be used to run tests multple times, with e.g. differing
QEMU settings, by adding something like this to the FATE configuration
file:

    target_exec="qemu-aarch64-static"
    fate_targets="fate-checkasm fate-cpu"

    fate_environments="sve128 sve256 sve512"
    sve128_env="QEMU_CPU=max,sve128=on"
    sve256_env="QEMU_CPU=max,sve256=on"
    sve512_env="QEMU_CPU=max,sve512=on"

It's also possible to customize the target_exec command further
by injecting a sufficiently quoted variable into it, which then can
be updated for each run, e.g. target_exec="\$(CUR_EXEC_CMD)".

For each of the environment names in fate_environments, the tests
that are run get the name suffixed on the fate tests in the
test log and fate report, e.g. "fate-checkasm-h264dsp_sve128".

Signed-off-by: Martin Storsjö <martin@martin.st>
This commit is contained in:
Martin Storsjö 2023-12-08 14:08:05 +02:00
parent d0e215fffc
commit 76cd71bf0c
3 changed files with 35 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
doc/fate_config.sh.template
View File

@ -33,3 +33,23 @@ makeopts= # extra options passed to 'make'
# defaults to 'tar c'
#fate_targets= # targets to make when running fate; defaults to "fate",
# can be set to run a subset of tests, e.g. "fate-checkasm".
#fate_environments= # a list of names of configurations to run tests for;
# each round is run with variables from ${${name}_env} set.
# One example of using fate_environments:
# target_exec="qemu-aarch64-static"
# fate_targets="fate-checkasm fate-cpu"
# fate_environments="sve128 sve256"
# sve128_env="QEMU_CPU=max,sve128=on"
# sve256_env="QEMU_CPU=max,sve256=on"
# The variables set by fate_environments can also be used explicitly
# by target_exec, e.g. like this:
# target_exec="qemu-aarch64-static -cpu \$(MY_CPU)"
# fate_targets="fate-checkasm fate-cpu"
# fate_environments="sve128 sve256"
# sve128_env="MY_CPU=max,sve128=on"
# sve256_env="MY_CPU=max,sve256=on"

4
tests/Makefile
View File

@ -305,8 +305,8 @@ $(FATE): export PROGSUF = $(PROGSSUF)
$(FATE): export EXECSUF = $(EXESUF)
$(FATE): export HOSTEXECSUF = $(HOSTEXESUF)
$(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | $(FATE_OUTDIRS)
@echo "TEST $(@:fate-%=%)"
$(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
@echo "TEST $(@:fate-%=%)$(FATE_SUFFIX)"
$(Q)$(SRC_PATH)/tests/fate-run.sh $@$(FATE_SUFFIX) "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
fate-list:
@printf '%s\n' $(sort $(FATE))

14
tests/fate.sh
View File

@ -75,7 +75,19 @@ compile()(
fate()(
test "$build_only" = "yes" && return
cd ${build} || return
${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
if [ -n "${fate_environments}" ]; then
ret=0
for e in ${fate_environments}; do
eval "curenv=\${${e}_env}"
echo Testing environment ${e}: ${curenv}
${make} ${makeopts_fate-${makeopts}} -k ${fate_targets} FATE_SUFFIX=_${e} ${curenv}
cur_ret=$?
test $cur_ret != 0 && ret=$cur_ret
done
return $ret
else
${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
fi
)
clean(){