ffmpeg | branch: master | Tobias Rapp <t.r...@noa-archive.com> | Tue Aug 29 14:45:13 2017 +0200| [bee01ee2ba2e62974447d5e8ea2afb27dbdb1e23] | committer: Tobias Rapp
fate: add tests for psnr and ssim filter Metadata filter output is passed through an Awk script comparing floats against reference values with specified "fuzz" tolerance to account for architectural differences (e.g. x86-32 vs. x86-64). Signed-off-by: Tobias Rapp <t.r...@noa-archive.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=bee01ee2ba2e62974447d5e8ea2afb27dbdb1e23 --- tests/fate-run.sh | 9 +++++ tests/fate/filter-video.mak | 14 ++++++++ tests/ref/fate/filter-refcmp-psnr-rgb | 45 ++++++++++++++++++++++++ tests/ref/fate/filter-refcmp-psnr-yuv | 45 ++++++++++++++++++++++++ tests/ref/fate/filter-refcmp-ssim-rgb | 30 ++++++++++++++++ tests/ref/fate/filter-refcmp-ssim-yuv | 30 ++++++++++++++++ tests/refcmp-metadata.awk | 64 +++++++++++++++++++++++++++++++++++ 7 files changed, 237 insertions(+) diff --git a/tests/fate-run.sh b/tests/fate-run.sh index 9aa9a22395..c5480c7ede 100755 --- a/tests/fate-run.sh +++ b/tests/fate-run.sh @@ -234,6 +234,15 @@ lavftest(){ ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples" } +refcmp_metadata(){ + refcmp=$1 + pixfmt=$2 + fuzz=${3:-0.001} + ffmpeg $FLAGS $ENC_OPTS \ + -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \ + -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk - +} + video_filter(){ filters=$1 shift diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak index d1e13414f6..78cd4711e6 100644 --- a/tests/fate/filter-video.mak +++ b/tests/fate/filter-video.mak @@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov +REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER + +FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb +fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24 + +FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv +fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p + +FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb +fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24 + +FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv +fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p + FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes) FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes) FATE_FFMPEG += $(FATE_FILTER-yes) diff --git a/tests/ref/fate/filter-refcmp-psnr-rgb b/tests/ref/fate/filter-refcmp-psnr-rgb new file mode 100644 index 0000000000..f06db575ac --- /dev/null +++ b/tests/ref/fate/filter-refcmp-psnr-rgb @@ -0,0 +1,45 @@ +frame:0 pts:0 pts_time:0 +lavfi.psnr.mse.r=1381.80 +lavfi.psnr.psnr.r=16.73 +lavfi.psnr.mse.g=896.00 +lavfi.psnr.psnr.g=18.61 +lavfi.psnr.mse.b=277.38 +lavfi.psnr.psnr.b=23.70 +lavfi.psnr.mse_avg=851.73 +lavfi.psnr.psnr_avg=18.83 +frame:1 pts:1 pts_time:1 +lavfi.psnr.mse.r=1380.37 +lavfi.psnr.psnr.r=16.73 +lavfi.psnr.mse.g=975.91 +lavfi.psnr.psnr.g=18.24 +lavfi.psnr.mse.b=435.72 +lavfi.psnr.psnr.b=21.74 +lavfi.psnr.mse_avg=930.67 +lavfi.psnr.psnr_avg=18.44 +frame:2 pts:2 pts_time:2 +lavfi.psnr.mse.r=1403.20 +lavfi.psnr.psnr.r=16.66 +lavfi.psnr.mse.g=954.05 +lavfi.psnr.psnr.g=18.34 +lavfi.psnr.mse.b=494.22 +lavfi.psnr.psnr.b=21.19 +lavfi.psnr.mse_avg=950.49 +lavfi.psnr.psnr_avg=18.35 +frame:3 pts:3 pts_time:3 +lavfi.psnr.mse.r=1452.80 +lavfi.psnr.psnr.r=16.51 +lavfi.psnr.mse.g=1001.02 +lavfi.psnr.psnr.g=18.13 +lavfi.psnr.mse.b=557.39 +lavfi.psnr.psnr.b=20.67 +lavfi.psnr.mse_avg=1003.74 +lavfi.psnr.psnr_avg=18.11 +frame:4 pts:4 pts_time:4 +lavfi.psnr.mse.r=1401.25 +lavfi.psnr.psnr.r=16.67 +lavfi.psnr.mse.g=1009.80 +lavfi.psnr.psnr.g=18.09 +lavfi.psnr.mse.b=602.42 +lavfi.psnr.psnr.b=20.33 +lavfi.psnr.mse_avg=1004.49 +lavfi.psnr.psnr_avg=18.11 diff --git a/tests/ref/fate/filter-refcmp-psnr-yuv b/tests/ref/fate/filter-refcmp-psnr-yuv new file mode 100644 index 0000000000..0e634ed0e4 --- /dev/null +++ b/tests/ref/fate/filter-refcmp-psnr-yuv @@ -0,0 +1,45 @@ +frame:0 pts:0 pts_time:0 +lavfi.psnr.mse.y=222.06 +lavfi.psnr.psnr.y=24.67 +lavfi.psnr.mse.u=339.38 +lavfi.psnr.psnr.u=22.82 +lavfi.psnr.mse.v=705.41 +lavfi.psnr.psnr.v=19.65 +lavfi.psnr.mse_avg=372.23 +lavfi.psnr.psnr_avg=22.42 +frame:1 pts:1 pts_time:1 +lavfi.psnr.mse.y=236.74 +lavfi.psnr.psnr.y=24.39 +lavfi.psnr.mse.u=416.17 +lavfi.psnr.psnr.u=21.94 +lavfi.psnr.mse.v=704.98 +lavfi.psnr.psnr.v=19.65 +lavfi.psnr.mse_avg=398.66 +lavfi.psnr.psnr_avg=22.12 +frame:2 pts:2 pts_time:2 +lavfi.psnr.mse.y=234.79 +lavfi.psnr.psnr.y=24.42 +lavfi.psnr.mse.u=435.72 +lavfi.psnr.psnr.u=21.74 +lavfi.psnr.mse.v=699.60 +lavfi.psnr.psnr.v=19.68 +lavfi.psnr.mse_avg=401.23 +lavfi.psnr.psnr_avg=22.10 +frame:3 pts:3 pts_time:3 +lavfi.psnr.mse.y=250.88 +lavfi.psnr.psnr.y=24.14 +lavfi.psnr.mse.u=479.73 +lavfi.psnr.psnr.u=21.32 +lavfi.psnr.mse.v=707.55 +lavfi.psnr.psnr.v=19.63 +lavfi.psnr.mse_avg=422.26 +lavfi.psnr.psnr_avg=21.88 +frame:4 pts:4 pts_time:4 +lavfi.psnr.mse.y=241.05 +lavfi.psnr.psnr.y=24.31 +lavfi.psnr.mse.u=505.04 +lavfi.psnr.psnr.u=21.10 +lavfi.psnr.mse.v=716.00 +lavfi.psnr.psnr.v=19.58 +lavfi.psnr.mse_avg=425.79 +lavfi.psnr.psnr_avg=21.84 diff --git a/tests/ref/fate/filter-refcmp-ssim-rgb b/tests/ref/fate/filter-refcmp-ssim-rgb new file mode 100644 index 0000000000..8c23c60b37 --- /dev/null +++ b/tests/ref/fate/filter-refcmp-ssim-rgb @@ -0,0 +1,30 @@ +frame:0 pts:0 pts_time:0 +lavfi.ssim.R=0.72 +lavfi.ssim.G=0.76 +lavfi.ssim.B=0.89 +lavfi.ssim.All=0.79 +lavfi.ssim.dB=6.74 +frame:1 pts:1 pts_time:1 +lavfi.ssim.R=0.70 +lavfi.ssim.G=0.74 +lavfi.ssim.B=0.85 +lavfi.ssim.All=0.77 +lavfi.ssim.dB=6.31 +frame:2 pts:2 pts_time:2 +lavfi.ssim.R=0.71 +lavfi.ssim.G=0.75 +lavfi.ssim.B=0.84 +lavfi.ssim.All=0.76 +lavfi.ssim.dB=6.29 +frame:3 pts:3 pts_time:3 +lavfi.ssim.R=0.70 +lavfi.ssim.G=0.73 +lavfi.ssim.B=0.83 +lavfi.ssim.All=0.76 +lavfi.ssim.dB=6.11 +frame:4 pts:4 pts_time:4 +lavfi.ssim.R=0.71 +lavfi.ssim.G=0.74 +lavfi.ssim.B=0.80 +lavfi.ssim.All=0.75 +lavfi.ssim.dB=6.05 diff --git a/tests/ref/fate/filter-refcmp-ssim-yuv b/tests/ref/fate/filter-refcmp-ssim-yuv new file mode 100644 index 0000000000..5c8ffb9483 --- /dev/null +++ b/tests/ref/fate/filter-refcmp-ssim-yuv @@ -0,0 +1,30 @@ +frame:0 pts:0 pts_time:0 +lavfi.ssim.Y=0.80 +lavfi.ssim.U=0.76 +lavfi.ssim.V=0.69 +lavfi.ssim.All=0.76 +lavfi.ssim.dB=6.25 +frame:1 pts:1 pts_time:1 +lavfi.ssim.Y=0.80 +lavfi.ssim.U=0.73 +lavfi.ssim.V=0.68 +lavfi.ssim.All=0.75 +lavfi.ssim.dB=6.08 +frame:2 pts:2 pts_time:2 +lavfi.ssim.Y=0.80 +lavfi.ssim.U=0.73 +lavfi.ssim.V=0.68 +lavfi.ssim.All=0.75 +lavfi.ssim.dB=6.10 +frame:3 pts:3 pts_time:3 +lavfi.ssim.Y=0.79 +lavfi.ssim.U=0.72 +lavfi.ssim.V=0.68 +lavfi.ssim.All=0.75 +lavfi.ssim.dB=5.94 +frame:4 pts:4 pts_time:4 +lavfi.ssim.Y=0.80 +lavfi.ssim.U=0.72 +lavfi.ssim.V=0.68 +lavfi.ssim.All=0.75 +lavfi.ssim.dB=5.97 diff --git a/tests/refcmp-metadata.awk b/tests/refcmp-metadata.awk new file mode 100644 index 0000000000..fa21aad0e0 --- /dev/null +++ b/tests/refcmp-metadata.awk @@ -0,0 +1,64 @@ +# Compare metadata filter output containing float value strings to reference +# output data. Returns the whole reference data if delta of each value is below +# threshold, else returns the whole input data. + +function abs(val) { + return ((val < 0.0) ? -val : val); +} + +function max(val1, val2) { + return ((val1 >= val2) ? val1 : val2); +} + +function is_numeric_str(str) { + return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/); +} + +BEGIN { + FS = "="; + # check for "fuzz" (threshold) program parameter, else use default + if (fuzz <= 0.0) { + fuzz = 0.1; + } + # check for "ref" (reference file) program parameter + if (ref) { + ref_nr = 0; + while ((getline line < ref) > 0) { + ref_nr++; + ref_lines[ref_nr] = line; + if (split(line, fields) == 2 && is_numeric_str(fields[2])) { + ref_keys[ref_nr] = fields[1]; + ref_vals[ref_nr] = fields[2] + 0; # convert to number + } + } + close(ref); + } + delta_max = 0; + result = (ref ? 1 : 0); +} + +{ + cmp_lines[NR] = $0; + if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) { + val = $2 + 0; # convert to number + delta = abs((val / ref_vals[NR]) - 1); + delta_max = max(delta_max, delta); + result = result && ($1 == ref_keys[NR]) && (delta <= fuzz); + } else { + result = result && ($0 == ref_lines[NR]); + } +} + +END { + if (result) { + for (i = 1; i <= ref_nr; i++) + print ref_lines[i]; + } else { + for (i = 1; i <= NR; i++) + print cmp_lines[i]; + if (NR != ref_nr) + print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr"; + if (delta_max >= fuzz) + print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr"; + } +} _______________________________________________ ffmpeg-cvslog mailing list ffmpeg-cvslog@ffmpeg.org http://ffmpeg.org/mailman/listinfo/ffmpeg-cvslog