Author: Christoph M. Becker (cmb69) Committer: GitHub (web-flow) Pusher: cmb69 Date: 2024-09-18T12:46:16+02:00
Commit: https://github.com/php/web-qa/commit/7eaaacf64449a03c47f7b39c77b0d21b4fb60db2 Raw diff: https://github.com/php/web-qa/commit/7eaaacf64449a03c47f7b39c77b0d21b4fb60db2.diff Document --FLAKY-- section of PHPTs (GH-43) The given example might not be the best one, since it actually wouldn't need the `--FLAKY--` section, but it's the only one we currently have. We also document the "flaky" convention for SKIPIF sections, Changed paths: A sample_tests/flaky.php A sample_tests/flakyif.php M phpt_details.php Diff: diff --git a/phpt_details.php b/phpt_details.php index 9066bc2..b77fc90 100644 --- a/phpt_details.php +++ b/phpt_details.php @@ -33,6 +33,7 @@ <a href="#file_section">--FILE--</a> | <a href="#fileeof_section">--FILEEOF--</a> | <a href="#file_external_section">--FILE_EXTERNAL--</a> | <a href="#redirecttest_section">--REDIRECTTEST--</a><br/> [<a href="#cgi_section">--CGI--</a>]<br/> [<a href="#xfail_section">--XFAIL--</a>]<br/> +[<a href="#flaky_section">--FLAKY--</a>]<br/> [<a href="#expectheaders_section">--EXPECTHEADERS</a>--]<br/> <a href="#expect_section">--EXPECT--</a> | <a href="#expectf_section">--EXPECTF--</a> | <a href="#expectregex_section">--EXPECTREGEX--</a> | <a href="#expect_external_section">--EXPECT_EXTERNAL--</a> | <a href="#expectf_external_section">--EXPECTF_EXTERNAL--</a> | <a href="#expectregex_external_section">--EXPECTREGEX_EXTERNAL--</a> @@ -116,7 +117,10 @@ <p><b>Format:</b><br/> PHP code enclosed by PHP tags. If the output of this scripts starts with "skip", the test is skipped. If the output starts with "xfail", the test is marked as -expected failure. The "xfail" convention is supported as of PHP 7.2.0.</p> +<a href="#xfail_section">expected failure</a>. If the output starts with "flaky", +the test is marked as <a href="#flaky_section">flaky test</a>. +The "xfail" convention is supported as of PHP 7.2.0. +The "flaky" convention is supported as of PHP 8.2.25 and PHP 8.3.13, respectively.</p> <p><b>Example 1 (snippet):</b><br/> <pre>--SKIPIF-- <?php if (!extension_loaded("filter")) die("Skipped: filter extension required."); ?></pre> @@ -132,6 +136,15 @@ <?php if (getenv('SKIP_ASAN')) die('xfail Startup failure leak'); ?></pre> </p> <p><b>Example 3 (full):</b> <a href="sample_tests/xfailif.php">xfailif.phpt</a></p> +<p><b>Example 4 (snippet):</b><br/> +<pre>--SKIPIF-- +<?php +if (getenv("GITHUB_ACTIONS") && PHP_OS_FAMILY === "Darwin") { + die("flaky Occasionally segfaults on macOS for unknown reasons"); +} +>></pre> +</p> +<p><b>Example 4 (full):</b> <a href="sample_tests/flakyif.php">flakyif.phpt</a></p> </dd> <dt id="conflicts_section">--CONFLICTS--</dt> @@ -666,6 +679,28 @@ <p><b>Example 1 (full):</b> <a href="sample_tests/sample017.php">sample017.phpt</a></p> </dd> +<dt id="flaky_section">--FLAKY--</dt> +<dd> +<p><b>Description:</b><br/> +This section identifies this test as one that occassionally fails. If the test +actually fails, it will be retried one more time, and that result will be reported. +The section should include a brief description of why the test is flaky. Reasons for +this include tests that rely on relatively precise timing, or +temporary disc states. Available as of PHP 8.1.22 and 8.2.9, respectively.</p> +<p>Please do NOT include a --FLAKY-- section without providing a text description for +the reason it is being used.</p> +<p><b>Required:</b><br/> +No.</p> +<p><b>Test Script Support:</b><br/> +run-tests.php</p> +<p><b>Format:</b><br/> +A short plain text description of why this test is flaky.</p> +<p><b>Example 1 (snippet):</b><br/> +<pre>--FLAKY-- +This test frequently fails in CI</pre></p> +<p><b>Example 1 (full):</b> <a href="sample_tests/flaky.php">flaky.phpt</a></p> +</dd> + <dt id="expectheaders_section">--EXPECTHEADERS--</dt> <dd> <p><b>Description:</b><br/> diff --git a/sample_tests/flaky.php b/sample_tests/flaky.php new file mode 100644 index 0000000..e2dabea --- /dev/null +++ b/sample_tests/flaky.php @@ -0,0 +1,47 @@ +<?php +include("../include/functions.php"); + +$TITLE = "Sample Test [PHP-QAT: Quality Assurance Team]"; +$SITE_UPDATE = date("D M d H:i:s Y T", filectime(__FILE__)); + +common_header(); +?> + +<div style="padding: 10px"> +<h1>Sample Test: sample001.phpt</h1> +<p>Back to "<a href="../phpt_details.php">PHPT Test File Layout</a>"</p> +<pre>--TEST-- +Test hrtime() aligns with microtime() +--FLAKY-- +This test frequently fails in CI +--FILE-- +<?php + +$m0 = microtime(true); +$h0 = hrtime(true); +for ($i = 0; $i < 1024*1024; $i++); +$h1 = hrtime(true); +$m1 = microtime(true); + +$d0 = ($m1 - $m0)*1000000000.0; +$d1 = $h1 - $h0; + +/* Relative uncertainty. */ +$d = abs($d0 - $d1)/$d1; + +if ($d > 0.05) { + print "FAIL, $d"; +} else { + print "OK, $d"; +} + +?> +--EXPECTF-- +OK, %f +</pre> +<p>Back to "<a href="../phpt_details.php">PHPT Test File Layout</a>"</p> +</div> + +<?php +common_footer(); +?> diff --git a/sample_tests/flakyif.php b/sample_tests/flakyif.php new file mode 100644 index 0000000..a7ecaef --- /dev/null +++ b/sample_tests/flakyif.php @@ -0,0 +1,59 @@ +<?php +include("../include/functions.php"); + +$TITLE = "Sample Test [PHP-QAT: Quality Assurance Team]"; +$SITE_UPDATE = date("D M d H:i:s Y T", filectime(__FILE__)); + +common_header(); +?> + +<div style="padding: 10px"> +<h1>Sample Test: sample001.phpt</h1> +<p>Back to "<a href="../phpt_details.php">PHPT Test File Layout</a>"</p> +<pre>--TEST-- +Phar::chmod +--EXTENSIONS-- +phar +--INI-- +phar.readonly=1 +phar.require_hash=0 +--SKIPIF-- +<?php +if (getenv("GITHUB_ACTIONS") && PHP_OS_FAMILY === "Darwin") { + die("flaky Occasionally segfaults on macOS for unknown reasons"); +} +?> +--FILE-- +<?php +$fname = __DIR__ . '/' . basename(__FILE__, '.php') . '.1.phar.php'; +$pname = 'phar://hio'; +$file = '<?php include "' . $pname . '/a.php"; __HALT_COMPILER(); ?>'; + +$files = array(); +$files['a.php'] = '<?php echo "This is a\n"; include "'.$pname.'/b.php"; ?>'; +include 'files/phar_test.inc'; +try { + $a = new Phar($fname); + var_dump($a['a.php']->isExecutable()); + $a['a.php']->chmod(0777); + var_dump($a['a.php']->isExecutable()); + $a['a.php']->chmod(0666); + var_dump($a['a.php']->isExecutable()); +} catch (Exception $e) { + echo $e->getMessage() . "\n"; +} +?> +--CLEAN-- +<?php +unlink(__DIR__ . '/' . basename(__FILE__, '.clean.php') . '.1.phar.php'); +?> +--EXPECTF-- +bool(false) +Cannot modify permissions for file "a.php" in phar "%s033a.1.phar.php", write operations are prohibited +</pre> +<p>Back to "<a href="../phpt_details.php">PHPT Test File Layout</a>"</p> +</div> + +<?php +common_footer(); +?>
