[
https://issues.apache.org/jira/browse/DRILL-7570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17030726#comment-17030726
]
ASF GitHub Bot commented on DRILL-7570:
---------------------------------------
arina-ielchiieva commented on pull request #1968: DRILL-7570: Fix unstable
statistics tests
URL: https://github.com/apache/drill/pull/1968#discussion_r375310419
##########
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java
##########
@@ -522,46 +601,45 @@ public void testHistogramWithColumnsWithAllNulls()
throws Exception {
.go();
} finally {
- resetSessionOption("planner.slice_target");
- resetSessionOption("store.format");
+ client.resetSession(ExecConstants.SLICE_TARGET);
+ client.resetSession(ExecConstants.OUTPUT_FORMAT_OPTION);
}
}
@Test
public void testHistogramWithIntervalPredicate() throws Exception {
try {
- test("ALTER SESSION SET `planner.slice_target` = 1");
- test("ALTER SESSION SET `store.format` = 'parquet'");
- test("create table dfs.tmp.orders2 as select * from
cp.`tpch/orders.parquet`");
- test("analyze table dfs.tmp.orders2 compute statistics");
- test("alter session set `planner.statistics.use` = true");
+ client.alterSession(ExecConstants.SLICE_TARGET, 1);
+ client.alterSession(ExecConstants.OUTPUT_FORMAT_OPTION, "parquet");
+ run("create table dfs.tmp.orders2 as select * from
cp.`tpch/orders.parquet`");
+ run("analyze table dfs.tmp.orders2 compute statistics");
+ client.alterSession(PlannerSettings.STATISTICS_USE.getOptionName(),
true);
String query = "select 1 from dfs.tmp.orders2 o where o.o_orderdate >=
date '1996-10-01' and o.o_orderdate < date '1996-10-01' + interval '3' month";
String[] expectedPlan1 = {"Filter\\(condition.*\\).*rowcount =
59?.*,.*", "Scan.*columns=\\[`o_orderdate`\\].*rowcount = 15000.0.*"};
- PlanTestBase.testPlanWithAttributesMatchingPatterns(query,
expectedPlan1, new String[]{});
+ queryBuilder()
+ .sql(query)
+ .detailedPlanMatcher()
+ .include(expectedPlan1)
+ .match();
} finally {
- resetSessionOption("planner.slice_target");
- resetSessionOption("store.format");
+ client.resetSession(ExecConstants.SLICE_TARGET);
+ client.resetSession(ExecConstants.OUTPUT_FORMAT_OPTION);
}
}
//Helper function to verify output of ANALYZE statement
private void verifyAnalyzeOutput(String query, String message) throws
Exception {
- List<QueryDataBatch>result = testRunAndReturn(QueryType.SQL, query);
- List<List<String>> output = new ArrayList<>();
- assertTrue(result.size() == 1);
- final QueryDataBatch batch = result.get(0);
- final RecordBatchLoader loader = new
RecordBatchLoader(getDrillbitContext().getAllocator());
- loader.load(batch.getHeader().getDef(), batch.getData());
- output.add(new ArrayList<String>());
- for (VectorWrapper<?> vw: loader) {
- ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
- Object o = accessor.getObject(0);
- output.get(0).add(o == null ? null: o.toString());
+ DirectRowSet rowSet = queryBuilder().sql(query).rowSet();
+ assertEquals(1, rowSet.rowCount());
+
+ RowSetReader reader = rowSet.reader();
+ assertEquals(2, reader.columnCount());
+ while (reader.next()) {
+ ObjectReader column = reader.column(1);
+ assertEquals(message, column.isNull() ? null:
column.getObject().toString());
}
- batch.release();
- loader.clear();
- assertTrue(output.get(0).size() == 2);
- assertEquals(message, output.get(0).get(1));
+
+ rowSet.clear();
Review comment:
finally?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Fix unstable statistics tests
> -----------------------------
>
> Key: DRILL-7570
> URL: https://issues.apache.org/jira/browse/DRILL-7570
> Project: Apache Drill
> Issue Type: Task
> Affects Versions: 1.17.0
> Reporter: Vova Vysotskyi
> Assignee: Vova Vysotskyi
> Priority: Major
> Fix For: 1.18.0
>
>
> Drill contains tests for checking that statistics is applied, some of them
> also use sampling to calculate statistics value.
> Sampling adds limit above scan, but tests check the value of the estimated
> row count to verify that statistics were applied. limit without sorting
> doesn't guarantee consistent results, so these tests may fail sometime:
> {noformat}
> [ERROR] TestMetastoreCommands.testAnalyzeWithSampleStatistics:2739 Did not
> find expected pattern in plan: Filter\(condition.*\).*rowcount = 96.25,
> 00-00 Screen : rowType = RecordType(ANY employee_id): rowcount = 105.0,
> cumulative cost = {2530.5 rows, 7570.5 cpu, 2310.0 io, 0.0 network, 0.0
> memory}, id = 336738
> 00-01 Project(employee_id=[$1]) : rowType = RecordType(ANY employee_id):
> rowcount = 105.0, cumulative cost = {2520.0 rows, 7560.0 cpu, 2310.0 io, 0.0
> network, 0.0 memory}, id = 336737
> 00-02 SelectionVectorRemover : rowType = RecordType(ANY department_id,
> ANY employee_id): rowcount = 105.0, cumulative cost = {2415.0 rows, 7455.0
> cpu, 2310.0 io, 0.0 network, 0.0 memory}, id = 336736
> 00-03 Filter(condition=[=($0, 2)]) : rowType = RecordType(ANY
> department_id, ANY employee_id): rowcount = 105.0, cumulative cost = {2310.0
> rows, 7350.0 cpu, 2310.0 io, 0.0 network, 0.0 memory}, id = 336735
> 00-04 Scan(table=[[dfs, tmp, employeeWithStatsFile]],
> groupscan=[ParquetGroupScan [entries=[ReadEntryWithPath
> [path=/home/runner/work/drill/drill/exec/java-exec/target/org.apache.drill.exec.sql.TestMetastoreCommands/dfsTestTmp/1580901135676-0/employeeWithStatsFile/0_0_0.parquet]],
>
> selectionRoot=/home/runner/work/drill/drill/exec/java-exec/target/org.apache.drill.exec.sql.TestMetastoreCommands/dfsTestTmp/1580901135676-0/employeeWithStatsFile,
> numFiles=1, numRowGroups=1, usedMetadataFile=false, usedMetastore=true,
> filter=equal(`department_id`, 2) , columns=[`department_id`,
> `employee_id`]]]) : rowType = RecordType(ANY department_id, ANY employee_id):
> rowcount = 1155.0, cumulative cost = {1155.0 rows, 2310.0 cpu, 2310.0 io, 0.0
> network, 0.0 memory}, id = 336734
> expected:<true> but was:<false> {noformat}
> List of tests to fix:
> - TestMetastoreCommands.testAnalyzeWithSampleStatistics;
> - TestAnalyze.basic3.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
