[
https://issues.apache.org/jira/browse/DRILL-7554?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17088879#comment-17088879
]
ASF GitHub Bot commented on DRILL-7554:
---------------------------------------
cgivre commented on a change in pull request #1962:
URL: https://github.com/apache/drill/pull/1962#discussion_r412333558
##########
File path:
contrib/format-ltsv/src/test/java/org/apache/drill/exec/store/ltsv/TestLTSVRecordReader.java
##########
@@ -17,84 +17,190 @@
*/
package org.apache.drill.exec.store.ltsv;
+import org.apache.drill.categories.RowSetTests;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.physical.rowSet.RowSet;
+import org.apache.drill.exec.physical.rowSet.RowSetBuilder;
import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.exec.record.metadata.SchemaBuilder;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.store.dfs.ZipCodec;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
+import org.apache.drill.test.QueryBuilder;
+import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Paths;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+@Category(RowSetTests.class)
public class TestLTSVRecordReader extends ClusterTest {
@BeforeClass
public static void setup() throws Exception {
- startCluster(ClusterFixture.builder(dirTestWatcher));
-
- DrillbitContext context = cluster.drillbit().getContext();
- FileSystemConfig original = (FileSystemConfig)
context.getStorage().getPlugin("cp").getConfig();
- Map<String, FormatPluginConfig> newFormats = new
HashMap<>(original.getFormats());
- newFormats.put("ltsv", new LTSVFormatPluginConfig());
- FileSystemConfig pluginConfig = new
FileSystemConfig(original.getConnection(), original.getConfig(),
original.getWorkspaces(), newFormats);
- pluginConfig.setEnabled(true);
- context.getStorage().createOrUpdate("cp", pluginConfig, true);
+ ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher));
+
+ LTSVFormatPluginConfig formatConfig = new LTSVFormatPluginConfig();
+ cluster.defineFormat("cp", "ltsv", formatConfig);
+
+ // Needed for compressed file unit test
+ dirTestWatcher.copyResourceToRoot(Paths.get("ltsv/"));
}
@Test
public void testWildcard() throws Exception {
- testBuilder()
- .sqlQuery("SELECT * FROM cp.`simple.ltsv`")
- .unOrdered()
- .baselineColumns("host", "forwardedfor", "req", "status", "size",
"referer", "ua", "reqtime", "apptime", "vhost")
- .baselineValues("xxx.xxx.xxx.xxx", "-", "GET /v1/xxx HTTP/1.1", "200",
"4968", "-", "Java/1.8.0_131", "2.532", "2.532", "api.example.com")
- .baselineValues("xxx.xxx.xxx.xxx", "-", "GET /v1/yyy HTTP/1.1", "200",
"412", "-", "Java/1.8.0_201", "3.580", "3.580", "api.example.com")
- .go();
+ String sql = "SELECT * FROM cp.`ltsv/simple.ltsv`";
+ QueryBuilder q = client.queryBuilder().sql(sql);
+ RowSet results = q.rowSet();
+
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .addNullable("host", TypeProtos.MinorType.VARCHAR)
+ .addNullable("forwardedfor", TypeProtos.MinorType.VARCHAR)
+ .addNullable("req", TypeProtos.MinorType.VARCHAR)
+ .addNullable("status", TypeProtos.MinorType.VARCHAR)
+ .addNullable("size", TypeProtos.MinorType.VARCHAR)
+ .addNullable("referer", TypeProtos.MinorType.VARCHAR)
+ .addNullable("ua", TypeProtos.MinorType.VARCHAR)
+ .addNullable("reqtime", TypeProtos.MinorType.VARCHAR)
+ .addNullable("apptime", TypeProtos.MinorType.VARCHAR)
+ .addNullable("vhost", TypeProtos.MinorType.VARCHAR)
+ .buildSchema();
+
+ RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
+ .addRow("xxx.xxx.xxx.xxx", "-", "GET /v1/xxx HTTP/1.1", "200", "4968",
"-", "Java/1.8.0_131", "2.532", "2.532", "api.example.com")
+ .addRow("xxx.xxx.xxx.xxx", "-", "GET /v1/yyy HTTP/1.1", "200", "412",
"-", "Java/1.8.0_201", "3.580", "3.580", "api.example.com")
+ .build();
+
+ new RowSetComparison(expected).verifyAndClearAll(results);
}
@Test
public void testSelectColumns() throws Exception {
- testBuilder()
- .sqlQuery("SELECT ua, reqtime FROM cp.`simple.ltsv`")
- .unOrdered()
- .baselineColumns("ua", "reqtime")
- .baselineValues("Java/1.8.0_131", "2.532")
- .baselineValues("Java/1.8.0_201", "3.580")
- .go();
+ String sql = "SELECT ua, reqtime FROM cp.`ltsv/simple.ltsv`";
+
+ QueryBuilder q = client.queryBuilder().sql(sql);
+ RowSet results = q.rowSet();
+
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .addNullable("ua", TypeProtos.MinorType.VARCHAR)
+ .addNullable("reqtime", TypeProtos.MinorType.VARCHAR)
+ .buildSchema();
+
+ RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
+ .addRow("Java/1.8.0_131", "2.532")
+ .addRow("Java/1.8.0_201", "3.580")
+ .build();
+
+ new RowSetComparison(expected).verifyAndClearAll(results);
}
@Test
public void testQueryWithConditions() throws Exception {
- testBuilder()
- .sqlQuery("SELECT * FROM cp.`simple.ltsv` WHERE reqtime > 3.0")
- .unOrdered()
- .baselineColumns("host", "forwardedfor", "req", "status", "size",
"referer", "ua", "reqtime", "apptime", "vhost")
- .baselineValues("xxx.xxx.xxx.xxx", "-", "GET /v1/yyy HTTP/1.1", "200",
"412", "-", "Java/1.8.0_201", "3.580", "3.580", "api.example.com")
- .go();
+ String sql = "SELECT * FROM cp.`ltsv/simple.ltsv` WHERE reqtime > 3.0";
+
+ QueryBuilder q = client.queryBuilder().sql(sql);
+ RowSet results = q.rowSet();
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .addNullable("host", TypeProtos.MinorType.VARCHAR)
+ .addNullable("forwardedfor", TypeProtos.MinorType.VARCHAR)
+ .addNullable("req", TypeProtos.MinorType.VARCHAR)
+ .addNullable("status", TypeProtos.MinorType.VARCHAR)
+ .addNullable("size", TypeProtos.MinorType.VARCHAR)
+ .addNullable("referer", TypeProtos.MinorType.VARCHAR)
+ .addNullable("ua", TypeProtos.MinorType.VARCHAR)
+ .addNullable("reqtime", TypeProtos.MinorType.VARCHAR)
+ .addNullable("apptime", TypeProtos.MinorType.VARCHAR)
+ .addNullable("vhost", TypeProtos.MinorType.VARCHAR)
+ .buildSchema();
+
+ RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
+ .addRow("xxx.xxx.xxx.xxx", "-", "GET /v1/yyy HTTP/1.1", "200", "412",
"-", "Java/1.8.0_201", "3.580", "3.580", "api.example.com")
+ .build();
+
+ new RowSetComparison(expected).verifyAndClearAll(results);
}
@Test
public void testSkipEmptyLines() throws Exception {
- assertEquals(2, queryBuilder().sql("SELECT * FROM
cp.`emptylines.ltsv`").run().recordCount());
+ assertEquals(2, queryBuilder().sql("SELECT * FROM
cp.`ltsv/emptylines.ltsv`").run().recordCount());
}
@Test
public void testReadException() throws Exception {
try {
- run("SELECT * FROM cp.`invalid.ltsv`");
+ run("SELECT * FROM cp.`ltsv/invalid.ltsv`");
fail();
} catch (UserException e) {
assertEquals(UserBitShared.DrillPBError.ErrorType.DATA_READ,
e.getErrorType());
- assertTrue(e.getMessage().contains("Failure while reading messages from
/invalid.ltsv. Record reader was at record: 1"));
+ assertTrue(e.getMessage().contains("Invalid LTSV format at line 1:
time:30/Nov/2016:00:55:08 +0900"));
}
}
+ @Test
+ public void testSerDe() throws Exception {
+ String sql = "SELECT COUNT(*) as cnt FROM cp.`ltsv/simple.ltsv`";
+ String plan = queryBuilder().sql(sql).explainJson();
+ long cnt = queryBuilder().physical(plan).singletonLong();
+ assertEquals("Counts should match",2L, cnt);
+ }
+
+ @Test
Review comment:
@arina-ielchiieva
I added a test and I'm getting a strange error here. This query works if I
run it from sqlline.
Here's the query:
```
SELECT ua, reqtime FROM table(dfs.`ltsv/simple.ltsv` (type => 'ltsv', schema
=> 'inline=(ua varchar, reqtime varchar)'))
```
Any suggestions? Full stack trace below.
```
org.apache.drill.exec.rpc.RpcException:
org.apache.drill.common.exceptions.UserRemoteException: SYSTEM ERROR:
AssertionError: BOOLEAN
Please, refer to logs for more information.
[Error Id: e36171e8-e4bf-4a78-ac60-1ec396fa6f28 on 192.168.1.25:31013]
at
org.apache.drill.exec.rpc.RpcException.mapException(RpcException.java:59)
at
org.apache.drill.exec.client.DrillClient$ListHoldingResultsListener.getResults(DrillClient.java:881)
at
org.apache.drill.exec.client.DrillClient.runQuery(DrillClient.java:583)
at org.apache.drill.test.QueryBuilder.results(QueryBuilder.java:331)
at org.apache.drill.test.QueryBuilder.rowSet(QueryBuilder.java:352)
at
org.apache.drill.exec.store.ltsv.TestLTSVRecordReader.testInlineSchema(TestLTSVRecordReader.java:193)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.drill.common.exceptions.UserRemoteException: SYSTEM
ERROR: AssertionError: BOOLEAN
Please, refer to logs for more information.
[Error Id: e36171e8-e4bf-4a78-ac60-1ec396fa6f28 on 192.168.1.25:31013]
at
org.apache.drill.exec.rpc.user.QueryResultHandler.resultArrived(QueryResultHandler.java:125)
at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:422)
at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:96)
at
org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:273)
at
org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:243)
at
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335)
at
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335)
at
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335)
at
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:312)
at
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:286)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335)
at
io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335)
at
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1294)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356)
at
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342)
at
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:911)
at
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131)
at
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
... 1 more
Caused by: org.apache.drill.exec.work.foreman.ForemanException: Unexpected
exception during fragment initialization: BOOLEAN
at org.apache.drill.exec.work.foreman.Foreman.run(Foreman.java:301)
at .......(:0)
Caused by: java.lang.AssertionError: BOOLEAN
at
org.apache.calcite.sql.type.SqlTypeExplicitPrecedenceList.compareTypePrecedence(SqlTypeExplicitPrecedenceList.java:140)
at org.apache.calcite.sql.SqlUtil.bestMatch(SqlUtil.java:687)
at
org.apache.calcite.sql.SqlUtil.filterRoutinesByTypePrecedence(SqlUtil.java:656)
at
org.apache.calcite.sql.SqlUtil.lookupSubjectRoutines(SqlUtil.java:515)
at org.apache.calcite.sql.SqlUtil.lookupRoutine(SqlUtil.java:435)
at org.apache.calcite.sql.SqlFunction.deriveType(SqlFunction.java:240)
at org.apache.calcite.sql.SqlFunction.deriveType(SqlFunction.java:218)
at
org.apache.calcite.sql.validate.SqlValidatorImpl$DeriveTypeVisitor.visit(SqlValidatorImpl.java:5640)
at
org.apache.calcite.sql.validate.SqlValidatorImpl$DeriveTypeVisitor.visit(SqlValidatorImpl.java:5627)
at org.apache.calcite.sql.SqlCall.accept(SqlCall.java:139)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.deriveTypeImpl(SqlValidatorImpl.java:1692)
at
org.apache.calcite.sql.validate.ProcedureNamespace.validateImpl(ProcedureNamespace.java:53)
at
org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:1009)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:969)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3129)
at
org.apache.drill.exec.planner.sql.conversion.DrillValidator.validateFrom(DrillValidator.java:63)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3111)
at
org.apache.drill.exec.planner.sql.conversion.DrillValidator.validateFrom(DrillValidator.java:63)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3383)
at
org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
at
org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:1009)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:969)
at org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:216)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:944)
at
org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:651)
at
org.apache.drill.exec.planner.sql.conversion.SqlConverter.validate(SqlConverter.java:189)
at
org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler.validateNode(DefaultSqlHandler.java:641)
at
org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler.validateAndConvert(DefaultSqlHandler.java:195)
at
org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler.getPlan(DefaultSqlHandler.java:169)
at
org.apache.drill.exec.planner.sql.DrillSqlWorker.getQueryPlan(DrillSqlWorker.java:283)
at
org.apache.drill.exec.planner.sql.DrillSqlWorker.getPhysicalPlan(DrillSqlWorker.java:163)
at
org.apache.drill.exec.planner.sql.DrillSqlWorker.convertPlan(DrillSqlWorker.java:128)
at
org.apache.drill.exec.planner.sql.DrillSqlWorker.getPlan(DrillSqlWorker.java:93)
at org.apache.drill.exec.work.foreman.Foreman.runSQL(Foreman.java:593)
at org.apache.drill.exec.work.foreman.Foreman.run(Foreman.java:274)
... 1 more
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Convert LTSV Format Plugin to EVF
> ---------------------------------
>
> Key: DRILL-7554
> URL: https://issues.apache.org/jira/browse/DRILL-7554
> Project: Apache Drill
> Issue Type: Improvement
> Components: Storage - Text & CSV
> Affects Versions: 1.17.0
> Reporter: Charles Givre
> Assignee: Charles Givre
> Priority: Major
> Fix For: 1.18.0
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
