This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 68dd10a58e59b784f08cd13a3fa5626282eedc2b Author: Charles Givre <[email protected]> AuthorDate: Mon Dec 16 08:14:48 2019 -0500 DRILL-7484: Malware found in the Drill test folder closes #1934 --- .../drill/exec/store/pcap/TestSessionizePCAP.java | 168 +++++++++++++++------ .../test/resources/store/pcap/attack-trace.pcap | Bin 189103 -> 0 bytes .../test/resources/store/pcap/dataFromRemote.txt | 1 + .../src/test/resources/store/pcap/http.pcap | Bin 0 -> 25803 bytes 4 files changed, 121 insertions(+), 48 deletions(-) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java index 8c2818d..fe1c3d5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java @@ -18,12 +18,20 @@ package org.apache.drill.exec.store.pcap; - +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.exec.physical.rowSet.RowSet; +import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.QueryBuilder; +import org.apache.drill.test.rowSet.RowSetComparison; import org.joda.time.Period; + +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Paths; -import java.time.LocalDateTime; import org.junit.BeforeClass; import org.junit.Test; import java.time.format.DateTimeFormatter; @@ -47,61 +55,125 @@ public class TestSessionizePCAP extends ClusterTest { @Test public void testSessionizedStarQuery() throws Exception { - String sql = "SELECT * FROM cp.`/store/pcap/attack-trace.pcap` WHERE src_port=1821 AND dst_port=445"; - - testBuilder() - .sqlQuery(sql) - .ordered() - .baselineColumns("session_start_time", "session_end_time", "session_duration", "total_packet_count", "connection_time", "src_ip", "dst_ip", "src_port", "dst_port", - "src_mac_address", "dst_mac_address", "tcp_session", "is_corrupt", "data_from_originator", "data_from_remote", "data_volume_from_origin", - "data_volume_from_remote", "packet_count_from_origin", "packet_count_from_remote") - .baselineValues(LocalDateTime.parse("2009-04-20T03:28:28.374", formatter), - LocalDateTime.parse("2009-04-20T03:28:28.508", formatter), - Period.parse("PT0.134S"), 4, - Period.parse("PT0.119S"), - "98.114.205.102", - "192.150.11.111", - 1821, 445, - "00:08:E2:3B:56:01", - "00:30:48:62:4E:4A", - -8791568836279708938L, - false, - "........I....>...>..........Ib...<...<..........I....>...>", "", 62,0, 3, 1) - .go(); + String sql = "SELECT * FROM cp.`/store/pcap/http.pcap`"; + String dataFromRemote = readAFileIntoString(dirTestWatcher.getRootDir().getAbsolutePath() + "/store/pcap/dataFromRemote.txt"); + + QueryBuilder q = client.queryBuilder().sql(sql); + RowSet results = q.rowSet(); + + TupleMetadata expectedSchema = new SchemaBuilder() + .addNullable("src_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("dst_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("src_port", TypeProtos.MinorType.INT) + .addNullable("dst_port", TypeProtos.MinorType.INT) + .addNullable("src_mac_address", TypeProtos.MinorType.VARCHAR) + .addNullable("dst_mac_address", TypeProtos.MinorType.VARCHAR) + .addNullable("session_start_time", TypeProtos.MinorType.TIMESTAMP) + .addNullable("session_end_time", TypeProtos.MinorType.TIMESTAMP) + .addNullable("session_duration", TypeProtos.MinorType.INTERVAL) + .addNullable("total_packet_count", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_origin", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_remote", TypeProtos.MinorType.INT) + .addNullable("packet_count_from_origin", TypeProtos.MinorType.INT) + .addNullable("packet_count_from_remote", TypeProtos.MinorType.INT) + .addNullable("connection_time", TypeProtos.MinorType.INTERVAL) + .addNullable("tcp_session", TypeProtos.MinorType.BIGINT) + .addNullable("is_corrupt", TypeProtos.MinorType.BIT) + .addNullable("data_from_originator", TypeProtos.MinorType.VARCHAR) + .addNullable("data_from_remote", TypeProtos.MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) + .addRow( + "145.254.160.237", + "65.208.228.223", + 3372, 80, + "00:00:01:00:00:00", + "FE:FF:20:00:01:00", + 1084443427311L, + 1084443445216L, + Period.parse("PT17.905S"), 31, + 437,18000,14, 17, + Period.parse("PT0.911S"), + -789689725566200012L, false, + "r-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/[email protected]";, + dataFromRemote + ) + .build(); + + new RowSetComparison(expected).verifyAndClearAll(results); } @Test public void testSessionizedSpecificQuery() throws Exception { - String sql = "SELECT session_start_time, session_end_time,session_duration, total_packet_count, connection_time, src_ip, dst_ip, src_port, dst_port, src_mac_address, dst_mac_address, tcp_session, " + - "is_corrupt, data_from_originator, data_from_remote, data_volume_from_origin, data_volume_from_remote, packet_count_from_origin, packet_count_from_remote " + - "FROM cp.`/store/pcap/attack-trace.pcap` WHERE src_port=1821 AND dst_port=445"; - - testBuilder() - .sqlQuery(sql) - .ordered() - .baselineColumns("session_start_time", "session_end_time", "session_duration", "total_packet_count", "connection_time", "src_ip", "dst_ip", "src_port", "dst_port", - "src_mac_address", "dst_mac_address", "tcp_session", "is_corrupt", "data_from_originator", "data_from_remote", "data_volume_from_origin", - "data_volume_from_remote", "packet_count_from_origin", "packet_count_from_remote") - .baselineValues(LocalDateTime.parse("2009-04-20T03:28:28.374", formatter), - LocalDateTime.parse("2009-04-20T03:28:28.508", formatter), - Period.parse("PT0.134S"), 4, - Period.parse("PT0.119S"), - "98.114.205.102", - "192.150.11.111", - 1821, 445, - "00:08:E2:3B:56:01", - "00:30:48:62:4E:4A", - -8791568836279708938L, - false, - "........I....>...>..........Ib...<...<..........I....>...>", "", 62,0, 3, 1) - .go(); + String sql = "SELECT src_ip, dst_ip, src_port, dst_port, src_mac_address, dst_mac_address," + + "session_start_time, session_end_time, session_duration, total_packet_count, data_volume_from_origin, data_volume_from_remote," + + "packet_count_from_origin, packet_count_from_remote, connection_time, tcp_session, is_corrupt, data_from_originator, data_from_remote " + + "FROM cp.`/store/pcap/http.pcap`"; + + String dataFromRemote = readAFileIntoString(dirTestWatcher.getRootDir().getAbsolutePath() + "/store/pcap/dataFromRemote.txt"); + + QueryBuilder q = client.queryBuilder().sql(sql); + RowSet results = q.rowSet(); + + TupleMetadata expectedSchema = new SchemaBuilder() + .addNullable("src_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("dst_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("src_port", TypeProtos.MinorType.INT) + .addNullable("dst_port", TypeProtos.MinorType.INT) + .addNullable("src_mac_address", TypeProtos.MinorType.VARCHAR) + .addNullable("dst_mac_address", TypeProtos.MinorType.VARCHAR) + .addNullable("session_start_time", TypeProtos.MinorType.TIMESTAMP) + .addNullable("session_end_time", TypeProtos.MinorType.TIMESTAMP) + .addNullable("session_duration", TypeProtos.MinorType.INTERVAL) + .addNullable("total_packet_count", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_origin", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_remote", TypeProtos.MinorType.INT) + .addNullable("packet_count_from_origin", TypeProtos.MinorType.INT) + .addNullable("packet_count_from_remote", TypeProtos.MinorType.INT) + .addNullable("connection_time", TypeProtos.MinorType.INTERVAL) + .addNullable("tcp_session", TypeProtos.MinorType.BIGINT) + .addNullable("is_corrupt", TypeProtos.MinorType.BIT) + .addNullable("data_from_originator", TypeProtos.MinorType.VARCHAR) + .addNullable("data_from_remote", TypeProtos.MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) + .addRow( + "145.254.160.237", + "65.208.228.223", + 3372, 80, + "00:00:01:00:00:00", + "FE:FF:20:00:01:00", + 1084443427311L, + 1084443445216L, + Period.parse("PT17.905S"), 31, + 437,18000,14, 17, + Period.parse("PT0.911S"), + -789689725566200012L, false, + "r-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/[email protected]";, + dataFromRemote + ) + .build(); + + new RowSetComparison(expected).verifyAndClearAll(results); } @Test public void testSerDe() throws Exception { - String sql = "SELECT COUNT(*) FROM cp.`/store/pcap/attack-trace.pcap`"; + String sql = "SELECT COUNT(*) FROM cp.`/store/pcap/http.pcap`"; String plan = queryBuilder().sql(sql).explainJson(); long cnt = queryBuilder().physical(plan).singletonLong(); - assertEquals("Counts should match", 5L, cnt); + assertEquals("Counts should match", 1L, cnt); + } + + /** + * Helper function to read a file into a String. + * @param filePath Input file which is to be read into a String + * @return String The text content of the file. + * @throws IOException If the file is unreachable or unreadable, throw IOException. + */ + private static String readAFileIntoString(String filePath) throws IOException { + return new String(Files.readAllBytes(Paths.get(filePath))); } } diff --git a/exec/java-exec/src/test/resources/store/pcap/attack-trace.pcap b/exec/java-exec/src/test/resources/store/pcap/attack-trace.pcap deleted file mode 100644 index 68e1fff..0000000 Binary files a/exec/java-exec/src/test/resources/store/pcap/attack-trace.pcap and /dev/null differ diff --git a/exec/java-exec/src/test/resources/store/pcap/dataFromRemote.txt b/exec/java-exec/src/test/resources/store/pcap/dataFromRemote.txt new file mode 100644 index 0000000..cde5c69 --- /dev/null +++ b/exec/java-exec/src/test/resources/store/pcap/dataFromRemote.txt @@ -0,0 +1 @@ +10:17:12 GMT..Server: Apache..Last-Modified: Tue, 20 Apr 2004 13:17:00 GMT..ETag: "9a01a-4696-7e354b00"..Accept-Ranges: bytes..Content-Length: 18070..Keep-Alive: timeout=15, max=100..Connection: Keep-Alive..Content-Type: text/html; charset=ISO-8859-1....<?xml version="1.0" encoding="UTF-8"?>.<!DOCTYPE html. PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN". "DTD/xhtml1-strict.dtd">.<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en">. <head>. <title>Ethereal: Download</title [...] \ No newline at end of file diff --git a/exec/java-exec/src/test/resources/store/pcap/http.pcap b/exec/java-exec/src/test/resources/store/pcap/http.pcap new file mode 100644 index 0000000..54f6f29 Binary files /dev/null and b/exec/java-exec/src/test/resources/store/pcap/http.pcap differ
