This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0ae7b8b565 create h2 tables for result comparison in
TPCHQueryIntegrationTest (#11389)
0ae7b8b565 is described below
commit 0ae7b8b5650d552b77d3c42ea279c4a828716279
Author: Haitao Zhang <[email protected]>
AuthorDate: Fri Sep 1 16:01:59 2023 -0700
create h2 tables for result comparison in TPCHQueryIntegrationTest (#11389)
---
.../tests/ClusterIntegrationTestUtils.java | 37 +++-
.../pinot/integration/tests/tpch/Constants.java | 44 ++++
.../apache/pinot/integration/tests/tpch/README.md | 43 ++++
.../tests/{ => tpch}/TPCHQueryIntegrationTest.java | 126 ++++++-----
.../pinot/integration/tests/tpch/TblToAvro.java | 233 +++++++++++++++++++++
.../src/test/resources/tpch/1.sql | 2 +-
.../src/test/resources/tpch/10.sql | 4 +-
.../src/test/resources/tpch/11.sql | 4 +-
.../src/test/resources/tpch/12.sql | 4 +-
.../src/test/resources/tpch/14.sql | 4 +-
.../src/test/resources/tpch/15.sql | 4 +-
.../src/test/resources/tpch/22.sql | 4 +-
.../src/test/resources/tpch/3.sql | 4 +-
.../src/test/resources/tpch/4.sql | 4 +-
.../src/test/resources/tpch/5.sql | 4 +-
.../src/test/resources/tpch/6.sql | 4 +-
.../src/test/resources/tpch/{7.sql => 7-h2.sql} | 6 +-
.../src/test/resources/tpch/7.sql | 6 +-
.../src/test/resources/tpch/{8.sql => 8-h2.sql} | 6 +-
.../src/test/resources/tpch/8.sql | 6 +-
.../src/test/resources/tpch/{9.sql => 9-h2.sql} | 2 +-
.../src/test/resources/tpch/9.sql | 2 +-
.../batch/tpch/customer/rawdata/customer.avro | Bin 1741 -> 608 bytes
.../batch/tpch/lineitem/lineitem_schema.json | 6 +-
.../batch/tpch/lineitem/rawdata/lineitem.avro | Bin 1721 -> 1615 bytes
.../examples/batch/tpch/nation/rawdata/nation.avro | Bin 1668 -> 2529 bytes
.../examples/batch/tpch/orders/orders_schema.json | 2 +-
.../examples/batch/tpch/orders/rawdata/orders.avro | Bin 1301 -> 616 bytes
.../examples/batch/tpch/part/rawdata/part.avro | Bin 1502 -> 632 bytes
.../batch/tpch/partsupp/rawdata/partsupp.avro | Bin 1322 -> 1017 bytes
.../examples/batch/tpch/region/rawdata/region.avro | Bin 580 -> 630 bytes
.../batch/tpch/supplier/rawdata/supplier.avro | Bin 1466 -> 533 bytes
32 files changed, 470 insertions(+), 91 deletions(-)
diff --git
a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
index 15c4f90ed7..0259126daf 100644
---
a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
+++
b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
@@ -132,9 +132,8 @@ public class ClusterIntegrationTestUtils {
break;
}
if (typesInUnion.size() == 2) {
- Schema.Type type = typesInUnion.get(0).getType();
- Assert.assertTrue(isSingleValueAvroFieldType(type));
- Assert.assertEquals(typesInUnion.get(1).getType(),
Schema.Type.NULL);
+ Schema.Type type =
extractSingleValueAvroFieldTypeFromTwoSizedUnion(typesInUnion.get(0).getType(),
+ typesInUnion.get(1).getType());
h2FieldNameAndTypes.add(buildH2FieldNameAndType(fieldName, type,
true));
break;
}
@@ -200,6 +199,26 @@ public class ClusterIntegrationTestUtils {
}
}
+ /**
+ * Helper method to extract the single value Avro field type from a two
sized UNION Avro field type if the UNION
+ * contains one single-value type and one NULL type; otherwise, fail the
test.
+ * @param type1 the first type in the UNION
+ * @param type2 the second type in the UNION
+ * @return the single value Avro field type in the UNION if the UNION
contains one single-value type and one NULL type
+ */
+ private static Schema.Type
extractSingleValueAvroFieldTypeFromTwoSizedUnion(Schema.Type type1, Schema.Type
type2) {
+ if (type1 == Schema.Type.NULL) {
+ Assert.assertTrue(isSingleValueAvroFieldType(type2));
+ return type2;
+ }
+ if (type2 == Schema.Type.NULL) {
+ Assert.assertTrue(isSingleValueAvroFieldType(type1));
+ return type1;
+ }
+ Assert.fail(String.format("Unsupported UNION Avro field with underlying
types: %s, %s", type1, type2));
+ return null;
+ }
+
/**
* Helper method to check whether the given Avro field type is a single
value type (non-NULL).
*
@@ -242,7 +261,7 @@ public class ClusterIntegrationTestUtils {
h2FieldType = "bigint";
break;
case "string":
- h2FieldType = "varchar(128)";
+ h2FieldType = "varchar(256)";
break;
default:
h2FieldType = avroFieldTypeName;
@@ -1007,6 +1026,16 @@ public class ClusterIntegrationTestUtils {
}
public static boolean fuzzyCompare(String h2Value, String brokerValue,
String connectionValue) {
+ if (("null".equals(h2Value) || h2Value == null)
+ && ("null".equals(brokerValue) || brokerValue == null)
+ && ("null".equals(connectionValue) || connectionValue == null)) {
+ return false;
+ }
+ if ("null".equals(h2Value) || h2Value == null
+ || "null".equals(brokerValue) || brokerValue == null
+ || "null".equals(connectionValue) || connectionValue == null) {
+ return true;
+ }
// Fuzzy compare expected value and actual value
boolean error = false;
if (isParsableDouble(h2Value)) {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/Constants.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/Constants.java
new file mode 100644
index 0000000000..7fbae29c08
--- /dev/null
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/Constants.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.tpch;
+
+import java.io.File;
+
+
+/**
+ * Constants used in TPCH integration tests.
+ */
+public final class Constants {
+ static final String[] TPCH_TABLE_NAMES = {
+ "customer", "lineitem", "nation", "orders", "part", "partsupp",
"region", "supplier"
+ };
+ static final String AVRO_FILE_SUFFIX = ".avro";
+ private static final String TPCH_TABLE_RESOURCE_FOLDER_PREFIX =
"examples/batch/tpch/";
+ private Constants() {
+ }
+
+ static String getTableResourceFolder(String tableName) {
+ return TPCH_TABLE_RESOURCE_FOLDER_PREFIX + tableName;
+ }
+
+ static String getTableAvroFilePath(String tableName) {
+ return getTableResourceFolder(tableName) + File.separator
+ + "rawdata" + File.separator + tableName + AVRO_FILE_SUFFIX;
+ }
+}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/README.md
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/README.md
new file mode 100644
index 0000000000..784dd0f2aa
--- /dev/null
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/README.md
@@ -0,0 +1,43 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+# How to run advanced TPCHQueryIntegrationTest
+## enrich data
+1. Follow https://dev.mysql.com/doc/heatwave/en/mys-hw-tpch-sample-data.html
to generate tpch data.
+The current tpch dataset in the `resource` folder was generated using scale
factor 0.0001, which is too small for the advanced query test.
+```
+./dbgen -vf -s 0.0001
+```
+If your encounter `fatal error: 'malloc.h' file not found` while building
`dbgen` binary, please replace `malloc.h` with
+`stdlib.h` in files having this problem.
+2. Run TblToAvro to convert the data to avro format. Remember to provide the
absolute folder path of those `tbl` files.
+3. Copy those replace the current dataset in the `resource` folder.
+```
+export TPCH_AVRO_FILE_SOURCE_FOLDER={the absolute folder path of tbl files}
+export PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER={the absolute folder path of
pinot tpch integration test source folder
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/customer.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/lineitem.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/nation.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/orders.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/part.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/partsupp.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/region.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+cp $TPCH_AVRO_FILE_SOURCE_FOLDER/supplier.avro
$PINOT_TPCH_INTEGRATION_TEST_SOURCE_FOLDER
+```
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TPCHQueryIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TPCHQueryIntegrationTest.java
similarity index 54%
rename from
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TPCHQueryIntegrationTest.java
rename to
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TPCHQueryIntegrationTest.java
index 42065672d0..e4e3181aa6 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TPCHQueryIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TPCHQueryIntegrationTest.java
@@ -16,24 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.integration.tests;
+package org.apache.pinot.integration.tests.tpch;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.Charset;
+import java.sql.ResultSet;
+import java.sql.Statement;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.pinot.client.ResultSetGroup;
+import org.apache.pinot.integration.tests.BaseClusterIntegrationTest;
+import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.tools.utils.JarUtils;
@@ -44,26 +45,20 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-
+// TODO: extract common functions from TPCHQueryIntegrationTest and
SSBQueryIntegrationTest
+/**
+ * Integration test that tests Pinot using TPCH data.
+ * Data is loaded into Pinot and H2 from /resources/examples/batch/tpch. The
dataset size is very small, please follow
+ * REAME.md to generate a larger dataset for better testing.
+ * Queries are executed against Pinot and H2, and the results are compared.
+ */
public class TPCHQueryIntegrationTest extends BaseClusterIntegrationTest {
- private static final Map<String, String> TPCH_QUICKSTART_TABLE_RESOURCES;
private static final int NUM_TPCH_QUERIES = 24;
- private static final Set<Integer> EXEMPT_QUERIES;
-
- static {
- TPCH_QUICKSTART_TABLE_RESOURCES = new HashMap<>();
- TPCH_QUICKSTART_TABLE_RESOURCES.put("orders",
"examples/batch/tpch/orders");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("lineitem",
"examples/batch/tpch/lineitem");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("region",
"examples/batch/tpch/region");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("partsupp",
"examples/batch/tpch/partsupp");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("customer",
"examples/batch/tpch/customer");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("nation",
"examples/batch/tpch/nation");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("part", "examples/batch/tpch/part");
- TPCH_QUICKSTART_TABLE_RESOURCES.put("supplier",
"examples/batch/tpch/supplier");
- EXEMPT_QUERIES = new HashSet<>();
- // The following queries fail due to lack of support for views.
- EXEMPT_QUERIES.addAll(ImmutableList.of(15, 16, 17));
- }
+
+ // Pinot query 6 fails due to mismatch results.
+ // Pinot queries 15, 16, 17 fail due to lack of support for views.
+ // Pinot queries 23, 24 fail due to java heap space problem or timeout.
+ private static final Set<Integer> EXEMPT_QUERIES = ImmutableSet.of(6, 15,
16, 17, 23, 24);
@BeforeClass
public void setUp()
@@ -76,12 +71,13 @@ public class TPCHQueryIntegrationTest extends
BaseClusterIntegrationTest {
startBroker();
startServer();
- for (Map.Entry<String, String> tableResource :
TPCH_QUICKSTART_TABLE_RESOURCES.entrySet()) {
- File tableSegmentDir = new File(_segmentDir, tableResource.getKey());
- File tarDir = new File(_tarDir, tableResource.getKey());
- String tableName = tableResource.getKey();
- URL resourceUrl =
getClass().getClassLoader().getResource(tableResource.getValue());
- Assert.assertNotNull(resourceUrl, "Unable to find resource from: " +
tableResource.getValue());
+ setUpH2Connection();
+ for (String tableName : Constants.TPCH_TABLE_NAMES) {
+ File tableSegmentDir = new File(_segmentDir, tableName);
+ File tarDir = new File(_tarDir, tableName);
+ String tableResourceFolder = Constants.getTableResourceFolder(tableName);
+ URL resourceUrl =
getClass().getClassLoader().getResource(tableResourceFolder);
+ Assert.assertNotNull(resourceUrl, "Unable to find resource from: " +
tableResourceFolder);
File resourceFile;
if ("jar".equals(resourceUrl.getProtocol())) {
String[] splits = resourceUrl.getFile().split("!");
@@ -92,7 +88,8 @@ public class TPCHQueryIntegrationTest extends
BaseClusterIntegrationTest {
} else {
resourceFile = new File(resourceUrl.getFile());
}
- File dataFile = new File(resourceFile.getAbsolutePath(), "rawdata" +
File.separator + tableName + ".avro");
+ File dataFile =
+ new
File(getClass().getClassLoader().getResource(Constants.getTableAvroFilePath(tableName)).getFile());
Assert.assertTrue(dataFile.exists(), "Unable to load resource file from
URL: " + dataFile);
File schemaFile = new File(resourceFile.getPath(), tableName +
"_schema.json");
File tableFile = new File(resourceFile.getPath(), tableName +
"_offline_table_config.json");
@@ -105,28 +102,53 @@ public class TPCHQueryIntegrationTest extends
BaseClusterIntegrationTest {
ClusterIntegrationTestUtils.buildSegmentsFromAvro(Collections.singletonList(dataFile),
tableConfig, schema, 0,
tableSegmentDir, tarDir);
uploadSegments(tableName, tarDir);
+ // H2
+
ClusterIntegrationTestUtils.setUpH2TableWithAvro(Collections.singletonList(dataFile),
tableName, _h2Connection);
}
}
@Test(dataProvider = "QueryDataProvider")
- public void testTPCHQueries(String query) {
- testQueriesSucceed(query);
+ public void testTPCHQueries(String[] pinotAndH2Queries) throws Exception {
+ testQueriesSucceed(pinotAndH2Queries[0], pinotAndH2Queries[1]);
}
- protected void testQueriesSucceed(String query) {
- ResultSetGroup pinotResultSetGroup = getPinotConnection().execute(query);
+ protected void testQueriesSucceed(String pinotQuery, String h2Query) throws
Exception {
+ ResultSetGroup pinotResultSetGroup =
getPinotConnection().execute(pinotQuery);
org.apache.pinot.client.ResultSet resultTableResultSet =
pinotResultSetGroup.getResultSet(0);
-
if (CollectionUtils.isNotEmpty(pinotResultSetGroup.getExceptions())) {
Assert.fail(String.format(
- "TPC-H query raised exception: %s. query: %s",
pinotResultSetGroup.getExceptions().get(0), query));
+ "TPC-H query raised exception: %s. query: %s",
pinotResultSetGroup.getExceptions().get(0), pinotQuery));
+ }
+
+ int numRows = resultTableResultSet.getRowCount();
+ int numColumns = resultTableResultSet.getColumnCount();
+
+ // h2 response
+ Assert.assertNotNull(_h2Connection);
+ Statement h2statement =
_h2Connection.createStatement(ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
+ h2statement.execute(h2Query);
+ ResultSet h2ResultSet = h2statement.getResultSet();
+
+ // compare results.
+ Assert.assertEquals(numColumns,
h2ResultSet.getMetaData().getColumnCount());
+ if (h2ResultSet.first()) {
+ for (int i = 0; i < numRows; i++) {
+ for (int c = 0; c < numColumns; c++) {
+ String h2Value = h2ResultSet.getString(c + 1);
+ String pinotValue = resultTableResultSet.getString(i, c);
+ boolean error = ClusterIntegrationTestUtils.fuzzyCompare(h2Value,
pinotValue, pinotValue);
+ if (error) {
+ throw new RuntimeException("Value: " + c + " does not match at ("
+ i + ", " + c + "), "
+ + "expected h2 value: " + h2Value + ", actual Pinot value: " +
pinotValue);
+ }
+ }
+ if (!h2ResultSet.next() && i != numRows - 1) {
+ throw new RuntimeException("H2 result set is smaller than Pinot
result set after: " + i + " rows!");
+ }
+ }
}
- // TODO: Enable the following 2 assertions after fixing the data so each
query returns non-zero rows
- /*
- Assert.assertTrue(resultTableResultSet.getRowCount() > 0,
- String.format("Expected non-zero rows for tpc-h query: %s", query));
- Assert.assertTrue(resultTableResultSet.getColumnCount() > 0,
- String.format("Expected non-zero columns for tpc-h query: %s",
query)); */
+ Assert.assertFalse(h2ResultSet.next(), "Pinot result set is smaller than
H2 result set after: "
+ + numRows + " rows!");
}
@Override
@@ -148,7 +170,7 @@ public class TPCHQueryIntegrationTest extends
BaseClusterIntegrationTest {
public void tearDown()
throws Exception {
// unload all TPCH tables.
- for (String table : TPCH_QUICKSTART_TABLE_RESOURCES.keySet()) {
+ for (String table : Constants.TPCH_TABLE_NAMES) {
dropOfflineTable(table);
}
@@ -169,11 +191,19 @@ public class TPCHQueryIntegrationTest extends
BaseClusterIntegrationTest {
if (EXEMPT_QUERIES.contains(query)) {
continue;
}
- String path = String.format("tpch/%s.sql", query);
- try (InputStream inputStream =
TPCHQueryIntegrationTest.class.getClassLoader()
- .getResourceAsStream(path)) {
- queries[iter] = new Object[1];
- queries[iter][0] =
IOUtils.toString(Objects.requireNonNull(inputStream), Charset.defaultCharset());
+ String pinotQueryFilePath = String.format("tpch/%s.sql", query);
+ try (InputStream pinotQueryIs =
TPCHQueryIntegrationTest.class.getClassLoader()
+ .getResourceAsStream(pinotQueryFilePath)) {
+ queries[iter] = new Object[2];
+ queries[iter][0] =
IOUtils.toString(Objects.requireNonNull(pinotQueryIs),
Charset.defaultCharset());
+ // TODO: remove this once we have Pinot support standard sql funtion
extract().
+ String h2QueryFilePath = String.format("tpch/%s-h2.sql", query);
+ try (InputStream h2QueryIs =
TPCHQueryIntegrationTest.class.getClassLoader()
+ .getResourceAsStream(h2QueryFilePath)) {
+ queries[iter][1] =
IOUtils.toString(Objects.requireNonNull(h2QueryIs), Charset.defaultCharset());
+ } catch (Exception e) {
+ queries[iter][1] = queries[iter][0];
+ }
iter++;
}
}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TblToAvro.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TblToAvro.java
new file mode 100644
index 0000000000..b6ccd7dd2f
--- /dev/null
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TblToAvro.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.tpch;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumWriter;
+
+
+/**
+ * A helper class to convert TPC-H .tbl files to Avro format.
+ */
+public final class TblToAvro {
+ private static final String LONG_TYPE = "long";
+ private static final String DOUBLE_TYPE = "double";
+ private static final String STRING_TYPE = "string";
+ private static final String FOLDER_PATH =
"/Users/haitaozhang/Downloads/TPC-H-V3.0.1/dbgen/";
+ private static final String TBL_FILE_SUFFIX = ".tbl";
+ private static final Map<String, String[]> TABLE_NAME_FIELDS_MAP =
+ ImmutableMap.of(
+ // One example record:
+ // {"c_custkey":{"long":1},"c_name":{"string":"Customer#000000001"},
+ // "c_address":{"string":"IVhzIApeRb
ot,c,E"},"c_nationkey":{"long":15},
+ //
"c_phone":{"string":"25-989-741-2988"},"c_acctbal":{"double":711.56},"c_mktsegment":{"string":"BUILDING"},
+ // "c_comment":{"string":"to the even, regular platelets. regular,
ironic epitaphs nag e"}}
+ "customer",
+ new String[]{
+ "c_custkey", LONG_TYPE,
+ "c_name", STRING_TYPE,
+ "c_address", STRING_TYPE,
+ "c_nationkey", LONG_TYPE,
+ "c_phone", STRING_TYPE,
+ "c_acctbal", DOUBLE_TYPE,
+ "c_mktsegment", STRING_TYPE,
+ "c_comment", STRING_TYPE},
+ // One example record:
+ //
{"l_orderkey":{"long":600000},"l_partkey":{"long":12916},"l_suppkey":{"long":917},
+ //
"l_linenumber":{"long":2},"l_quantity":{"long":1},"l_extendedprice":{"double":1828.91},
+ //
"l_discount":{"double":0.03},"l_tax":{"double":0.0},"l_returnflag":{"string":"N"},
+ //
"l_linestatus":{"string":"O"},"l_shipdate":{"string":"1998-04-13"},"l_commitdate":{"string":"1998-05-24"},
+ //
"l_receiptdate":{"string":"1998-04-30"},"l_shipinstruct":{"string":"DELIVER IN
PERSON"},
+ // "l_shipmode":{"string":"RAIL"},"l_comment":{"string":" wake
braids. "}}
+ "lineitem",
+ new String[]{
+ "l_orderkey", LONG_TYPE,
+ "l_partkey", LONG_TYPE,
+ "l_suppkey", LONG_TYPE,
+ "l_linenumber", LONG_TYPE,
+ "l_quantity", LONG_TYPE,
+ "l_extendedprice", DOUBLE_TYPE,
+ "l_discount", DOUBLE_TYPE,
+ "l_tax", DOUBLE_TYPE,
+ "l_returnflag", STRING_TYPE,
+ "l_linestatus", STRING_TYPE,
+ "l_shipdate", STRING_TYPE,
+ "l_commitdate", STRING_TYPE,
+ "l_receiptdate", STRING_TYPE,
+ "l_shipinstruct", STRING_TYPE,
+ "l_shipmode", STRING_TYPE,
+ "l_comment", STRING_TYPE},
+ // One example record:
+ //
{"n_nationkey":{"long":0},"n_name":{"string":"ALGERIA"},"n_regionkey":{"long":0},
+ // "n_comment":{"string":" haggle. carefully final deposits detect
slyly agai"}}
+ "nation",
+ new String[]{
+ "n_nationkey", LONG_TYPE,
+ "n_name", STRING_TYPE,
+ "n_regionkey", LONG_TYPE,
+ "n_comment", STRING_TYPE},
+ // One example record:
+ //
{"o_orderkey":{"long":1},"o_custkey":{"long":370},"o_orderstatus":{"string":"O"},
+ //
"o_totalprice":{"double":173665.47},"o_orderdate":{"string":"1996-01-02"},
+ //
"o_orderpriority":{"string":"5-LOW"},"o_clerk":{"string":"Clerk#000000951"},"o_shippriority":{"long":0},
+ // "o_comment":{"string":"ly final dependencies: slyly bold "}}
+ "orders",
+ new String[]{
+ "o_orderkey", LONG_TYPE,
+ "o_custkey", LONG_TYPE,
+ "o_orderstatus", STRING_TYPE,
+ "o_totalprice", DOUBLE_TYPE,
+ "o_orderdate", STRING_TYPE,
+ "o_orderpriority", STRING_TYPE,
+ "o_clerk", STRING_TYPE,
+ "o_shippriority", LONG_TYPE,
+ "o_comment", STRING_TYPE},
+ // One example record:
+ // {"p_partkey":{"long":1},"p_name":{"string":"goldenrod lavender
spring chocolate lace"},
+ //
"p_mfgr":{"string":"Manufacturer#1"},"p_brand":{"string":"Brand#13"},
+ // "p_type":{"string":"PROMO BRUSHED
STEEL"},"p_size":{"long":7},"p_container":{"string":"JUMBO PKG"},
+ // "p_retailprice":{"double":901.0},"p_comment":{"string":"ly final
dependencies: slyly bold "}}
+ "part",
+ new String[]{
+ "p_partkey", LONG_TYPE,
+ "p_name", STRING_TYPE,
+ "p_mfgr", STRING_TYPE,
+ "p_brand", STRING_TYPE,
+ "p_type", STRING_TYPE,
+ "p_size", LONG_TYPE,
+ "p_container", STRING_TYPE,
+ "p_retailprice", DOUBLE_TYPE,
+ "p_comment", STRING_TYPE},
+ // One example record:
+ //
{"ps_partkey":{"long":1},"ps_suppkey":{"long":2},"ps_availqty":{"long":997},
+ // "ps_supplycost":{"double":6.02},"ps_comment":{"string":"ly final
dependencies: slyly bold "}}
+ "partsupp",
+ new String[]{
+ "ps_partkey", LONG_TYPE,
+ "ps_suppkey", LONG_TYPE,
+ "ps_availqty", LONG_TYPE,
+ "ps_supplycost", DOUBLE_TYPE,
+ "ps_comment", STRING_TYPE},
+ // One example record:
+ //
{"r_regionkey":{"long":0},"r_name":{"string":"AFRICA"},"r_comment":{"string":"lar
deposits. blithe"}}
+ "region",
+ new String[]{
+ "r_regionkey", LONG_TYPE,
+ "r_name", STRING_TYPE,
+ "r_comment", STRING_TYPE},
+ // One example record:
+ // {"s_suppkey":{"long":1},"s_name":{"string":"Supplier#000000001"},
+ // "s_address":{"string":" N kD4on9OM
Ipw3,gf0JBoQDd7tgrzrddZ"},"s_nationkey":{"long":15},
+ //
"s_phone":{"string":"25-989-741-2988"},"s_acctbal":{"double":711.56},
+ // "s_comment":{"string":" deposits eat slyly ironic, even
instructions. express foxes detect slyly.
+ // blithely even accounts abov"}}
+ "supplier",
+ new String[]{
+ "s_suppkey", LONG_TYPE,
+ "s_name", STRING_TYPE,
+ "s_address", STRING_TYPE,
+ "s_nationkey", LONG_TYPE,
+ "s_phone", STRING_TYPE,
+ "s_acctbal", DOUBLE_TYPE,
+ "s_comment", STRING_TYPE});
+
+ private TblToAvro() {
+ }
+
+ public static void main(String[] args) throws IOException {
+// String tpchTblFileFolder = args[0];
+ String tpchTblFileFolder = FOLDER_PATH;
+ for (Map.Entry<String, String[]> entry: TABLE_NAME_FIELDS_MAP.entrySet()) {
+ String tableName = entry.getKey();
+ String[] fieldsAndTypes = entry.getValue();
+ SchemaBuilder.FieldAssembler<Schema> schemaFields =
SchemaBuilder.record(tableName).fields();
+ for (int i = 0; i < fieldsAndTypes.length; i += 2) {
+ String fieldName = fieldsAndTypes[i];
+ String fieldType = fieldsAndTypes[i + 1];
+ switch (fieldType) {
+ case LONG_TYPE:
+
schemaFields.name(fieldName).type().unionOf().longType().and().nullType().endUnion().noDefault();
+ break;
+ case DOUBLE_TYPE:
+
schemaFields.name(fieldName).type().unionOf().doubleType().and().nullType().endUnion().noDefault();
+ break;
+ case STRING_TYPE:
+
schemaFields.name(fieldName).type().unionOf().stringType().and().nullType().endUnion().noDefault();
+ break;
+ default:
+ throw new IllegalStateException("Unsupported field type: " +
fieldType);
+ }
+ }
+ Schema schema = schemaFields.endRecord();
+
+ // Open Avro data file for writing
+ DatumWriter<GenericRecord> datumWriter = new
GenericDatumWriter<>(schema);
+ DataFileWriter<GenericRecord> dataFileWriter = new
DataFileWriter<>(datumWriter);
+ Path avroFilePath = Paths.get(tpchTblFileFolder + tableName +
Constants.AVRO_FILE_SUFFIX);
+ Files.delete(avroFilePath);
+ OutputStream outputStream = Files.newOutputStream(avroFilePath);
+ dataFileWriter.create(schema, outputStream);
+
+ // Read TPC-H .tbl files and convert to Avro format
+ try (BufferedReader reader = new BufferedReader(
+ new FileReader(tpchTblFileFolder + tableName + TBL_FILE_SUFFIX))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ String[] fields = line.split("\\|");
+ GenericRecord record = new GenericData.Record(schema);
+ for (int i = 0; i < fields.length; i++) {
+ String fieldName = fieldsAndTypes[2 * i];
+ String fieldType = fieldsAndTypes[2 * i + 1];
+ switch (fieldType) {
+ case LONG_TYPE:
+ record.put(fieldName, Long.parseLong(fields[i]));
+ break;
+ case DOUBLE_TYPE:
+ record.put(fieldName, Double.parseDouble(fields[i]));
+ break;
+ case STRING_TYPE:
+ record.put(fieldName, fields[i]);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported field type: " +
fieldType);
+ }
+ }
+ dataFileWriter.append(record);
+ }
+ }
+
+ // Close Avro data file
+ dataFileWriter.close();
+ }
+ }
+}
diff --git a/pinot-integration-tests/src/test/resources/tpch/1.sql
b/pinot-integration-tests/src/test/resources/tpch/1.sql
index c43e3122c8..e62b0cc0ce 100644
--- a/pinot-integration-tests/src/test/resources/tpch/1.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/1.sql
@@ -16,7 +16,7 @@ select
from
lineitem
where
- l_shipdate <= 904694400
+ l_shipdate <= '1998-12-01'
group by
l_returnflag,
l_linestatus
diff --git a/pinot-integration-tests/src/test/resources/tpch/10.sql
b/pinot-integration-tests/src/test/resources/tpch/10.sql
index 847d7a67df..6ef51cbc85 100644
--- a/pinot-integration-tests/src/test/resources/tpch/10.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/10.sql
@@ -17,8 +17,8 @@ from
where
c_custkey = o_custkey
and l_orderkey = o_orderkey
- and o_orderdate >= 749413800
- and o_orderdate < 757362600
+ and o_orderdate >= '1993-10-01'
+ and o_orderdate < '1994-01-01'
and l_returnflag = 'R'
and c_nationkey = n_nationkey
group by
diff --git a/pinot-integration-tests/src/test/resources/tpch/11.sql
b/pinot-integration-tests/src/test/resources/tpch/11.sql
index ed0d037b7b..9fb349c3a4 100644
--- a/pinot-integration-tests/src/test/resources/tpch/11.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/11.sql
@@ -1,6 +1,6 @@
select
ps_partkey,
- sum(ps_supplycost * ps_availqty) as value
+ sum(ps_supplycost * ps_availqty) as total_cost
from
partsupp,
supplier,
@@ -25,4 +25,4 @@ having
and n_name = 'GERMANY'
)
order by
- value desc;
+ total_cost desc;
diff --git a/pinot-integration-tests/src/test/resources/tpch/12.sql
b/pinot-integration-tests/src/test/resources/tpch/12.sql
index 4f18a9fb51..772f0e842c 100644
--- a/pinot-integration-tests/src/test/resources/tpch/12.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/12.sql
@@ -16,8 +16,8 @@ where
and l_shipmode in ('MAIL', 'SHIP')
and l_commitdate < l_receiptdate
and l_shipdate < l_commitdate
- and l_receiptdate >= 757362600
- and l_receiptdate < 788898600
+ and l_receiptdate >= '1994-01-01'
+ and l_receiptdate < '1995-01-01'
group by
l_shipmode
order by
diff --git a/pinot-integration-tests/src/test/resources/tpch/14.sql
b/pinot-integration-tests/src/test/resources/tpch/14.sql
index 96cc20eed7..b66a11561b 100644
--- a/pinot-integration-tests/src/test/resources/tpch/14.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/14.sql
@@ -9,5 +9,5 @@ from
part
where
l_partkey = p_partkey
- and l_shipdate >= 809893800
- and l_shipdate < 812485800;
+ and l_shipdate >= '1995-09-01'
+ and l_shipdate < '1995-10-01';
diff --git a/pinot-integration-tests/src/test/resources/tpch/15.sql
b/pinot-integration-tests/src/test/resources/tpch/15.sql
index 33b7ab34eb..a9e6694dca 100644
--- a/pinot-integration-tests/src/test/resources/tpch/15.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/15.sql
@@ -7,7 +7,7 @@ select
from
lineitem
where
- l_shipdate >= 820434600
- and l_shipdate < 828297000
+ l_shipdate >= '1996-01-01'
+ and l_shipdate < '1996-04-01'
group by
l_suppkey;
diff --git a/pinot-integration-tests/src/test/resources/tpch/22.sql
b/pinot-integration-tests/src/test/resources/tpch/22.sql
index 31663daad4..896c092e31 100644
--- a/pinot-integration-tests/src/test/resources/tpch/22.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/22.sql
@@ -27,8 +27,8 @@ where
where
l_partkey = ps_partkey
and l_suppkey = ps_suppkey
- and l_shipdate >= 757362600
- and l_shipdate < 788898600
+ and l_shipdate >= '1994-01-01'
+ and l_shipdate < '1995-01-01'
)
)
and s_nationkey = n_nationkey
diff --git a/pinot-integration-tests/src/test/resources/tpch/3.sql
b/pinot-integration-tests/src/test/resources/tpch/3.sql
index f702c55dc5..95b1e52224 100644
--- a/pinot-integration-tests/src/test/resources/tpch/3.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/3.sql
@@ -13,8 +13,8 @@ where
c_mktsegment = 'BUILDING'
and c_custkey = o_custkey
and l_orderkey = o_orderkey
- and o_orderdate < 795225600
- and l_shipdate > 795225600
+ and o_orderdate < '1995-03-15'
+ and l_shipdate > '1995-03-15'
group by
l_orderkey,
o_orderdate,
diff --git a/pinot-integration-tests/src/test/resources/tpch/4.sql
b/pinot-integration-tests/src/test/resources/tpch/4.sql
index 8c5572cc29..aef0a10d60 100644
--- a/pinot-integration-tests/src/test/resources/tpch/4.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/4.sql
@@ -4,8 +4,8 @@ select
from
orders
where
- o_orderdate >= 741484800
- and o_orderdate < 749433600
+ o_orderdate >= '1993-07-01'
+ and o_orderdate < '1993-10-01'
and exists (
select
*
diff --git a/pinot-integration-tests/src/test/resources/tpch/5.sql
b/pinot-integration-tests/src/test/resources/tpch/5.sql
index 3dbc214431..e6f08da1c7 100644
--- a/pinot-integration-tests/src/test/resources/tpch/5.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/5.sql
@@ -18,8 +18,8 @@ where
and s_nationkey = n_nationkey
and n_regionkey = r_regionkey
and r_name = 'ASIA'
- and o_orderdate >= 757382400
- and o_orderdate < 788918400
+ and o_orderdate >= '1994-01-01'
+ and o_orderdate < '1995-01-01'
group by
n_name
order by
diff --git a/pinot-integration-tests/src/test/resources/tpch/6.sql
b/pinot-integration-tests/src/test/resources/tpch/6.sql
index e29fa408c8..14a3404d7e 100644
--- a/pinot-integration-tests/src/test/resources/tpch/6.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/6.sql
@@ -3,8 +3,8 @@ select
from
lineitem
where
- l_shipdate >= 757382400
- and l_shipdate < 788918400
+ l_shipdate >= '1994-01-01'
+ and l_shipdate < '1995-01-01'
and l_discount between.06 - 0.01
and.06 + 0.01
and l_quantity < 24;
diff --git a/pinot-integration-tests/src/test/resources/tpch/7.sql
b/pinot-integration-tests/src/test/resources/tpch/7-h2.sql
similarity index 87%
copy from pinot-integration-tests/src/test/resources/tpch/7.sql
copy to pinot-integration-tests/src/test/resources/tpch/7-h2.sql
index 852d61f838..f2260da8f7 100644
--- a/pinot-integration-tests/src/test/resources/tpch/7.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/7-h2.sql
@@ -8,7 +8,7 @@ from
select
n1.n_name as supp_nation,
n2.n_name as cust_nation,
- ToDateTime(l_shipdate * 1000, 'yyyy') as l_year,
+ extract(year from l_shipdate) as l_year,
l_extendedprice * (1 - l_discount) as volume
from
supplier,
@@ -33,8 +33,8 @@ from
and n2.n_name = 'FRANCE'
)
)
- and l_shipdate between 788898600
- and 851970600
+ and l_shipdate between '1995-01-01'
+ and '1997-01-01'
) as shipping
group by
supp_nation,
diff --git a/pinot-integration-tests/src/test/resources/tpch/7.sql
b/pinot-integration-tests/src/test/resources/tpch/7.sql
index 852d61f838..512d76ef06 100644
--- a/pinot-integration-tests/src/test/resources/tpch/7.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/7.sql
@@ -8,7 +8,7 @@ from
select
n1.n_name as supp_nation,
n2.n_name as cust_nation,
- ToDateTime(l_shipdate * 1000, 'yyyy') as l_year,
+ substring(l_shipdate, 0, 4) as l_year,
l_extendedprice * (1 - l_discount) as volume
from
supplier,
@@ -33,8 +33,8 @@ from
and n2.n_name = 'FRANCE'
)
)
- and l_shipdate between 788898600
- and 851970600
+ and l_shipdate between '1995-01-01'
+ and '1997-01-01'
) as shipping
group by
supp_nation,
diff --git a/pinot-integration-tests/src/test/resources/tpch/8.sql
b/pinot-integration-tests/src/test/resources/tpch/8-h2.sql
similarity index 86%
copy from pinot-integration-tests/src/test/resources/tpch/8.sql
copy to pinot-integration-tests/src/test/resources/tpch/8-h2.sql
index 8416732df6..eb00f52835 100644
--- a/pinot-integration-tests/src/test/resources/tpch/8.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/8-h2.sql
@@ -6,7 +6,7 @@ select
from
(
select
- ToDateTime(o_orderdate * 1000, 'yyyy') as o_year,
+ extract(year from o_orderdate) as o_year,
l_extendedprice * (1 - l_discount) as volume,
n2.n_name as nation
from
@@ -27,8 +27,8 @@ from
and n1.n_regionkey = r_regionkey
and r_name = 'AMERICA'
and s_nationkey = n2.n_nationkey
- and o_orderdate between 788898600
- and 851970600
+ and o_orderdate between '1995-01-01'
+ and '1997-01-01'
and p_type = 'ECONOMY ANODIZED STEEL'
) as all_nations
group by
diff --git a/pinot-integration-tests/src/test/resources/tpch/8.sql
b/pinot-integration-tests/src/test/resources/tpch/8.sql
index 8416732df6..7a039ad3d1 100644
--- a/pinot-integration-tests/src/test/resources/tpch/8.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/8.sql
@@ -6,7 +6,7 @@ select
from
(
select
- ToDateTime(o_orderdate * 1000, 'yyyy') as o_year,
+ substring(o_orderdate, 0, 4) as o_year,
l_extendedprice * (1 - l_discount) as volume,
n2.n_name as nation
from
@@ -27,8 +27,8 @@ from
and n1.n_regionkey = r_regionkey
and r_name = 'AMERICA'
and s_nationkey = n2.n_nationkey
- and o_orderdate between 788898600
- and 851970600
+ and o_orderdate between '1995-01-01'
+ and '1997-01-01'
and p_type = 'ECONOMY ANODIZED STEEL'
) as all_nations
group by
diff --git a/pinot-integration-tests/src/test/resources/tpch/9.sql
b/pinot-integration-tests/src/test/resources/tpch/9-h2.sql
similarity index 91%
copy from pinot-integration-tests/src/test/resources/tpch/9.sql
copy to pinot-integration-tests/src/test/resources/tpch/9-h2.sql
index 84ca85dcdc..2d91725b18 100644
--- a/pinot-integration-tests/src/test/resources/tpch/9.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/9-h2.sql
@@ -6,7 +6,7 @@ from
(
select
n_name as nation,
- ToDateTime(o_orderdate * 1000, 'yyyy') as o_year,
+ extract(year from o_orderdate) as o_year,
l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount
from
part,
diff --git a/pinot-integration-tests/src/test/resources/tpch/9.sql
b/pinot-integration-tests/src/test/resources/tpch/9.sql
index 84ca85dcdc..d7541ae695 100644
--- a/pinot-integration-tests/src/test/resources/tpch/9.sql
+++ b/pinot-integration-tests/src/test/resources/tpch/9.sql
@@ -6,7 +6,7 @@ from
(
select
n_name as nation,
- ToDateTime(o_orderdate * 1000, 'yyyy') as o_year,
+ substring(o_orderdate, 0, 4) as o_year,
l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount
from
part,
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/customer/rawdata/customer.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/customer/rawdata/customer.avro
index f446eef075..96f38a4bff 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/customer/rawdata/customer.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/customer/rawdata/customer.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/lineitem_schema.json
b/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/lineitem_schema.json
index 4a2ae474cf..4e1ffb6594 100644
---
a/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/lineitem_schema.json
+++
b/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/lineitem_schema.json
@@ -44,15 +44,15 @@
"name": "l_linestatus"
},
{
- "dataType": "LONG",
+ "dataType": "STRING",
"name": "l_shipdate"
},
{
- "dataType": "LONG",
+ "dataType": "STRING",
"name": "l_commitdate"
},
{
- "dataType": "LONG",
+ "dataType": "STRING",
"name": "l_receiptdate"
},
{
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/rawdata/lineitem.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/rawdata/lineitem.avro
index fa68458255..8d7360c2fc 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/rawdata/lineitem.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/lineitem/rawdata/lineitem.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/nation/rawdata/nation.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/nation/rawdata/nation.avro
index 12cb1fc805..b926c8ea9a 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/nation/rawdata/nation.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/nation/rawdata/nation.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/orders/orders_schema.json
b/pinot-tools/src/main/resources/examples/batch/tpch/orders/orders_schema.json
index 6efde76bed..e2198a3773 100644
---
a/pinot-tools/src/main/resources/examples/batch/tpch/orders/orders_schema.json
+++
b/pinot-tools/src/main/resources/examples/batch/tpch/orders/orders_schema.json
@@ -20,7 +20,7 @@
"name": "o_orderstatus"
},
{
- "dataType": "LONG",
+ "dataType": "STRING",
"name": "o_orderdate"
},
{
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/orders/rawdata/orders.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/orders/rawdata/orders.avro
index 003f91eb20..06c55ae8ca 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/orders/rawdata/orders.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/orders/rawdata/orders.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/part/rawdata/part.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/part/rawdata/part.avro
index ae74dfa289..26ad2c318a 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/part/rawdata/part.avro and
b/pinot-tools/src/main/resources/examples/batch/tpch/part/rawdata/part.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/partsupp/rawdata/partsupp.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/partsupp/rawdata/partsupp.avro
index 83e846fba2..a000b3f2cb 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/partsupp/rawdata/partsupp.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/partsupp/rawdata/partsupp.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/region/rawdata/region.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/region/rawdata/region.avro
index f8ac435feb..6e689a3bf5 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/region/rawdata/region.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/region/rawdata/region.avro
differ
diff --git
a/pinot-tools/src/main/resources/examples/batch/tpch/supplier/rawdata/supplier.avro
b/pinot-tools/src/main/resources/examples/batch/tpch/supplier/rawdata/supplier.avro
index 0961696e69..d3c52f0ea6 100644
Binary files
a/pinot-tools/src/main/resources/examples/batch/tpch/supplier/rawdata/supplier.avro
and
b/pinot-tools/src/main/resources/examples/batch/tpch/supplier/rawdata/supplier.avro
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
