This is an automated email from the ASF dual-hosted git repository.
shahrs87 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 1f88ca83f0 PHOENIX-7309 Support specifying splits.txt file while
creating a table (#1931)
1f88ca83f0 is described below
commit 1f88ca83f0efb576f01bdffb35becdc5ebccf58a
Author: Rushabh Shah <[email protected]>
AuthorDate: Tue Jul 23 09:29:03 2024 -0700
PHOENIX-7309 Support specifying splits.txt file while creating a table
(#1931)
---
.../apache/phoenix/exception/SQLExceptionCode.java | 5 +-
.../org/apache/phoenix/query/QueryConstants.java | 1 +
.../org/apache/phoenix/schema/MetaDataClient.java | 54 +++++++++++
.../org/apache/phoenix/end2end/CreateTableIT.java | 108 +++++++++++++++++++++
4 files changed, 167 insertions(+), 1 deletion(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index d36827306a..24abfc4bb4 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -485,7 +485,10 @@ public enum SQLExceptionCode {
"Missing ENCODED_QUALIFIER."),
EXECUTE_BATCH_FOR_STMT_WITH_RESULT_SET(1151, "XCL51", "A batch operation
can't include a "
+ "statement that produces result sets.",
Factory.BATCH_UPDATE_ERROR),
-
+ SPLITS_AND_SPLIT_FILE_EXISTS(1152, "XCL52", "Both splits and split file
are passed"),
+ // 1153 code is taken by CANNOT_DROP_CDC_INDEX
+ SPLIT_FILE_DONT_EXIST(1154, "XCL54", "Either split file don't exist or is
not a file"),
+ UNABLE_TO_OPEN_SPLIT_FILE(1155, "XCL55", "Exception occurred while opening
splits file"),
/**
* Implementation defined class. Phoenix internal error. (errorcode 20,
sqlstate INT).
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 39332d9e66..aac042b4e0 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -344,6 +344,7 @@ public interface QueryConstants {
String CDC_CHANGE_IMAGE = "change_image";
String CDC_UPSERT_EVENT_TYPE = "upsert";
String CDC_DELETE_EVENT_TYPE = "delete";
+ String SPLITS_FILE = "SPLITS_FILE";
/**
* We mark counter values 0 to 10 as reserved. Value 0 is used by
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 672deb5397..fb7b3a9595 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -25,6 +25,7 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_NOT_DEFINED;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STREAMING_TOPIC_NAME;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL;
+import static org.apache.phoenix.query.QueryConstants.SPLITS_FILE;
import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
import static
org.apache.phoenix.query.QueryServices.INDEX_CREATE_DEFAULT_STATE;
import static org.apache.phoenix.schema.PTableType.CDC;
@@ -134,8 +135,13 @@ import static org.apache.phoenix.schema.PTableType.VIEW;
import static org.apache.phoenix.schema.types.PDataType.FALSE_BYTES;
import static org.apache.phoenix.schema.types.PDataType.TRUE_BYTES;
+import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
@@ -1053,6 +1059,7 @@ public class MetaDataClient {
Map<String,Object> commonFamilyProps =
Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1);
populatePropertyMaps(statement.getProps(), tableProps,
commonFamilyProps, statement.getTableType());
+ splits = processSplits(tableProps, splits);
boolean isAppendOnlySchema = false;
long updateCacheFrequency = (Long)
ConnectionProperty.UPDATE_CACHE_FREQUENCY.getValue(
connection.getQueryServices().getProps().get(
@@ -1150,6 +1157,53 @@ public class MetaDataClient {
return connection.getQueryServices().updateData(plan);
}
+ /*
+ Create splits either from the provided splits or reading from
SPLITS_FILE.
+ */
+ private byte[][] processSplits(Map<String, Object> tableProperties,
byte[][] splits)
+ throws SQLException {
+ String splitFilesLocation = (String) tableProperties.get(SPLITS_FILE);
+ if (splitFilesLocation == null || splitFilesLocation.isEmpty()) {
+ splitFilesLocation = null;
+ }
+
+ // Both splits and split file location are not passed, so return empty
split.
+ if (splits.length == 0 && splitFilesLocation == null) {
+ return splits;
+ }
+
+ // Both splits[] and splitFileLocation are provided. Throw an
exception in this case.
+ if (splits.length != 0 && splitFilesLocation != null) {
+ throw new
SQLExceptionInfo.Builder(SQLExceptionCode.SPLITS_AND_SPLIT_FILE_EXISTS)
+ .build().buildException();
+ }
+
+ // This means we only have splits[] and no split file location is
specified
+ if (splitFilesLocation == null) {
+ return splits;
+ }
+ // This means splits[] is empty and split file location is not null.
+ File splitFile = new File(splitFilesLocation);
+ // Check if file exists and is a file not a directory.
+ if (!splitFile.exists() || !splitFile.isFile()) {
+ throw new
SQLExceptionInfo.Builder(SQLExceptionCode.SPLIT_FILE_DONT_EXIST)
+ .build().buildException();
+ }
+ List<byte[]> splitsListFromFile = new ArrayList<>();
+ Path path = Paths.get(splitFilesLocation);
+ try (BufferedReader reader = Files.newBufferedReader(path)) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ splitsListFromFile.add(Bytes.toBytes(line));
+ }
+ } catch (IOException ioe) {
+ LOGGER.warn("Exception while reading splits file", ioe);
+ throw new
SQLExceptionInfo.Builder(SQLExceptionCode.UNABLE_TO_OPEN_SPLIT_FILE)
+ .build().buildException();
+ }
+ return splitsListFromFile.toArray(new
byte[splitsListFromFile.size()][]);
+ }
+
/**
* Populate properties for the table and common properties for all column
families of the table
* @param statementProps Properties specified in SQL statement
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 9d28654dca..879539473d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -30,6 +30,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertThrows;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -90,6 +93,7 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -129,6 +133,110 @@ public class CreateTableIT extends
ParallelStatsDisabledIT {
assertTrue(splits.size() > 0);
}
+ @Test
+ public void testSplitsWithFile() throws Exception {
+ File splitFile = new File("splitFile.txt");
+ try {
+ try (BufferedWriter writer = new BufferedWriter(new
FileWriter(splitFile))) {
+ writer.write("EA");
+ writer.newLine();
+ writer.write("EZ");
+ }
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ conn.createStatement().execute("CREATE TABLE " + tableName
+ + " (pk char(2) not null primary key)
SPLITS_FILE='splitFile.txt'");
+ conn.close();
+ String query = "select * from " + tableName;
+ conn = DriverManager.getConnection(getUrl(), props);
+ Statement statement = conn.createStatement();
+ statement.execute(query);
+ PhoenixStatement pstatement =
statement.unwrap(PhoenixStatement.class);
+ List<KeyRange> splits = pstatement.getQueryPlan().getSplits();
+ // There will be 3 region splits: '' - EA, EA - EZ, EZ - ''
+ assertEquals(3, splits.size());
+ } finally {
+ // Delete split file.
+ splitFile.delete();
+ }
+ }
+
+ /**
+ * Pass the absolute path of the splits file while creating the table.
+ * @throws Exception
+ */
+ @Test
+ public void testSplitsWithAbsoluteFileName() throws Exception {
+ File splitFile = new File("splitFile.txt");
+ try {
+ try (BufferedWriter writer = new BufferedWriter(new
FileWriter(splitFile))) {
+ writer.write("EA");
+ writer.newLine();
+ writer.write("EZ");
+ }
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String createTableSql = "CREATE TABLE " + tableName
+ + " (pk char(2) not null primary key) SPLITS_FILE='" +
splitFile.getAbsolutePath() + "'";
+ conn.createStatement().execute(createTableSql);
+ conn.close();
+ String query = "select * from " + tableName;
+ conn = DriverManager.getConnection(getUrl(), props);
+ Statement statement = conn.createStatement();
+ statement.execute(query);
+ PhoenixStatement pstatement =
statement.unwrap(PhoenixStatement.class);
+ List<KeyRange> splits = pstatement.getQueryPlan().getSplits();
+ // There will be 3 region splits: '' - EA, EA - EZ, EZ - ''
+ assertEquals(3, splits.size());
+ } finally {
+ // Delete split file.
+ splitFile.delete();
+ }
+ }
+
+ /**
+ * Test create table fails with an invalid file name.
+ * @throws Exception
+ */
+ @Test
+ public void testSplitsWithBadFileName() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ try {
+ conn.createStatement().execute("CREATE TABLE " + tableName
+ + " (pk char(2) not null primary key)
SPLITS_FILE='bad-split-file.txt'");
+ Assert.fail("Shouldn't come here");
+ } catch (SQLException e) {
+
assertEquals(SQLExceptionCode.SPLIT_FILE_DONT_EXIST.getErrorCode(),
e.getErrorCode());
+ } finally {
+ conn.close();
+ }
+ }
+
+ /**
+ * Test create table fails when both split file and split points are
provided.
+ * @throws Exception
+ */
+ @Test
+ public void testSplitsWithBothSplitPointsAndSplitFileProvided() throws
Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ try {
+ conn.createStatement().execute("CREATE TABLE " + tableName
+ + " (pk char(2) not null primary key)
SPLITS_FILE='bad-split-file.txt'" +
+ " SPLIT ON ('EA','EZ')" );
+ Assert.fail("Shouldn't come here");
+ } catch (SQLException e) {
+
assertEquals(SQLExceptionCode.SPLITS_AND_SPLIT_FILE_EXISTS.getErrorCode(),
e.getErrorCode());
+ } finally {
+ conn.close();
+ }
+ }
+
@Test
public void testCreateAlterTableWithDuplicateColumn() throws Exception {
Properties props = new Properties();
