This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch 5.1 in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.1 by this push: new 54206d88a1 PHOENIX-6889 Improve extraction of ENCODED_QUALIFIERs 54206d88a1 is described below commit 54206d88a143b329ff047856127bcb8ef22569f6 Author: Aron Meszaros <meszaros.aron.att...@gmail.com> AuthorDate: Tue Feb 28 16:43:36 2023 +0100 PHOENIX-6889 Improve extraction of ENCODED_QUALIFIERs --- .../apache/phoenix/end2end/ShowCreateTableIT.java | 126 ++++++++++++++++---- .../schema/tool/SchemaToolExtractionIT.java | 10 +- .../schema/tool/SchemaExtractionProcessor.java | 128 +++++++++++++++------ 3 files changed, 202 insertions(+), 62 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java index 9b8619bf36..b9037e3b85 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java @@ -45,9 +45,7 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + - "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + - ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; + "INT INTEGER, INT2 INTEGER) IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } @@ -64,8 +62,7 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); String expected = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " + - "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; + "INT INTEGER) IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } @@ -84,8 +81,7 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); String expected = "CREATE TABLE " + tableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, " + - "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; + "INT INTEGER) IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } @@ -108,14 +104,42 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableName ); assertTrue(rs.next()); + String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "\"dF\".INT1 INTEGER, \"dF\".INT2 INTEGER, A.INT3 INTEGER, A.INT4 INTEGER, " + + "\"b\".INT5 INTEGER, B.INT6 INTEGER) " + + "IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY='dF'"; + assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), + rs.getString(1).equals(expected)); + } + + @Test + public void testShowCreateTableDefaultFamilyNonConsecutive() throws Exception { + Properties props = new Properties(); + Connection conn = DriverManager.getConnection(getUrl(), props); + String tableName = generateUniqueName(); + String ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " + + "INT1 INTEGER, " + + "INT2 INTEGER, " + + "a.INT3 INTEGER, " + + "\"A\".INT4 INTEGER, " + + "\"b\".INT5 INTEGER, " + + "\"B\".INT6 INTEGER) " + + "DEFAULT_COLUMN_FAMILY='dF'"; + conn.createStatement().execute(ddl); + + String dropInt2 = "ALTER TABLE " + tableName + " DROP COLUMN INT2, INT4, INT5"; + conn.createStatement().execute(dropInt2); + + ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableName ); + assertTrue(rs.next()); + String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + "\"dF\".INT1 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", \"dF\".INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + ", A.INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", A.INT4 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + - ", \"b\".INT5 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", B.INT6 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ") IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY='dF'"; + ", B.INT6 INTEGER) " + + "IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY='dF' COLUMN_QUALIFIER_COUNTER " + + "('A'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + + ", 'dF'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + ")"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } @@ -139,10 +163,10 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { } @Test - public void testShowCreateTableColumnQualifierDrop() throws Exception { + public void testShowCreateTableColumnQualifierNonConsecutive() throws Exception { Properties props = new Properties(); Connection conn = DriverManager.getConnection(getUrl(), props); - String tableName = generateUniqueName();; + String tableName = generateUniqueName(); String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + "INT INTEGER, INT2 INTEGER, INT3 INTEGER)"; conn.createStatement().execute(ddl); @@ -160,6 +184,42 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { "('" + DEFAULT_COLUMN_FAMILY +"'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) + ")"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); + + tableName = generateUniqueName(); + ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "INT INTEGER, INT2 INTEGER, INT3 INTEGER)"; + conn.createStatement().execute(ddl); + + dropInt2 = "ALTER TABLE " + tableName + " DROP COLUMN INT"; + conn.createStatement().execute(dropInt2); + + rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\""); + assertTrue(rs.next()); + + expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + + ", INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + + ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; + assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), + rs.getString(1).equals(expected)); + + tableName = generateUniqueName(); + ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "INT INTEGER, INT2 INTEGER, INT3 INTEGER)"; + conn.createStatement().execute(ddl); + + dropInt2 = "ALTER TABLE " + tableName + " DROP COLUMN INT2"; + conn.createStatement().execute(dropInt2); + + rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\""); + assertTrue(rs.next()); + + expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + + ", INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + + ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'"; + assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), + rs.getString(1).equals(expected)); } @Test @@ -202,15 +262,13 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + - "A.INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", B.INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ") IMMUTABLE_ROWS=true"; + "A.INT INTEGER, B.INT2 INTEGER) IMMUTABLE_ROWS=true"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } @Test - public void testShowCreateTableColumnQualifierMultipleFamiliesDrop() throws Exception { + public void testShowCreateTableColumnQualifierMultipleFamiliesNonConsecutive() throws Exception { Properties props = new Properties(); Connection conn = DriverManager.getConnection(getUrl(), props); String tableName = generateUniqueName();; @@ -220,18 +278,38 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT { "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; conn.createStatement().execute(ddl); - String dropInt3 = "ALTER TABLE " + tableName + " DROP COLUMN INT4"; - conn.createStatement().execute(dropInt3); + String dropInt = "ALTER TABLE " + tableName + " DROP COLUMN INT4"; + conn.createStatement().execute(dropInt); ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\""); assertTrue(rs.next()); String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + + "A.INT INTEGER, A.INT2 INTEGER, B.INT3 INTEGER ENCODED_QUALIFIER " + + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + ") IMMUTABLE_ROWS=true " + + "COLUMN_QUALIFIER_COUNTER ('B'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + ")"; + assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), + rs.getString(1).equals(expected)); + + + tableName = generateUniqueName();; + ddl = "CREATE IMMUTABLE TABLE " + tableName + + "(K VARCHAR NOT NULL PRIMARY KEY, A.INT INTEGER, A.INT2 INTEGER, " + + "B.INT3 INTEGER, B.INT4 INTEGER) " + + "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; + conn.createStatement().execute(ddl); + + dropInt = "ALTER TABLE " + tableName + " DROP COLUMN INT2, INT3"; + conn.createStatement().execute(dropInt); + + rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\""); + assertTrue(rs.next()); + + expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " + "A.INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ", A.INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + - ", B.INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) + - ") IMMUTABLE_ROWS=true COLUMN_QUALIFIER_COUNTER ('B'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) - + ")"; + ", B.INT4 INTEGER ENCODED_QUALIFIER " + + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) + ") IMMUTABLE_ROWS=true " + + "COLUMN_QUALIFIER_COUNTER ('A'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) + ")"; assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1), rs.getString(1).equals(expected)); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java index d4a2aaa2d3..b4bd32d734 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java @@ -66,8 +66,7 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String schemaName = generateUniqueName(); String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName); String createTableStmt = "CREATE TABLE "+ pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, " - + "V1 VARCHAR ENCODED_QUALIFIER 11, V2 VARCHAR ENCODED_QUALIFIER 12) " + - "TTL=2592000, IMMUTABLE_ROWS=TRUE, DISABLE_WAL=TRUE"; + + "V1 VARCHAR, V2 VARCHAR) TTL=2592000, IMMUTABLE_ROWS=TRUE, DISABLE_WAL=TRUE"; List<String> queries = new ArrayList<String>(){}; queries.add(createTableStmt); String result = runSchemaExtractionTool(schemaName, tableName, null, queries); @@ -80,8 +79,7 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String schemaName = "lowecaseschemaname1"; String pTableFullName = SchemaUtil.getEscapedTableName(schemaName, tableName); String createTableStmt = "CREATE TABLE "+ pTableFullName + "(\"smallK\" VARCHAR NOT NULL PRIMARY KEY, " - + "\"asd\".V1 VARCHAR ENCODED_QUALIFIER 11, \"foo\".\"bar\" VARCHAR ENCODED_QUALIFIER 11) " + - "TTL=2592000, IMMUTABLE_ROWS=true, DISABLE_WAL=true"; + + "\"asd\".V1 VARCHAR, \"foo\".\"bar\" VARCHAR) TTL=2592000, IMMUTABLE_ROWS=true, DISABLE_WAL=true"; List<String> queries = new ArrayList<String>(){}; queries.add(createTableStmt); String result = runSchemaExtractionTool("\"" + schemaName + "\"", "\"" + tableName + "\"", null, queries); @@ -418,8 +416,8 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String query = "create table " + pTableFullName + "(a_char CHAR(15) NOT NULL, " + "b_char CHAR(10) NOT NULL, " + - "c_var_array VARCHAR ARRAY ENCODED_QUALIFIER 11, " + - "d_char_array CHAR(15) ARRAY[3] ENCODED_QUALIFIER 12 " + + "c_var_array VARCHAR ARRAY, " + + "d_char_array CHAR(15) ARRAY[3] " + "CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + "TTL=2592000, IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', REPLICATION_SCOPE=1"; List<String> queries = new ArrayList<String>(){}; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java index 4d2297c927..8af7cbf3b4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java @@ -30,16 +30,17 @@ import org.apache.phoenix.mapreduce.util.ConnectionUtil; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.SortOrder; -import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; +import java.util.Collections; import java.util.HashSet; import java.util.HashMap; import java.util.Map; @@ -47,6 +48,7 @@ import java.util.Set; import java.util.List; import java.util.ArrayList; import java.util.Arrays; +import java.util.stream.Collectors; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; @@ -92,11 +94,11 @@ public class SchemaExtractionProcessor implements SchemaProcessor { if (ddl != null) { return ddl; } - if(this.table.getType().equals(PTableType.TABLE)) { + if (this.table.getType().equals(PTableType.TABLE)) { ddl = extractCreateTableDDL(this.table); - } else if(this.table.getType().equals(PTableType.INDEX)) { + } else if (this.table.getType().equals(PTableType.INDEX)) { ddl = extractCreateIndexDDL(this.table); - } else if(this.table.getType().equals(PTableType.VIEW)) { + } else if (this.table.getType().equals(PTableType.VIEW)) { ddl = extractCreateViewDDL(this.table); } return ddl; @@ -148,7 +150,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { indexPKName.add(indexColumn); indexSortOrderMap.put(indexColumn, indexedColumn.getSortOrder()); } - for(PColumn pColumn : dataPK) { + for (PColumn pColumn : dataPK) { dataPKName.add(pColumn.getName().getString()); } @@ -159,11 +161,11 @@ public class SchemaExtractionProcessor implements SchemaProcessor { } for (String column : indexPKName) { - if(indexedColumnsBuilder.length()!=0) { + if (indexedColumnsBuilder.length()!=0) { indexedColumnsBuilder.append(", "); } indexedColumnsBuilder.append(column); - if(indexSortOrderMap.containsKey(column) + if (indexSortOrderMap.containsKey(column) && indexSortOrderMap.get(column) != SortOrder.getDefault()) { indexedColumnsBuilder.append(" "); indexedColumnsBuilder.append(indexSortOrderMap.get(column)); @@ -174,14 +176,14 @@ public class SchemaExtractionProcessor implements SchemaProcessor { private List<PColumn> getSymmetricDifferencePColumns(List<PColumn> firstList, List<PColumn> secondList) { List<PColumn> effectivePK = new ArrayList<>(); - for(PColumn column : firstList) { - if(secondList.contains(column)) { + for (PColumn column : firstList) { + if (secondList.contains(column)) { continue; } effectivePK.add(column); } - for(PColumn column : secondList) { - if(firstList.contains(column)) { + for (PColumn column : secondList) { + if (firstList.contains(column)) { continue; } effectivePK.add(column); @@ -194,7 +196,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { return null; } String [] columnNameSplit = columnName.split(":"); - if(columnNameSplit[0].equals("") || columnNameSplit[0].equalsIgnoreCase(defaultCF) || + if (columnNameSplit[0].equals("") || columnNameSplit[0].equalsIgnoreCase(defaultCF) || (defaultCF.startsWith("L#") && columnNameSplit[0].equalsIgnoreCase(defaultCF.substring(2)))) { return formatColumnOrExpression(columnNameSplit[1]); } else { @@ -225,10 +227,10 @@ public class SchemaExtractionProcessor implements SchemaProcessor { StringBuilder coveredColumnsBuilder = new StringBuilder(); List<PColumn> pkColumns = indexPTable.getColumns(); for (PColumn cc : pkColumns) { - if(coveredColumnsBuilder.length()!=0) { + if (coveredColumnsBuilder.length()!=0) { coveredColumnsBuilder.append(", "); } - if(cc.getFamilyName()!=null) { + if (cc.getFamilyName()!=null) { String indexColumn = extractIndexColumn(cc.getName().getString(), defaultCF); if (indexColumn != null) { coveredColumnsBuilder.append(indexColumn); @@ -245,7 +247,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { outputBuilder.append("("); outputBuilder.append(indexedColumnString); outputBuilder.append(")"); - if(!coveredColumnString.equals("")) { + if (!coveredColumnString.equals("")) { outputBuilder.append(" INCLUDE ("); outputBuilder.append(coveredColumnString); outputBuilder.append(")"); @@ -272,7 +274,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { String columnInfoString = getColumnInfoStringForView(table, baseTable); String whereClause = table.getViewStatement(); - if(whereClause != null) { + if (whereClause != null) { whereClause = whereClause.substring(whereClause.indexOf("WHERE")); } return generateCreateViewDDL(columnInfoString, quotedBaseTableName, @@ -329,7 +331,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { if (key.equalsIgnoreCase(HColumnDescriptor.COMPRESSION)) { defaultProps.put(key, "NONE"); } - if(key.equalsIgnoreCase(HColumnDescriptor.DATA_BLOCK_ENCODING)) { + if (key.equalsIgnoreCase(HColumnDescriptor.DATA_BLOCK_ENCODING)) { defaultProps.put(key, String.valueOf(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING)); } } @@ -341,7 +343,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { for (Map.Entry<Bytes, Bytes> entry : propsMap.entrySet()) { Bytes key = entry.getKey(); Bytes value = entry.getValue(); - if(Bytes.toString(key.get()).contains("coprocessor") || Bytes.toString(key.get()).contains( + if (Bytes.toString(key.get()).contains("coprocessor") || Bytes.toString(key.get()).contains( HTableDescriptor.IS_META)) { continue; } @@ -352,7 +354,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { private void setHColumnFamilyProperties(HColumnDescriptor[] columnDescriptors) { Map<Bytes, Bytes> propsMap = columnDescriptors[0].getValues(); - for(Map.Entry<Bytes, Bytes> entry : propsMap.entrySet()) { + for (Map.Entry<Bytes, Bytes> entry : propsMap.entrySet()) { Bytes key = entry.getKey(); Bytes globalValue = entry.getValue(); Map<String, String> cfToPropertyValueMap = new HashMap<String, String>(); @@ -369,7 +371,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { cfPropertyValueSet.add(value); } if (cfPropertyValueSet.size() > 1) { - for(Map.Entry<String, String> mapEntry: cfToPropertyValueMap.entrySet()) { + for (Map.Entry<String, String> mapEntry: cfToPropertyValueMap.entrySet()) { definedProps.put(String.format("%s.%s", mapEntry.getKey(), Bytes.toString(key.get())), mapEntry.getValue()); } } else { @@ -380,10 +382,10 @@ public class SchemaExtractionProcessor implements SchemaProcessor { private void setPTableProperties(PTable table) { Map <String, String> map = table.getPropertyValues(); - for(Map.Entry<String, String> entry : map.entrySet()) { + for (Map.Entry<String, String> entry : map.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - if(value != null) { + if (value != null) { definedProps.put(key, value); } } @@ -429,7 +431,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { private String convertPropertiesToString() { StringBuilder optionBuilder = new StringBuilder(); - for(Map.Entry<String, String> entry : definedProps.entrySet()) { + for (Map.Entry<String, String> entry : definedProps.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); String columnFamilyName = QueryConstants.DEFAULT_COLUMN_FAMILY; @@ -442,7 +444,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { key = colPropKey[1]; } - if(value!=null && (shouldGenerateWithDefaults || (defaultProps.get(key) != null && !value.equals(defaultProps.get(key))))) { + if (value!=null && (shouldGenerateWithDefaults || (defaultProps.get(key) != null && !value.equals(defaultProps.get(key))))) { if (optionBuilder.length() != 0) { optionBuilder.append(", "); } @@ -450,7 +452,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { key : String.format("\"%s\".%s", columnFamilyName, key); // properties value that corresponds to a number will not need single quotes around it // properties value that corresponds to a boolean value will not need single quotes around it - if(!(NumberUtils.isNumber(value)) && + if (!(NumberUtils.isNumber(value)) && !(value.equalsIgnoreCase(Boolean.TRUE.toString()) ||value.equalsIgnoreCase(Boolean.FALSE.toString()))) { value= "'" + value + "'"; } @@ -472,7 +474,7 @@ public class SchemaExtractionProcessor implements SchemaProcessor { } public Connection getConnection() throws SQLException { - if(tenantId!=null) { + if (tenantId!=null) { conf.set(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); } return ConnectionUtil.getInputConnection(conf); @@ -486,25 +488,87 @@ public class SchemaExtractionProcessor implements SchemaProcessor { return getColumnInfoString(table, colInfo, columns, pkColumns); } - private boolean shouldContainQualifier(PTable table, List<PColumn> columns) + private boolean hasEncodedQualifier(PTable table) { - if (columns.size() == 0 || shouldGenerateWithDefaults) { + return table.getColumns().size() > 0 + && !shouldGenerateWithDefaults + && table.getType() == PTableType.TABLE + && table.getEncodingScheme() != PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + } + + private boolean areEncodedIdsComplete(List<Integer> encodedIds, Integer initialID, + Integer lastEncodedID) { + if (encodedIds.size() == 0) { + return true; + } + if (encodedIds.get(0) > initialID || + encodedIds.get(encodedIds.size() - 1) < lastEncodedID) { return false; } - - return table.getType() == PTableType.TABLE && table.getEncodingScheme() != PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + for (int i = 1; i < encodedIds.size(); i++) { + if (encodedIds.get(i - 1) + 1 != encodedIds.get(i)) { + return false; + } + } + return true; + } + + private List<String> getNonConsecutiveQualifierFamilies(PTable table) { + List<String> ret = new ArrayList<>(); + if (!hasEncodedQualifier(table)) { + return ret; + } + + PTable.QualifierEncodingScheme scheme = table.getEncodingScheme(); + PTable.EncodedCQCounter encodedCQCounter = table.getEncodedCQCounter(); + if (table.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS) { + // For this scheme we track column qualifier counters at the column family level + for (PColumnFamily colFamily : table.getColumnFamilies()) { + String colFamilyName = colFamily.getName().getString(); + List<Integer> encodedIds = colFamily.getColumns().stream() + .filter(c -> !table.getPKColumns().contains(c)) + .map(pColumn -> scheme.decode(pColumn.getColumnQualifierBytes())) + .collect(Collectors.toList()); + Collections.sort(encodedIds); + if (!areEncodedIdsComplete(encodedIds, + QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE, + encodedCQCounter.getNextQualifier(colFamilyName) - 1)) { + ret.add(colFamilyName); + } + } + } else { + // For other schemes, column qualifier counters are tracked using the default column + // family. + List<Integer> encodedIds = table.getColumns().stream() + .filter(c -> !table.getPKColumns().contains(c)) + .map(pColumn -> scheme.decode(pColumn.getColumnQualifierBytes())) + .collect(Collectors.toList()); + Collections.sort(encodedIds); + String defaultFamilyName = table.getDefaultFamilyName() == null ? + QueryConstants.DEFAULT_COLUMN_FAMILY + : table.getDefaultFamilyName().getString(); + if (!areEncodedIdsComplete(encodedIds, + QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE, + encodedCQCounter.getNextQualifier(defaultFamilyName) - 1)) { + ret = table.getColumnFamilies().stream() + .map(pColumnFamily -> pColumnFamily.getName().getString()) + .collect(Collectors.toList()); + } + } + return ret; } private String getColumnInfoString(PTable table, StringBuilder colInfo, List<PColumn> columns, List<PColumn> pkColumns) { - boolean appendQualifier = shouldContainQualifier(table, columns); + List<String> nonConsecutiveCounterFamilies = getNonConsecutiveQualifierFamilies(table); ArrayList<String> colDefs = new ArrayList<>(columns.size()); for (PColumn col : columns) { String def = extractColumn(col); if (pkColumns.size() == 1 && pkColumns.contains(col)) { def += " PRIMARY KEY" + extractPKColumnAttributes(col); } - if (appendQualifier && !pkColumns.contains(col)) { + if (!pkColumns.contains(col) + && nonConsecutiveCounterFamilies.contains(col.getFamilyName().getString())) { def += " ENCODED_QUALIFIER " + table.getEncodingScheme().decode(col.getColumnQualifierBytes()); }