This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new 59e52407b7 PHOENIX-6560 Fix Spotbugs security errors with code SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE 59e52407b7 is described below commit 59e52407b71619ef6bad455cfaded4fce57d187b Author: Abhishek Kothalikar <kabhis...@cloudera.com> AuthorDate: Thu Mar 23 11:43:52 2023 +0530 PHOENIX-6560 Fix Spotbugs security errors with code SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE --- .../apache/phoenix/mapreduce/OrphanViewTool.java | 83 +++++++-- .../apache/phoenix/mapreduce/index/IndexTool.java | 31 ++-- .../phoenix/query/ConnectionQueryServicesImpl.java | 81 +++++---- .../org/apache/phoenix/schema/MetaDataClient.java | 61 ++++--- .../apache/phoenix/schema/transform/Transform.java | 64 +++++-- .../java/org/apache/phoenix/util/MetaDataUtil.java | 44 +++-- .../java/org/apache/phoenix/util/QueryUtil.java | 22 ++- .../java/org/apache/phoenix/util/UpgradeUtil.java | 195 ++++++++++++++------- 8 files changed, 391 insertions(+), 190 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java index efa57d7e04..2d91bce07e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java @@ -40,6 +40,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -437,16 +438,37 @@ public class OrphanViewTool extends Configured implements Tool { } private void removeLink(PhoenixConnection phoenixConnection, Key src, Key dst, PTable.LinkType linkType) throws Exception { - String deleteQuery = "DELETE FROM " + - ((linkType == PTable.LinkType.PHYSICAL_TABLE || linkType == PTable.LinkType.PARENT_TABLE) ? SYSTEM_CATALOG_NAME : SYSTEM_CHILD_LINK_NAME) + - " WHERE " + TENANT_ID + (src.getTenantId() == null ? " IS NULL" : " = '" + src.getTenantId() + "'") + " AND " + - TABLE_SCHEM + (src.getSchemaName() == null ? " IS NULL " : " = '" + src.getSchemaName() + "'") + " AND " + - TABLE_NAME + " = '" + src.getTableName() + "' AND " + - COLUMN_NAME + (dst.getTenantId() == null ? " IS NULL" : " = '" + dst.getTenantId() + "'") + " AND " + - COLUMN_FAMILY + " = '" + (dst.getSchemaName() == null ? dst.getTableName() : dst.getSchemaName() + "." + - dst.getTableName()) + "'"; - phoenixConnection.createStatement().execute(deleteQuery); - phoenixConnection.commit(); + String delTable = (linkType == PTable.LinkType.PHYSICAL_TABLE + || linkType == PTable.LinkType.PARENT_TABLE) ? SYSTEM_CATALOG_NAME + : SYSTEM_CHILD_LINK_NAME; + + String deleteQuery = String.format(" DELETE FROM %s WHERE " + TENANT_ID + " %s AND " + + TABLE_SCHEM + " %s AND " + TABLE_NAME + " = ? AND " + COLUMN_NAME + " %s AND " + + COLUMN_FAMILY + " = ? ", delTable, + src.getTenantId() == null ? " IS NULL" : " = ? ", + src.getSchemaName() == null ? " IS NULL " : " = ? ", + dst.getTenantId() == null ? " IS NULL" : " = ?"); + + try (PreparedStatement delStmt = phoenixConnection.prepareStatement(deleteQuery)) { + int param = 0; + if (src.getTenantId() != null) { + delStmt.setString(++param, src.getTenantId()); + } + if (src.getSchemaName() != null) { + delStmt.setString(++param, src.getSchemaName()); + } + delStmt.setString(++param, src.getTableName()); + if (dst.getTenantId() != null) { + delStmt.setString(++param, dst.getTenantId()); + } + if (dst.getSchemaName() == null) { + delStmt.setString(++param, dst.getTableName()); + } else { + delStmt.setString(++param, dst.getSchemaName() + "." + dst.getTableName()); + } + delStmt.execute(); + phoenixConnection.commit(); + } } private byte getLinkType(PTable.LinkType linkType) { @@ -500,18 +522,41 @@ public class OrphanViewTool extends Configured implements Tool { } } } + + private void forcefullyDropView(PhoenixConnection phoenixConnection, Key key) throws Exception { - String deleteRowsFromCatalog = "DELETE FROM " + SYSTEM_CATALOG_NAME + - " WHERE " + TENANT_ID + (key.getTenantId() == null ? " IS NULL" : " = '" + key.getTenantId() + "'") + " AND " + - TABLE_SCHEM + (key.getSchemaName() == null ? " IS NULL " : " = '" + key.getSchemaName() + "'") + " AND " + - TABLE_NAME + " = '" + key.getTableName() + "'"; - String deleteRowsFromChildLink = "DELETE FROM " + SYSTEM_CHILD_LINK_NAME + - " WHERE " + COLUMN_NAME + (key.getTenantId() == null ? " IS NULL" : " = '" + key.getTenantId() + "'") + " AND " + - COLUMN_FAMILY + " = '" + (key.getSchemaName() == null ? key.getTableName() : key.getSchemaName() + "." + key.getTableName()) + "'"; + String deleteRowsFromCatalog = String.format("DELETE FROM " + SYSTEM_CATALOG_NAME + + " WHERE " + TENANT_ID + " %s AND " + TABLE_SCHEM + " %s AND " + + TABLE_NAME + " = ? ", + key.getTenantId() == null ? " IS NULL" : " = ? ", + key.getSchemaName() == null ? " IS NULL " : " = ? "); + String deleteRowsFromChildLink = String.format("DELETE FROM " + SYSTEM_CHILD_LINK_NAME + + " WHERE " + COLUMN_NAME + " %s AND " + COLUMN_FAMILY + " = ? ", + key.getTenantId() == null ? " IS NULL" : " = ? "); try { - phoenixConnection.createStatement().execute(deleteRowsFromCatalog); - phoenixConnection.createStatement().execute(deleteRowsFromChildLink); + try (PreparedStatement delSysCat = + phoenixConnection.prepareStatement(deleteRowsFromCatalog)) { + int param = 0; + if (key.getTenantId() != null) { + delSysCat.setString(++param, key.getTenantId()); + } + if (key.getSchemaName() != null) { + delSysCat.setString(++param, key.getSchemaName()); + } + delSysCat.setString(++param, key.getTableName()); + delSysCat.execute(); + } + try (PreparedStatement delChLink = + phoenixConnection.prepareStatement(deleteRowsFromChildLink)) { + int param = 0; + if (key.getTenantId() != null) { + delChLink.setString(++param, key.getTenantId()); + } + delChLink.setString(++param, key.getSchemaName() == null + ? key.getTableName() : (key.getSchemaName() + "." + key.getTableName())); + delChLink.execute(); + } phoenixConnection.commit(); } catch (SQLException e) { throw new IOException(e); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index dc9bcbd99d..38a0c3f3c3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -28,6 +28,7 @@ import static org.apache.phoenix.mapreduce.index.IndexVerificationResultReposito import java.io.IOException; import java.sql.Connection; import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.AbstractMap; @@ -598,18 +599,24 @@ public class IndexTool extends Configured implements Tool { } private long getMaxRebuildAsyncDate(String schemaName, List<String> disableIndexes) throws SQLException { - Long maxRebuilAsyncDate=HConstants.LATEST_TIMESTAMP; - Long maxDisabledTimeStamp=0L; - if (disableIndexes == null || disableIndexes.isEmpty()) { return 0; } - List<String> quotedIndexes = new ArrayList<String>(disableIndexes.size()); - for (String index : disableIndexes) { - quotedIndexes.add("'" + index + "'"); + Long maxRebuilAsyncDate = HConstants.LATEST_TIMESTAMP; + Long maxDisabledTimeStamp = 0L; + if (disableIndexes == null || disableIndexes.isEmpty()) { + return 0; } - try (ResultSet rs = connection.createStatement() - .executeQuery("SELECT MAX(" + ASYNC_REBUILD_TIMESTAMP + "),MAX("+INDEX_DISABLE_TIMESTAMP+") FROM " + SYSTEM_CATALOG_NAME + " (" - + ASYNC_REBUILD_TIMESTAMP + " BIGINT) WHERE " + TABLE_SCHEM - + (schemaName != null && schemaName.length() > 0 ? "='" + schemaName + "'" : " IS NULL") - + " and " + TABLE_NAME + " IN (" + StringUtils.join(",", quotedIndexes) + ")")) { + String query = String.format("SELECT MAX(" + ASYNC_REBUILD_TIMESTAMP + "), " + + "MAX(" + INDEX_DISABLE_TIMESTAMP + ") FROM " + + SYSTEM_CATALOG_NAME + " (" + ASYNC_REBUILD_TIMESTAMP + + " BIGINT) WHERE " + TABLE_SCHEM + " %s AND " + TABLE_NAME + " IN ( %s )", + (schemaName != null && schemaName.length() > 0) ? " = ? " : " IS NULL ", + QueryUtil.generateInListParams(disableIndexes.size())); + try (PreparedStatement selSyscat = connection.prepareStatement(query)) { + int param = 0; + if (schemaName != null && schemaName.length() > 0) { + selSyscat.setString(++param, schemaName); + } + QueryUtil.setQuoteInListElements(selSyscat, disableIndexes, param); + ResultSet rs = selSyscat.executeQuery(); if (rs.next()) { maxRebuilAsyncDate = rs.getLong(1); maxDisabledTimeStamp = rs.getLong(2); @@ -619,7 +626,7 @@ public class IndexTool extends Configured implements Tool { return maxRebuilAsyncDate; } else { throw new RuntimeException( - "Inconsistent state we have one or more index tables which are disabled after the async is called!!"); + "Inconsistent state we have one or more index tables which are disabled after the async is called!!"); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 4a1256f054..cb119a8401 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -3994,21 +3994,32 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0, PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " " + PTinyint.INSTANCE.getSqlTypeName()); - metaConnection.createStatement().executeUpdate("ALTER TABLE " + - PhoenixDatabaseMetaData.SYSTEM_CATALOG + " SET " + - HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS) - ); - metaConnection.createStatement().executeUpdate("ALTER TABLE " + - PhoenixDatabaseMetaData.SYSTEM_FUNCTION + " SET " + - TableDescriptorBuilder.SPLIT_POLICY + "='" + SystemFunctionSplitPolicy.class.getName() + "',\n" + - HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" + - ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS) - ); - metaConnection.createStatement().executeUpdate("ALTER TABLE " + - PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + - TableDescriptorBuilder.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName() +"'" - ); + try (Statement altQry = metaConnection.createStatement()) { + altQry.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_CATALOG + " SET " + + HConstants.VERSIONS + "= " + + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions + .DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" + + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, + QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); + + altQry.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_FUNCTION + " SET " + + TableDescriptorBuilder.SPLIT_POLICY + "='" + + SystemFunctionSplitPolicy.class.getName() + "',\n" + + HConstants.VERSIONS + "= " + + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions + .DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" + + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=" + + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, + QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)); + + altQry.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + TableDescriptorBuilder.SPLIT_POLICY + "='" + + SystemStatsSplitPolicy.class.getName() + "'"); + } } if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_4_15_0) { addViewIndexToParentLinks(metaConnection); @@ -4476,18 +4487,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } if (UpgradeUtil.tableHasKeepDeleted( metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { - try (Statement stmt = metaConnection.createStatement()){ - stmt.executeUpdate("ALTER TABLE " - + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " - + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + try (Statement altStmt = metaConnection.createStatement()) { + altStmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); } } if (UpgradeUtil.tableHasMaxVersions( metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { - try (Statement stmt = metaConnection.createStatement()){ - stmt.executeUpdate("ALTER TABLE " - + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " - + HConstants.VERSIONS + "='1'"); + try (Statement altStats = metaConnection.createStatement()) { + altStats.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + HConstants.VERSIONS + " = '1' "); } } } @@ -4522,10 +4533,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement metaConnection = addColumnsIfNotExists(metaConnection, taskTableFullName, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd); - try (Statement statement = metaConnection.createStatement()) { - String setTtlQuery = String.format(ALTER_TABLE_SET_PROPS, - taskTableFullName, TTL, TASK_TABLE_TTL); - statement.executeUpdate(setTtlQuery); + String altQuery = String.format(ALTER_TABLE_SET_PROPS, + taskTableFullName, TTL, TASK_TABLE_TTL); + try (PreparedStatement altQueryStmt = metaConnection.prepareStatement(altQuery)) { + altQueryStmt.executeUpdate(); } clearCache(); } @@ -4607,18 +4618,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement takeSnapshotOfSysTable(systemTableToSnapshotMap, e); if (UpgradeUtil.tableHasKeepDeleted( metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME) ) { - try (Statement stmt = metaConnection.createStatement()) { - stmt.executeUpdate("ALTER TABLE " + - PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + - KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + try (Statement altLogStmt = metaConnection.createStatement()) { + altLogStmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); } } if (UpgradeUtil.tableHasMaxVersions( metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME)) { - try (Statement stmt = metaConnection.createStatement()) { - stmt.executeUpdate("ALTER TABLE " + - PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + - HConstants.VERSIONS + "='1'"); + try (Statement altLogVer = metaConnection.createStatement()) { + altLogVer.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + HConstants.VERSIONS + "='1'"); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 1d53b68f88..5efbf7a79b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1164,19 +1164,23 @@ public class MetaDataClient { private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps, List<byte[]> cfs, boolean checkLastStatsUpdateTime) throws SQLException { ReadOnlyProps props = connection.getQueryServices().getProps(); final long msMinBetweenUpdates = props - .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, QueryServicesOptions.DEFAULT_MIN_STATS_UPDATE_FREQ_MS); + .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, + QueryServicesOptions.DEFAULT_MIN_STATS_UPDATE_FREQ_MS); Long scn = connection.getSCN(); // Always invalidate the cache long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn; long msSinceLastUpdate = Long.MAX_VALUE; if (checkLastStatsUpdateTime) { - String query = "SELECT CURRENT_DATE()," + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME - + " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY - + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL"; - ResultSet rs = connection.createStatement().executeQuery(query); - - if (rs.next()) { - msSinceLastUpdate = rs.getLong(1) - rs.getLong(2); + String query = "SELECT CURRENT_DATE()," + LAST_STATS_UPDATE_TIME + " FROM " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + + " WHERE " + PHYSICAL_NAME + "= ? AND " + COLUMN_FAMILY + + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL"; + try (PreparedStatement selectStatsStmt = connection.prepareStatement(query)) { + selectStatsStmt.setString(1, physicalName.getString()); + ResultSet rs = selectStatsStmt.executeQuery(query); + if (rs.next()) { + msSinceLastUpdate = rs.getLong(1) - rs.getLong(2); + } } } long rowCount = 0; @@ -4482,8 +4486,9 @@ public class MetaDataClient { } buf.setCharAt(buf.length()-1, ')'); - connection.createStatement().execute(buf.toString()); - + try (PreparedStatement delCol = connection.prepareStatement(buf.toString())) { + delCol.execute(); + } Collections.sort(columnsToDrop,new Comparator<PColumn> () { @Override public int compare(PColumn left, PColumn right) { @@ -4493,21 +4498,25 @@ public class MetaDataClient { boolean isSalted = table.getBucketNum() != null; int columnsToDropIndex = 0; - PreparedStatement colUpdate = connection.prepareStatement(UPDATE_COLUMN_POSITION); - colUpdate.setString(1, tenantId); - colUpdate.setString(2, schemaName); - colUpdate.setString(3, tableName); - for (int i = columnsToDrop.get(columnsToDropIndex).getPosition() + 1; i < table.getColumns().size(); i++) { - PColumn column = table.getColumns().get(i); - if (columnsToDrop.contains(column)) { - columnsToDropIndex++; - continue; + try (PreparedStatement colUpdate = connection.prepareStatement(UPDATE_COLUMN_POSITION)) { + colUpdate.setString(1, tenantId); + colUpdate.setString(2, schemaName); + colUpdate.setString(3, tableName); + for (int i = columnsToDrop.get(columnsToDropIndex).getPosition() + 1; + i < table.getColumns().size(); i++) { + PColumn column = table.getColumns().get(i); + if (columnsToDrop.contains(column)) { + columnsToDropIndex++; + continue; + } + colUpdate.setString(4, column.getName().getString()); + colUpdate.setString(5, column.getFamilyName() == null + ? null : column.getFamilyName().getString()); + // Adjust position to not include the salt column + colUpdate.setInt(6, + column.getPosition() - columnsToDropIndex - (isSalted ? 1 : 0)); + colUpdate.execute(); } - colUpdate.setString(4, column.getName().getString()); - colUpdate.setString(5, column.getFamilyName() == null ? null : column.getFamilyName().getString()); - // Adjust position to not include the salt column - colUpdate.setInt(6, column.getPosition() - columnsToDropIndex - (isSalted ? 1 : 0)); - colUpdate.execute(); } return familyName; } @@ -5032,7 +5041,9 @@ public class MetaDataClient { } else { Long scn = connection.getSCN(); long ts = scn == null ? HConstants.LATEST_TIMESTAMP : scn; - MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(indexRef), null, null, Collections.<PColumn>emptyList(), ts); + MutationPlan plan = new PostDDLCompiler(connection) + .compile(Collections.singletonList(indexRef), null, + null, Collections.<PColumn>emptyList(), ts); connection.getQueryServices().updateData(plan); } NamedTableNode dataTableNode = NamedTableNode.create(null, diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/transform/Transform.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/transform/Transform.java index 54f1450f27..383f7d2275 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/transform/Transform.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/transform/Transform.java @@ -47,6 +47,7 @@ import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.JacksonUtil; import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TableViewFinderResult; import org.apache.phoenix.util.UpgradeUtil; @@ -472,16 +473,15 @@ public class Transform { getMetadataDifference(connection, systemTransformRecord, columnNames, columnValues); // TODO In the future, we need to handle rowkey changes and column type changes as well - String - changeViewStmt = "UPSERT INTO SYSTEM.CATALOG (TENANT_ID, TABLE_SCHEM, TABLE_NAME %s) VALUES (%s, %s, '%s' %s)"; + String changeViewStmt = "UPSERT INTO SYSTEM.CATALOG " + + "(TENANT_ID, TABLE_SCHEM, TABLE_NAME %s) VALUES (?, ?, ? %s)"; String - changeTable = String.format( - "UPSERT INTO SYSTEM.CATALOG (TENANT_ID, TABLE_SCHEM, TABLE_NAME, PHYSICAL_TABLE_NAME %s) VALUES (%s, %s, '%s','%s' %s)", - (columnNames.size() > 0? "," + String.join(",", columnNames):""), - (tenantId==null? null: ("'" + tenantId + "'")), - (schema==null ? null : ("'" + schema + "'")), tableName, newTableName, - (columnValues.size() > 0? "," + String.join(",", columnValues):"")); + changeTable = String.format("UPSERT INTO SYSTEM.CATALOG " + + "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, PHYSICAL_TABLE_NAME %s ) " + + "VALUES(?, ?, ?, ? %s)", columnNames.size() > 0 ? "," + + String.join(",", columnNames) : "", columnNames.size() > 0 + ? "," + QueryUtil.generateInListParams(columnValues.size()) : ""); LOGGER.info("About to do cutover via " + changeTable); TableViewFinderResult childViewsResult = ViewUtil.findChildViews(connection, tenantId, schema, tableName); @@ -489,8 +489,25 @@ public class Transform { connection.setAutoCommit(false); List<TableInfo> viewsToUpdateCache = new ArrayList<>(); try { - connection.createStatement().execute(changeTable); - + try (PreparedStatement stmt = connection.prepareStatement(changeTable)) { + int param = 0; + if (tenantId == null) { + stmt.setNull(++param, Types.VARCHAR); + } else { + stmt.setString(++param, tenantId); + } + if (schema == null) { + stmt.setNull(++param, Types.VARCHAR); + } else { + stmt.setString(++param, schema); + } + stmt.setString(++param, tableName); + stmt.setString(++param, newTableName); + for (int i = 0; i < columnValues.size(); i++) { + stmt.setInt(++param, Integer.parseInt(columnValues.get(i))); + } + stmt.execute(); + } // Update column qualifiers PTable pNewTable = PhoenixRuntime.getTable(connection, systemTransformRecord.getNewPhysicalTableName()); PTable pOldTable = PhoenixRuntime.getTable(connection, SchemaUtil.getTableName(schema, tableName)); @@ -520,13 +537,28 @@ public class Transform { int batchSize = 0; for (TableInfo view : childViewsResult.getLinks()) { String changeView = String.format(changeViewStmt, - (columnNames.size() > 0? "," + String.join(",", columnNames):""), - (view.getTenantId()==null || view.getTenantId().length == 0? null: ("'" + Bytes.toString(view.getTenantId()) + "'")), - (view.getSchemaName()==null || view.getSchemaName().length == 0? null : ("'" + Bytes.toString(view.getSchemaName()) + "'")), - Bytes.toString(view.getTableName()), - (columnValues.size() > 0? "," + String.join(",", columnValues):"")); + columnNames.size() > 0 ? "," + String.join(",", columnNames) : "", + columnNames.size() > 0 ? "," + + QueryUtil.generateInListParams(columnValues.size()) : ""); LOGGER.info("Cutover changing view via " + changeView); - connection.createStatement().execute(changeView); + try (PreparedStatement stmt = connection.prepareStatement(changeView)) { + int param = 0; + if (view.getTenantId() == null || view.getTenantId().length == 0) { + stmt.setNull(++param, Types.VARCHAR); + } else { + stmt.setString(++param, Bytes.toString(view.getTenantId())); + } + if (view.getSchemaName() == null || view.getSchemaName().length == 0) { + stmt.setNull(++param, Types.VARCHAR); + } else { + stmt.setString(++param, Bytes.toString(view.getSchemaName())); + } + stmt.setString(++param, Bytes.toString(view.getTableName())); + for (int i = 0; i < columnValues.size(); i++) { + stmt.setInt(++param, Integer.parseInt(columnValues.get(i))); + } + stmt.execute(); + } viewsToUpdateCache.add(view); batchSize++; if (batchSize >= maxBatchSize) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 9d1d120e5b..d148884703 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -22,6 +22,7 @@ import static org.apache.phoenix.util.SchemaUtil.getVarChars; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.*; @@ -914,11 +915,19 @@ public class MetaDataUtil { throws SQLException { String schemaName = getViewIndexSequenceSchemaName(name, isNamespaceMapped); String sequenceName = getViewIndexSequenceName(name, null, isNamespaceMapped); - connection.createStatement().executeUpdate("DELETE FROM " - + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE - + " WHERE " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA - + (schemaName.length() > 0 ? "='" + schemaName + "'" : " IS NULL") - + " AND " + PhoenixDatabaseMetaData.SEQUENCE_NAME + " = '" + sequenceName + "'" ); + String delQuery = String.format(" DELETE FROM " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + + " WHERE " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " %s AND " + + PhoenixDatabaseMetaData.SEQUENCE_NAME + " = ? ", + schemaName.length() > 0 ? "= ? " : " IS NULL"); + try (PreparedStatement delSeqStmt = connection.prepareStatement(delQuery)) { + if (schemaName.length() > 0) { + delSeqStmt.setString(1, schemaName); + delSeqStmt.setString(2, sequenceName); + } else { + delSeqStmt.setString(1, sequenceName); + } + delSeqStmt.executeUpdate(); + } } /** @@ -1199,9 +1208,8 @@ public class MetaDataUtil { physicalTablesSet.add(s.getPhysicalNames().get(0).getString()); } StringBuilder buf = new StringBuilder("DELETE FROM SYSTEM.STATS WHERE PHYSICAL_NAME IN ("); - Iterator itr = physicalTablesSet.iterator(); - while (itr.hasNext()) { - buf.append("'" + itr.next() + "',"); + for (int i = 0; i < physicalTablesSet.size(); i++) { + buf.append(" ?,"); } buf.setCharAt(buf.length() - 1, ')'); if (table.getIndexType()==IndexType.LOCAL) { @@ -1209,13 +1217,25 @@ public class MetaDataUtil { if (table.getColumnFamilies().isEmpty()) { buf.append("'" + QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY + "',"); } else { - for(PColumnFamily cf : table.getColumnFamilies()) { - buf.append("'" + cf.getName().getString() + "',"); - } + buf.append(QueryUtil.generateInListParams(table + .getColumnFamilies().size())); } buf.setCharAt(buf.length() - 1, ')'); } - connection.createStatement().execute(buf.toString()); + try (PreparedStatement delStatsStmt = connection.prepareStatement(buf.toString())) { + int param = 0; + Iterator itr = physicalTablesSet.iterator(); + while (itr.hasNext()) { + delStatsStmt.setString(++param, itr.next().toString()); + } + if (table.getIndexType() == IndexType.LOCAL + && !table.getColumnFamilies().isEmpty()) { + for (PColumnFamily cf : table.getColumnFamilies()) { + delStatsStmt.setString(++param, cf.getName().getString()); + } + } + delStatsStmt.execute(); + } } finally { connection.setAutoCommit(isAutoCommit); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java index 288cbeff10..00a8d8dc52 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java @@ -707,7 +707,7 @@ public final class QueryUtil { addTenantIdFilter(connection, buf, catalog, parameterValues); if (schemaPattern != null) { buf.append(" and " + TABLE_SCHEM + (schemaPattern.length() == 0 ? " is null" : " like ?" )); - if(schemaPattern.length() > 0) { + if (schemaPattern.length() > 0) { parameterValues.add(schemaPattern); } } @@ -751,7 +751,7 @@ public final class QueryUtil { if (schemaPattern != null) { appendConjunction(whereClause); whereClause.append(SEQUENCE_SCHEMA + (schemaPattern.length() == 0 ? " is null" : " like ?\n" )); - if(schemaPattern.length() > 0) { + if (schemaPattern.length() > 0) { parameterValues.add(schemaPattern); } } @@ -767,7 +767,7 @@ public final class QueryUtil { } buf.append(" order by 4, 1, 2, 3\n"); PreparedStatement stmt = connection.prepareStatement(buf.toString()); - for(int i = 0; i < parameterValues.size(); i++) { + for (int i = 0; i < parameterValues.size(); i++) { stmt.setString(i+1, parameterValues.get(i)); } return stmt; @@ -826,5 +826,19 @@ public final class QueryUtil { private static void appendConjunction(StringBuilder buf) { buf.append(buf.length() == 0 ? "" : " and "); } - + + public static String generateInListParams(int nParams) { + List<String> paramList = Lists.newArrayList(); + for (int i = 0; i < nParams; i++) { + paramList.add("?"); + } + return Joiner.on(", ").join(paramList); + } + + public static void setQuoteInListElements(PreparedStatement ps, List<String> unQuotedString, + int index) throws SQLException { + for (int i = 0; i < unQuotedString.size(); i++) { + ps.setString(++index, "'" + unQuotedString + "'"); + } + } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 409ad13bea..f64faa7d7a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -1823,6 +1823,25 @@ public class UpgradeUtil { } } + private static String getTableRVCWithParam(List<String> tableNames) { + StringBuilder query = new StringBuilder("("); + for (int i = 0; i < tableNames.size(); i += 3) { + String tenantId = tableNames.get(i); + String schemaName = tableNames.get(i + 1); + String tableName = tableNames.get(i + 2); + query.append('('); + query.append(tenantId == null ? "null" : " ? "); + query.append(','); + query.append(schemaName == null ? "null" : " ? "); + query.append(','); + query.append(" ? "); + query.append("),"); + } + // Replace trailing , with ) to end IN expression + query.setCharAt(query.length() - 1, ')'); + return query.toString(); + } + private static String getTableRVC(List<String> tableNames) { StringBuilder query = new StringBuilder("("); for (int i = 0; i < tableNames.size(); i+=3) { @@ -1857,25 +1876,44 @@ public class UpgradeUtil { // Find the header rows for tables that have not been upgraded already. // We don't care about views, as the row key cannot be different than the table. // We need this query to find physical tables which won't have a link row. - String query = "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME,TABLE_TYPE\n" + - "FROM SYSTEM.CATALOG (ROW_KEY_ORDER_OPTIMIZABLE BOOLEAN)\n" + - "WHERE COLUMN_NAME IS NULL\n" + - "AND COLUMN_FAMILY IS NULL\n" + - "AND ROW_KEY_ORDER_OPTIMIZABLE IS NULL\n" + - "AND TABLE_TYPE IN ('" + PTableType.TABLE.getSerializedValue() + "','" + otherType.getSerializedValue() + "')\n" + - "AND (TENANT_ID, TABLE_SCHEM, TABLE_NAME) IN " + getTableRVC(tableNames); - rs = conn.createStatement().executeQuery(query); - - while (rs.next()) { - if (PTableType.TABLE.getSerializedValue().equals(rs.getString(4))) { - physicalTables.add(SchemaUtil.getTableName(rs.getString(2), rs.getString(3))); - } else { - otherTables.add(rs.getString(1)); - otherTables.add(rs.getString(2)); - otherTables.add(rs.getString(3)); + + String query = String.format("SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME,TABLE_TYPE" + + "FROM SYSTEM.CATALOG (ROW_KEY_ORDER_OPTIMIZABLE BOOLEAN)" + + "WHERE COLUMN_NAME IS NULL" + + "AND COLUMN_FAMILY IS NULL" + + "AND ROW_KEY_ORDER_OPTIMIZABLE IS NULL" + + "AND TABLE_TYPE IN (%s , %s )" + + "AND (TENANT_ID, TABLE_SCHEM, TABLE_NAME) IN %s ", + PTableType.TABLE.getSerializedValue(), otherType.getSerializedValue(), + getTableRVCWithParam(tableNames)); + try (PreparedStatement selSysCat = conn.prepareStatement(query)) { + int param = 0; + for (int i = 0; i < tableNames.size(); i += 3) { + String tenantId = tableNames.get(i); + String schemaName = tableNames.get(i + 1); + String tableName = tableNames.get(i + 2); + if (tenantId != null) { + selSysCat.setString(++param, tenantId); + } + if (schemaName != null) { + selSysCat.setString(++param, schemaName); + } + selSysCat.setString(++param, tableName); } + rs = selSysCat.executeQuery(); + while (rs.next()) { + if (PTableType.TABLE.getSerializedValue() + .equals(rs.getString(4))) { + physicalTables.add(SchemaUtil + .getTableName(rs.getString(2), rs.getString(3))); + } else { + otherTables.add(rs.getString(1)); + otherTables.add(rs.getString(2)); + otherTables.add(rs.getString(3)); + } + } + return otherTables; } - return otherTables; } // Return all types that are descending and either: @@ -2031,16 +2069,28 @@ public class UpgradeUtil { String theTenantId = tableNames.get(i); String theSchemaName = tableNames.get(i+1); String theTableName = tableNames.get(i+2); - globalConn.createStatement().execute("UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + - " (" + PhoenixDatabaseMetaData.TENANT_ID + "," + - PhoenixDatabaseMetaData.TABLE_SCHEM + "," + - PhoenixDatabaseMetaData.TABLE_NAME + "," + - MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE + " BOOLEAN" - + ") VALUES (" + - "'" + (theTenantId == null ? StringUtil.EMPTY_STRING : theTenantId) + "'," + - "'" + (theSchemaName == null ? StringUtil.EMPTY_STRING : theSchemaName) + "'," + - "'" + theTableName + "'," + - "TRUE)"); + String upsSyscat = String.format("UPSERT INTO " + + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + + " (" + PhoenixDatabaseMetaData.TENANT_ID + "," + + PhoenixDatabaseMetaData.TABLE_SCHEM + "," + + PhoenixDatabaseMetaData.TABLE_NAME + "," + + MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE + " BOOLEAN" + + ") VALUES ( ?, ?, ?, TRUE)"); + try (PreparedStatement upsSyscatStmt = globalConn.prepareStatement(upsSyscat)) { + int param = 0; + if (theTenantId == null) { + upsSyscatStmt.setNull(++param, Types.VARCHAR); + } else { + upsSyscatStmt.setString(++param, theTenantId); + } + if (theSchemaName == null) { + upsSyscatStmt.setNull(++param, Types.VARCHAR); + } else { + upsSyscatStmt.setString(++param, theSchemaName); + } + upsSyscatStmt.setString(++param, theTableName); + upsSyscatStmt.execute(); + } } globalConn.commit(); for (int i = 0; i < tableNames.size(); i += 3) { @@ -2163,45 +2213,51 @@ public class UpgradeUtil { * @return true if any upgrades were performed and false otherwise. */ private static boolean upgradeSharedIndex(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String physicalName, boolean bypassUpgrade) throws SQLException { - String query = - "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + - "FROM SYSTEM.CATALOG cat1\n" + - "WHERE COLUMN_NAME IS NULL\n" + - "AND COLUMN_FAMILY = '" + physicalName + "'\n" + - "AND LINK_TYPE = " + LinkType.PHYSICAL_TABLE.getSerializedValue() + "\n" + - "ORDER BY TENANT_ID"; - ResultSet rs = globalConn.createStatement().executeQuery(query); - String lastTenantId = null; - Connection conn = globalConn; - String url = globalConn.getURL(); - boolean wasUpgraded = false; - while (rs.next()) { - String fullTableName = SchemaUtil.getTableName( + String query = String.format( + "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME" + + "FROM SYSTEM.CATALOG cat1" + + "WHERE COLUMN_NAME IS NULL" + + "AND COLUMN_FAMILY = ? " + + "AND LINK_TYPE = %s " + + "ORDER BY TENANT_ID", LinkType.PHYSICAL_TABLE.getSerializedValue()); + try (PreparedStatement selSysCatstmt = globalConn.prepareStatement(query)) { + selSysCatstmt.setString(1, physicalName); + ResultSet rs = selSysCatstmt.executeQuery(); + String lastTenantId = null; + Connection conn = globalConn; + String url = globalConn.getURL(); + boolean wasUpgraded = false; + while (rs.next()) { + String fullTableName = SchemaUtil.getTableName( rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM), rs.getString(PhoenixDatabaseMetaData.TABLE_NAME)); - String tenantId = rs.getString(1); - if (tenantId != null && !tenantId.equals(lastTenantId)) { - if (lastTenantId != null) { - conn.close(); + String tenantId = rs.getString(1); + if (tenantId != null && !tenantId.equals(lastTenantId)) { + if (lastTenantId != null) { + conn.close(); + } + // Open tenant-specific connection when we find a new one + Properties props = new Properties(globalConn.getClientInfo()); + props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); + conn = DriverManager.getConnection(url, props); + lastTenantId = tenantId; + } + PTable table = PhoenixRuntime.getTable(conn, fullTableName); + String tableTenantId = + table.getTenantId() == null ? null : table.getTenantId().getString(); + if (Objects.equal(lastTenantId, tableTenantId) && !table.rowKeyOrderOptimizable()) { + upgradeDescVarLengthRowKeys(upgradeConn, globalConn, + table.getSchemaName().getString(), table.getTableName().getString(), false, + bypassUpgrade); + wasUpgraded = true; } - // Open tenant-specific connection when we find a new one - Properties props = new Properties(globalConn.getClientInfo()); - props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); - conn = DriverManager.getConnection(url, props); - lastTenantId = tenantId; } - PTable table = PhoenixRuntime.getTable(conn, fullTableName); - String tableTenantId = table.getTenantId() == null ? null : table.getTenantId().getString(); - if (Objects.equal(lastTenantId, tableTenantId) && !table.rowKeyOrderOptimizable()) { - upgradeDescVarLengthRowKeys(upgradeConn, globalConn, table.getSchemaName().getString(), table.getTableName().getString(), false, bypassUpgrade); - wasUpgraded = true; + rs.close(); + if (lastTenantId != null) { + conn.close(); } + return wasUpgraded; } - rs.close(); - if (lastTenantId != null) { - conn.close(); - } - return wasUpgraded; } public static void addRowKeyOrderOptimizableCell(List<Mutation> tableMetadata, byte[] tableHeaderRowKey, long clientTimeStamp) { @@ -2563,13 +2619,18 @@ public class UpgradeUtil { String newSchemaName = MetaDataUtil.getViewIndexSequenceSchemaName(physicalName, true); String newSequenceName = MetaDataUtil.getViewIndexSequenceName(physicalName, tenantId, true); // create new entry with new schema format - String upsert = "UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + " SELECT NULL,\'" + newSchemaName + - "\',\'" + newSequenceName - + "\'," + START_WITH + "," + CURRENT_VALUE + "," + INCREMENT_BY + "," + CACHE_SIZE + "," + MIN_VALUE - + "," + MAX_VALUE + "," + CYCLE_FLAG + "," + LIMIT_REACHED_FLAG + " FROM " - + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + " WHERE " + PhoenixDatabaseMetaData.TENANT_ID - + " IS NULL AND " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " = '" + oldSchemaName + "'"; - connection.createStatement().executeUpdate(upsert); + String upsert = "UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + + " SELECT NULL, ?, ?, " + START_WITH + "," + CURRENT_VALUE + "," + INCREMENT_BY + + "," + CACHE_SIZE + "," + MIN_VALUE + "," + MAX_VALUE + "," + CYCLE_FLAG + "," + + LIMIT_REACHED_FLAG + " FROM " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + " WHERE " + + PhoenixDatabaseMetaData.TENANT_ID + " IS NULL AND " + + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " = ?"; + try (PreparedStatement upsertSeqStmt = connection.prepareStatement(upsert)) { + upsertSeqStmt.setString(1, newSchemaName); + upsertSeqStmt.setString(2, newSequenceName); + upsertSeqStmt.setString(3, oldSchemaName); + upsertSeqStmt.executeUpdate(); + } } private static void updateLink(PhoenixConnection conn, String srcTableName,