This is an automated email from the ASF dual-hosted git repository.
richardantal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 27d6b1b251 PHOENIX-7474 Migrate IndexTool tables and make sure they
are created (#2031)
27d6b1b251 is described below
commit 27d6b1b251b2ea771b75f4b6d992e8fae97ec50d
Author: richardantal <[email protected]>
AuthorDate: Thu Jan 29 09:43:58 2026 +0100
PHOENIX-7474 Migrate IndexTool tables and make sure they are created (#2031)
---
.../mapreduce/index/IndexToolTableUtil.java | 168 +++++++++++
.../phoenix/query/ConnectionQueryServicesImpl.java | 14 +
.../apache/phoenix/mapreduce/index/IndexTool.java | 12 +
.../index/IndexVerificationOutputRepository.java | 41 +--
.../index/IndexVerificationResultRepository.java | 49 +--
.../phoenix/end2end/AsyncIndexPermissionIT.java | 331 +++++++++++++++++++++
.../apache/phoenix/end2end/BasePermissionsIT.java | 8 +-
.../org/apache/phoenix/end2end/IndexToolIT.java | 48 ++-
.../end2end/LoadSystemTableSnapshotBase.java | 37 +--
.../MigrateSystemTablesToSystemNamespaceIT.java | 19 +-
.../end2end/SkipSystemTablesExistenceCheckIT.java | 3 +-
.../SystemTablesCreationOnConnectionIT.java | 12 +-
.../phoenix/end2end/UpgradeIndexToolTablesIT.java | 99 ++++++
...eIndexToolTablesNameSpaceMappingDisabledIT.java | 31 ++
.../indexToolsnapshot.tar.gz | Bin 0 -> 20480 bytes
15 files changed, 761 insertions(+), 111 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java
new file mode 100644
index 0000000000..fd03a5c18e
--- /dev/null
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.index;
+
+
+
+import java.io.IOException;
+import java.sql.Connection;
+
+import java.sql.SQLException;
+import java.util.UUID;
+
+
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import
org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
+import org.apache.phoenix.query.QueryConstants;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.TableName;
+
+import org.apache.phoenix.jdbc.PhoenixConnection;
+
+import org.apache.phoenix.query.ConnectionQueryServices;
+
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.ClientUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
+
+/**
+ * Utility class to create index tables and/or migrate them.
+ *
+ */
+public class IndexToolTableUtil extends Configured {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(IndexToolTableUtil.class);
+
+ public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
+ public static String OUTPUT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME,
+ OUTPUT_TABLE_NAME);
+
+ public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
+ public static String RESULT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME,
+ RESULT_TABLE_NAME);
+
+ public static void setIndexToolTableName(Connection connection) throws
Exception {
+ ConnectionQueryServices queryServices =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
queryServices.getConfiguration())) {
+ OUTPUT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME).replace(
+ QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ RESULT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME).replace(
+ QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ } else {
+ OUTPUT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME);
+ RESULT_TABLE_FULL_NAME =
SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME);
+ }
+ }
+
+ public static Table createResultTable(Connection connection) throws
IOException, SQLException {
+ ConnectionQueryServices queryServices =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ try (Admin admin = queryServices.getAdmin()) {
+ TableName resultTableName =
TableName.valueOf(RESULT_TABLE_FULL_NAME);
+ if
(RESULT_TABLE_FULL_NAME.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+ createSystemNamespaceTable(connection);
+ }
+ return createTable(admin, resultTableName);
+ }
+ }
+
+ public static Table createOutputTable(Connection connection) throws
IOException, SQLException {
+ ConnectionQueryServices queryServices =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ try (Admin admin = queryServices.getAdmin()) {
+ TableName outputTableName =
TableName.valueOf(OUTPUT_TABLE_FULL_NAME);
+ if
(OUTPUT_TABLE_FULL_NAME.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+ createSystemNamespaceTable(connection);
+ }
+ return createTable(admin, outputTableName);
+ }
+ }
+
+ public static void createSystemNamespaceTable(Connection connection)
throws IOException, SQLException {
+ ConnectionQueryServices queryServices =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
queryServices.getConfiguration())) {
+ try (Admin admin = queryServices.getAdmin()) {
+ if (!ClientUtil.isHBaseNamespaceAvailable(admin,
SYSTEM_SCHEMA_NAME)) {
+ NamespaceDescriptor namespaceDescriptor =
+
NamespaceDescriptor.create(SYSTEM_SCHEMA_NAME).build();
+ admin.createNamespace(namespaceDescriptor);
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ private static Table createTable(Admin admin, TableName tableName) throws
IOException {
+ if (!admin.tableExists(tableName)) {
+ ColumnFamilyDescriptor columnDescriptor =
+ ColumnFamilyDescriptorBuilder
+
.newBuilder(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)
+ .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL)
+ .build();
+ TableDescriptor tableDescriptor =
TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(columnDescriptor).build();
+ try {
+ admin.createTable(tableDescriptor);
+ } catch (TableExistsException e) {
+ LOGGER.warn("Table exists, ignoring", e);
+ }
+ }
+ return admin.getConnection().getTable(tableName);
+ }
+
+
+ public static void createNewIndexToolTables(Connection connection) throws
Exception {
+ setIndexToolTableName(connection);
+
+ migrateTable(connection, OUTPUT_TABLE_NAME);
+ migrateTable(connection, RESULT_TABLE_NAME);
+ }
+
+ private static void migrateTable(Connection connection, String tableName)
throws Exception {
+ if (!tableName.equals(OUTPUT_TABLE_NAME) &&
!tableName.equals(RESULT_TABLE_NAME)) {
+ LOGGER.info("Only migrating PHOENIX_INDEX_TOOL tables!");
+ } else {
+ ConnectionQueryServices queryServices =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ try (Admin admin = queryServices.getAdmin()) {
+ TableName oldTableName = TableName.valueOf(tableName);
+ String newTableNameString =
tableName.equals(OUTPUT_TABLE_NAME) ?
+ OUTPUT_TABLE_FULL_NAME : RESULT_TABLE_FULL_NAME;
+
+ TableName newTableName = TableName.valueOf(newTableNameString);
+
+ if (admin.tableExists(oldTableName)) {
+ String snapshotName = tableName + "_" + UUID.randomUUID();
+ admin.disableTable(oldTableName);
+ admin.snapshot(snapshotName, oldTableName);
+ admin.cloneSnapshot(snapshotName, newTableName);
+ admin.deleteSnapshot(snapshotName);
+ admin.deleteTable(oldTableName);
+ } else {
+ createTable(admin, newTableName);
+ }
+ }
+ }
+ }
+}
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index cd141e1182..4ab5b6336b 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -259,6 +259,7 @@ import org.apache.phoenix.log.ConnectionLimiter;
import org.apache.phoenix.log.DefaultConnectionLimiter;
import org.apache.phoenix.log.LoggingConnectionLimiter;
import org.apache.phoenix.log.QueryLoggerDisruptor;
+import org.apache.phoenix.mapreduce.index.IndexToolTableUtil;
import org.apache.phoenix.monitoring.HTableThreadPoolHistograms;
import org.apache.phoenix.monitoring.TableMetricsManager;
import org.apache.phoenix.parse.PFunction;
@@ -4434,6 +4435,12 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices
metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
} catch (TableAlreadyExistsException ignore) {
}
+ try {
+ // check if we have old PHOENIX_INDEX_TOOL tables
+ // move data to the new tables under System, or simply create the new
tables
+ IndexToolTableUtil.createNewIndexToolTables(metaConnection);
+ } catch (Exception ignore) {
+ }
}
/**
@@ -4936,6 +4943,13 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices
// with SYSTEM Namespace
createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
+ try {
+ // check if we have old PHOENIX_INDEX_TOOL tables
+ // move data to the new tables under System, or simply create the new
tables
+ IndexToolTableUtil.createNewIndexToolTables(metaConnection);
+
+ } catch (Exception ignore) {}
+
clearUpgradeRequired();
success = true;
} catch (UpgradeInProgressException | UpgradeNotRequiredException e) {
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 92d4d01c55..c3f0d1aa81 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -83,10 +83,12 @@ import
org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.types.PVarchar;
@@ -1002,10 +1004,20 @@ public class IndexTool extends Configured implements
Tool {
.format(" %s is not an index table for %s for this connection",
indexTable, qDataTable));
}
qSchemaName = SchemaUtil.normalizeIdentifier(schemaName);
+
pIndexTable = connection.unwrap(PhoenixConnection.class)
.getTable(SchemaUtil.getQualifiedTableName(schemaName, indexTable));
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) {
+ pIndexTable = connection.unwrap(PhoenixConnection.class).getTable(
+ SchemaUtil.getQualifiedTableName(schemaName, indexTable).replace(
+ QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR));
+ }
indexType = pIndexTable.getIndexType();
qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) {
+ qIndexTable = qIndexTable.replace(QueryConstants.NAME_SEPARATOR,
QueryConstants.NAMESPACE_SEPARATOR);
+ }
if (IndexType.LOCAL.equals(indexType)) {
isLocalIndexBuild = true;
if (useSnapshot) {
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
index b5331febde..00d389b069 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
@@ -18,25 +18,19 @@
package org.apache.phoenix.mapreduce.index;
import java.io.IOException;
-import java.sql.Connection;
import java.sql.SQLException;
+import java.sql.Connection;
+
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import java.util.Iterator;
+
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.hbase.index.table.HTableFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -60,10 +54,9 @@ public class IndexVerificationOutputRepository implements
AutoCloseable {
IndexTool.IndexDisableLoggingType.NONE;
private boolean shouldLogBeyondMaxLookback = true;
- public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
- public final static byte[] OUTPUT_TABLE_NAME_BYTES =
Bytes.toBytes(OUTPUT_TABLE_NAME);
- public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY =
- QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
+ public final static String OUTPUT_TABLE_NAME =
IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME;
+ public final static byte[] OUTPUT_TABLE_NAME_BYTES =
Bytes.toBytes(IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME);
+ public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY =
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
public final static String DATA_TABLE_NAME = "DTName";
public final static byte[] DATA_TABLE_NAME_BYTES =
Bytes.toBytes(DATA_TABLE_NAME);
@@ -180,25 +173,7 @@ public class IndexVerificationOutputRepository implements
AutoCloseable {
}
public void createOutputTable(Connection connection) throws IOException,
SQLException {
- ConnectionQueryServices queryServices =
- connection.unwrap(PhoenixConnection.class).getQueryServices();
- try (Admin admin = queryServices.getAdmin()) {
- TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME);
- if (!admin.tableExists(outputTableName)) {
- ColumnFamilyDescriptor columnDescriptor =
- ColumnFamilyDescriptorBuilder.newBuilder(OUTPUT_TABLE_COLUMN_FAMILY)
- .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL).build();
- TableDescriptor tableDescriptor =
-
TableDescriptorBuilder.newBuilder(TableName.valueOf(OUTPUT_TABLE_NAME))
- .setColumnFamily(columnDescriptor).build();
- try {
- admin.createTable(tableDescriptor);
- } catch (TableExistsException e) {
- LOGGER.warn("Table exists, ignoring", e);
- }
- outputTable = admin.getConnection().getTable(outputTableName);
- }
- }
+ outputTable = IndexToolTableUtil.createOutputTable(connection);
}
@VisibleForTesting
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
index 1481f8cf0a..ec63f825aa 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
@@ -21,35 +21,23 @@ import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.IndexToolVerificationResult;
-import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.hbase.index.table.HTableFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.ByteUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class IndexVerificationResultRepository implements AutoCloseable {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(IndexVerificationResultRepository.class);
public static final String RUN_STATUS_SKIPPED = "Skipped";
public static final String RUN_STATUS_EXECUTED = "Executed";
@@ -57,8 +45,8 @@ public class IndexVerificationResultRepository implements
AutoCloseable {
private Table indexTable;
public static final String ROW_KEY_SEPARATOR = "|";
public static final byte[] ROW_KEY_SEPARATOR_BYTE =
Bytes.toBytes(ROW_KEY_SEPARATOR);
- public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
- public final static byte[] RESULT_TABLE_NAME_BYTES =
Bytes.toBytes(RESULT_TABLE_NAME);
+ public static String RESULT_TABLE_NAME =
IndexToolTableUtil.RESULT_TABLE_FULL_NAME;
+ public static byte[] RESULT_TABLE_NAME_BYTES =
Bytes.toBytes(IndexToolTableUtil.RESULT_TABLE_FULL_NAME);
public final static byte[] RESULT_TABLE_COLUMN_FAMILY =
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
public final static String SCANNED_DATA_ROW_COUNT = "ScannedDataRowCount";
@@ -172,37 +160,18 @@ public class IndexVerificationResultRepository implements
AutoCloseable {
}
- public IndexVerificationResultRepository(Connection conn, byte[]
indexNameBytes)
- throws SQLException {
- resultTable = getTable(conn, RESULT_TABLE_NAME_BYTES);
+ public IndexVerificationResultRepository(Connection conn, byte[]
indexNameBytes) throws SQLException {
+ resultTable = getTable(conn,
Bytes.toBytes(IndexToolTableUtil.RESULT_TABLE_FULL_NAME));
indexTable = getTable(conn, indexNameBytes);
}
- public IndexVerificationResultRepository(byte[] indexName, HTableFactory
hTableFactory)
- throws IOException {
- resultTable = hTableFactory.getTable(new
ImmutableBytesPtr(RESULT_TABLE_NAME_BYTES));
+ public IndexVerificationResultRepository(byte[] indexName, HTableFactory
hTableFactory) throws IOException {
+ resultTable = hTableFactory.getTable(new
ImmutableBytesPtr(Bytes.toBytes(IndexToolTableUtil.RESULT_TABLE_FULL_NAME)));
indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
}
public void createResultTable(Connection connection) throws IOException,
SQLException {
- ConnectionQueryServices queryServices =
- connection.unwrap(PhoenixConnection.class).getQueryServices();
- try (Admin admin = queryServices.getAdmin()) {
- TableName resultTableName = TableName.valueOf(RESULT_TABLE_NAME);
- if (!admin.tableExists(resultTableName)) {
- ColumnFamilyDescriptor columnDescriptor =
- ColumnFamilyDescriptorBuilder.newBuilder(RESULT_TABLE_COLUMN_FAMILY)
- .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL).build();
- TableDescriptor tableDescriptor =
TableDescriptorBuilder.newBuilder(resultTableName)
- .setColumnFamily(columnDescriptor).build();
- try {
- admin.createTable(tableDescriptor);
- } catch (TableExistsException e) {
- LOGGER.warn("Table exists, ignoring", e);
- }
- resultTable = admin.getConnection().getTable(resultTableName);
- }
- }
+ resultTable = IndexToolTableUtil.createResultTable(connection);
}
private static byte[] generatePartialResultTableRowKey(long ts, byte[]
indexTableName) {
@@ -387,7 +356,7 @@ public class IndexVerificationResultRepository implements
AutoCloseable {
public IndexToolVerificationResult getVerificationResult(Connection conn,
long ts,
byte[] indexTableName) throws IOException, SQLException {
- try (Table hTable = getTable(conn, RESULT_TABLE_NAME_BYTES)) {
+ try (Table hTable = getTable(conn,
Bytes.toBytes(IndexToolTableUtil.RESULT_TABLE_FULL_NAME))) {
byte[] startRowKey = generatePartialResultTableRowKey(ts,
indexTableName);
byte[] stopRowKey =
ByteUtil.calculateTheClosestNextRowKeyForPrefix(startRowKey);
IndexToolVerificationResult verificationResult = new
IndexToolVerificationResult(ts);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java
new file mode 100644
index 0000000000..692e65f032
--- /dev/null
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.AccessControlClient;
+import org.apache.hadoop.hbase.security.access.Permission;
+import org.apache.hadoop.hbase.security.access.UserPermission;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+
+import java.security.PrivilegedExceptionAction;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.PreparedStatement;
+
+import java.util.Collections;
+import java.util.Properties;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import static org.apache.phoenix.end2end.BasePermissionsIT.updateACLs;
+import static
org.apache.phoenix.end2end.BasePermissionsIT.enablePhoenixHBaseAuthorization;
+import static
org.apache.phoenix.end2end.BasePermissionsIT.configureNamespacesOnServer;
+import static
org.apache.phoenix.end2end.BasePermissionsIT.configureStatsConfigurations;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class AsyncIndexPermissionIT extends BaseTest{
+
+// static HBaseTestingUtility testUtil;
+
+ private static final String SUPER_USER = System.getProperty("user.name");
+
+ boolean isNamespaceMapped;
+
+ // Super User has all the access
+ protected static User superUser1 = null;
+
+ // Regular users are granted and revoked permissions as needed
+ protected User regularUser1 = null;
+
+ public AsyncIndexPermissionIT() throws Exception {
+ isNamespaceMapped = true;
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ if (null != utility) {
+ utility.shutdownMiniCluster();
+ utility = null;
+ }
+
+ enablePhoenixHBaseAuthorization(config, false);
+ configureNamespacesOnServer(config, true);
+ configureStatsConfigurations(config);
+ config.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true);
+
+
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
Boolean.toString(true));
+
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
Boolean.toString(true));
+
+ utility = new HBaseTestingUtility(config);
+
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
+ new ReadOnlyProps(clientProps.entrySet().iterator()));
+
+ superUser1 = User.createUserForTesting(config, SUPER_USER, new
String[0]);
+ }
+
+ @Before
+ public void initUsersAndTables() {
+ regularUser1 = User.createUserForTesting(config, "regularUser1_"
+ + generateUniqueName(), new String[0]);
+ }
+
+ private BasePermissionsIT.AccessTestAction createIndex(final String
indexName, final String dataTable, final String columns) throws SQLException {
+ return new BasePermissionsIT.AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement();) {
+ String indexStmtSQL = "CREATE index " + indexName + " on "
+ dataTable + " (" + columns +")";
+ assertFalse(stmt.execute(indexStmtSQL));
+ }
+ return null;
+ }
+ };
+ }
+
+ private BasePermissionsIT.AccessTestAction createIndexAsync(final String
indexName, final String schema, final String tableName, final String columns,
final int status) throws SQLException {
+ return new BasePermissionsIT.AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ final String dataTable = SchemaUtil.getTableName(schema,
tableName);
+ try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement();) {
+ String indexStmtSQL = "CREATE index " + indexName + " on "
+ dataTable + " (" + columns +") ASYNC";
+ assertFalse(stmt.execute(indexStmtSQL));
+ }
+ try {
+ IndexToolIT.runIndexTool(false, schema, tableName,
indexName, null, status, "-op", "/tmp/regular_User1_dir");
+ } catch (Exception ignored) {
+ // Running the indexTool might fail because of
AccessDeniedException
+ }
+ return null;
+ }
+ };
+ }
+
+ @Test(timeout = 80000)
+ public void testCreateIndex() throws Throwable {
+ final String schema = generateUniqueName();
+ final String tableName = generateUniqueName();
+ verifyAllowed(createSchema(schema), superUser1);
+ grantPermissions(regularUser1.getShortName(), schema,
Permission.Action.WRITE,
+ Permission.Action.READ, Permission.Action.EXEC,
Permission.Action.ADMIN);
+ grantPermissions(regularUser1.getShortName(), "SYSTEM",
Permission.Action.READ, Permission.Action.EXEC);
+ grantPermissions(regularUser1.getShortName(),
Collections.singleton("SYSTEM:CATALOG"), Permission.Action.EXEC,
Permission.Action.WRITE, Permission.Action.READ);
+
+ Path workDir = new Path("/tmp/regular_User1_dir");
+ FileSystem fs = workDir.getFileSystem(config);
+
+ fs.mkdirs(workDir, FsPermission.valueOf("-rwxrwxrwx"));
+
+ fs.setOwner(workDir, regularUser1.getShortName(), "");
+
+ superUser1.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Admin admin = utility.getAdmin();
+ TableDescriptorBuilder tdb =
TableDescriptorBuilder.newBuilder(TableName.valueOf(schema + ":" + tableName));
+ ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build();
+ tdb.setColumnFamily(cfd);
+ TableDescriptor td = tdb.build();
+ admin.createTable(td);
+ return null;
+ }
+ });
+
+ List<UserPermission> perms =
AccessControlClient.getUserPermissions(getUtility().getConnection(),
SYSTEM_CATALOG_NAME, regularUser1.getShortName());
+ for (UserPermission perm : perms) {
+ Assert.assertEquals(perm.getUser(), regularUser1.getShortName() );
+ Permission.Action[] actions = { Permission.Action.READ,
Permission.Action.WRITE, Permission.Action.EXEC };
+ Assert.assertEquals(perm.getPermission().getActions(), actions );
+ }
+
+ verifyAllowed(createTable(SchemaUtil.getTableName(schema, tableName),
2), regularUser1);
+ verifyAllowed(createIndex("ind1", SchemaUtil.getTableName(schema,
tableName), "PK"), regularUser1);
+
+ String ind3name = "IND3";
+ regularUser1.runAs(createIndexAsync(ind3name, schema, tableName, "PK",
0));
+
+ validateIndex(ind3name, schema, "a");
+ }
+
+ private void validateIndex(String ind3name, String schema, String
expectedStatus) throws SQLException {
+ String sql = "SELECT " +
"TABLE_SCHEM,TABLE_NAME,TABLE_TYPE,INDEX_STATE" + " FROM " + SYSTEM_CATALOG_NAME
+ + " WHERE TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' and
TABLE_TYPE = 'i'";
+ ResultSet rs =
getConnection().createStatement().executeQuery(String.format(sql, schema,
ind3name));
+ assertTrue(rs.next());
+ assertEquals(expectedStatus, rs.getString(4));
+ }
+
+ public Connection getConnection() throws SQLException {
+ return getConnection(null);
+ }
+
+ public Connection getConnection(String tenantId) throws SQLException {
+ return DriverManager.getConnection(getUrl(),
getClientProperties(tenantId));
+ }
+
+ private Properties getClientProperties(String tenantId) {
+ Properties props = new Properties();
+ if(tenantId != null) {
+ props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ }
+ props.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
Boolean.toString(isNamespaceMapped));
+ return props;
+ }
+
+ public BasePermissionsIT.AccessTestAction createSchema(final String
schemaName) throws SQLException {
+ return new BasePermissionsIT.AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ if (isNamespaceMapped) {
+ try (Connection conn = getConnection(); Statement stmt =
conn.createStatement();) {
+ assertFalse(stmt.execute("CREATE SCHEMA " +
schemaName));
+ }
+ }
+ return null;
+ }
+ };
+ }
+
+ BasePermissionsIT.AccessTestAction createTable(final String tableName, int
numRecordsToInsert) throws SQLException {
+ return new BasePermissionsIT.AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement();) {
+ assertFalse(stmt.execute("CREATE TABLE " + tableName +
"(pk INTEGER not null primary key, data VARCHAR, val integer)"));
+ try (PreparedStatement pstmt =
conn.prepareStatement("UPSERT INTO " + tableName + " values(?, ?, ?)")) {
+ for (int i = 0; i < numRecordsToInsert; i++) {
+ pstmt.setInt(1, i);
+ pstmt.setString(2, Integer.toString(i));
+ pstmt.setInt(3, i);
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ }
+ conn.commit();
+ }
+ return null;
+ }
+ };
+ }
+
+ /** This fails only in case of ADE or empty list for any of the users. */
+ public void verifyAllowed(BasePermissionsIT.AccessTestAction action,
User... users) throws Exception {
+ if(users.length == 0) {
+ throw new Exception("Action needs at least one user to run");
+ }
+ for (User user : users) {
+ verifyAllowed(user, action);
+ }
+ }
+
+ private void verifyAllowed(User user,
BasePermissionsIT.AccessTestAction... actions) throws Exception {
+ for (BasePermissionsIT.AccessTestAction action : actions) {
+ try {
+ Object obj = user.runAs(action);
+ if (obj != null && obj instanceof List<?>) {
+ List<?> results = (List<?>) obj;
+ if (results.isEmpty()) {
+ fail("Empty non null results from action for user '" +
user.getShortName() + "'");
+ }
+ }
+ } catch (AccessDeniedException ade) {
+ fail("Expected action to pass for user '" +
user.getShortName() + "' but was denied");
+ }
+ }
+ }
+
+ void grantPermissions(String toUser, String namespace,
Permission.Action... actions) throws Throwable {
+ updateACLs(getUtility(), new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ try {
+ AccessControlClient.grant(getUtility().getConnection(),
namespace, toUser, actions);
+
+ return null;
+ } catch (Throwable t) {
+ if (t instanceof Exception) {
+ throw (Exception) t;
+ } else {
+ throw new Exception(t);
+ }
+ }
+ }
+ });
+ }
+
+ void grantPermissions(String toUser, Set<String> tablesToGrant,
Permission.Action... actions)
+ throws Throwable {
+ updateACLs(getUtility(), new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ try {
+ for (String table : tablesToGrant) {
+
AccessControlClient.grant(getUtility().getConnection(),
TableName.valueOf(table),
+ toUser, null, null, actions);
+ }
+ return null;
+ } catch (Throwable t) {
+ if (t instanceof Exception) {
+ throw (Exception) t;
+ } else {
+ throw new Exception(t);
+ }
+ }
+ }
+ });
+ }
+}
\ No newline at end of file
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
index e0d3bc1b35..53d6d146d6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
@@ -224,8 +224,8 @@ public abstract class BasePermissionsIT extends BaseTest {
view2TableName = tableName + "_V2";
}
- private static void enablePhoenixHBaseAuthorization(Configuration config,
- boolean useCustomAccessController) {
+ static void enablePhoenixHBaseAuthorization(Configuration config,
+ boolean
useCustomAccessController) {
config.set("hbase.superuser", SUPER_USER + "," + "superUser2");
config.set("hbase.security.authorization", Boolean.TRUE.toString());
config.set("hbase.security.exec.permission.checks",
Boolean.TRUE.toString());
@@ -247,11 +247,11 @@ public abstract class BasePermissionsIT extends BaseTest {
"org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec");
}
- private static void configureNamespacesOnServer(Configuration conf, boolean
isNamespaceMapped) {
+ static void configureNamespacesOnServer(Configuration conf, boolean
isNamespaceMapped) {
conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
Boolean.toString(isNamespaceMapped));
}
- private static void configureStatsConfigurations(Configuration conf) {
+ static void configureStatsConfigurations(Configuration conf) {
conf.set(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
Long.toString(20));
conf.set(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
conf.set(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB,
Long.toString(5));
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index 3d03378dbb..c658210832 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -867,6 +867,14 @@ public class IndexToolIT extends BaseTest {
String indxTable, String tenantId, IndexTool.IndexVerifyType verifyType,
Long startTime,
Long endTime, IndexTool.IndexDisableLoggingType disableLoggingType, Long
incrementalVerify,
boolean useIndexTableAsSource) {
+ return getArgList(useSnapshot, schemaName, dataTable, indxTable,
tenantId,verifyType, startTime,
+ endTime, disableLoggingType, incrementalVerify,
useIndexTableAsSource, "/tmp/" + UUID.randomUUID().toString());
+ }
+
+ private static List<String> getArgList (boolean useSnapshot, String
schemaName, String dataTable,
+ String indxTable, String tenantId, IndexTool.IndexVerifyType verifyType,
Long startTime,
+ Long endTime, IndexTool.IndexDisableLoggingType disableLoggingType, Long
incrementalVerify,
+ boolean useIndexTableAsSource, String outputPath) {
List<String> args = Lists.newArrayList();
if (schemaName != null) {
args.add("--schema=" + schemaName);
@@ -911,7 +919,7 @@ public class IndexToolIT extends BaseTest {
}
args.add("-op");
- args.add("/tmp/" + UUID.randomUUID().toString());
+ args.add(outputPath);
return args;
}
@@ -930,6 +938,14 @@ public class IndexToolIT extends BaseTest {
return args.toArray(new String[0]);
}
+ public static String[] getArgValues(boolean useSnapshot, String schemaName,
String dataTable,
+ String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType,
+ IndexTool.IndexDisableLoggingType disableLoggingType, String outputPath) {
+ List<String> args = getArgList(useSnapshot, schemaName, dataTable,
indexTable,
+ tenantId, verifyType, null, null, disableLoggingType, null, false,
outputPath);
+ return args.toArray(new String[0]);
+ }
+
public static String[] getArgValues(boolean useSnapshot, String schemaName,
String dataTable,
String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType,
Long startTime,
Long endTime) {
@@ -1018,12 +1034,34 @@ public class IndexToolIT extends BaseTest {
IndexTool indexingTool = new IndexTool();
conf.set(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
indexingTool.setConf(conf);
- final String[] cmdArgs = getArgValues(useSnapshot, schemaName,
dataTableName, indexTableName,
- tenantId, verifyType, disableLoggingType);
+ boolean additionalArgsContainPath = false;
+ String path = "";
+ List<String> newadditionalArgs = Lists.newArrayList();
+ for (String arg : additionalArgs){
+ if (additionalArgsContainPath == true) {
+ path = arg;
+ }
+ else if ( arg.equals("-op") || arg.equals("-output-path")) {
+ additionalArgsContainPath = true;
+ }
+ else {
+ newadditionalArgs.add(arg);
+ }
+ }
+ additionalArgs = newadditionalArgs.toArray(new String[0]);
+
+ String[] cmdArgs;
+ if (additionalArgsContainPath) {
+ cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
+ indexTableName, tenantId, verifyType, disableLoggingType, path);
+ }
+ else {
+ cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
+ indexTableName, tenantId, verifyType, disableLoggingType);
+ }
List<String> cmdArgList = new ArrayList<>(Arrays.asList(cmdArgs));
cmdArgList.addAll(Arrays.asList(additionalArgs));
- LOGGER.info("Running IndexTool with {}",
Arrays.toString(cmdArgList.toArray()),
- new Exception("Stack Trace"));
+ LOGGER.info("Running IndexTool with {}",
Arrays.toString(cmdArgList.toArray()), new Exception("Stack Trace"));
int status = indexingTool.run(cmdArgList.toArray(new
String[cmdArgList.size()]));
if (expectedStatus == 0) {
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java
index e54f9715e9..56adb57673 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java
@@ -73,22 +73,13 @@ public abstract class LoadSystemTableSnapshotBase extends
BaseTest {
private static final Logger LOGGER =
LoggerFactory.getLogger(LoadSystemTableSnapshotBase.class);
- public static final String SNAPSHOT_DIR = "snapshots4_7/";
- public static String rootDir;
+ public static String SNAPSHOT_DIR;
+ public static String rootDir;
- private static final HashMap<String, String> SNAPSHOTS_TO_LOAD;
+ private static HashMap<String, String> SNAPSHOTS_TO_LOAD;
public static final byte[] MUTEX_LOCKED =
"MUTEX_LOCKED".getBytes(StandardCharsets.UTF_8);
- static {
- SNAPSHOTS_TO_LOAD = new HashMap<>();
- // Add any HBase tables, including Phoenix System tables
-
- SNAPSHOTS_TO_LOAD.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG");
- SNAPSHOTS_TO_LOAD.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION");
- SNAPSHOTS_TO_LOAD.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE");
- SNAPSHOTS_TO_LOAD.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS");
- }
private static void decompress(String in, File out) throws IOException {
try (TarArchiveInputStream fin = new TarArchiveInputStream(new
FileInputStream(in))) {
@@ -108,12 +99,24 @@ public abstract class LoadSystemTableSnapshotBase extends
BaseTest {
}
public static synchronized void setupCluster(boolean
createBlockUpgradeMutex) throws Exception {
+ //Add any HBase tables, including Phoenix System tables
+ HashMap<String, String> snapshotsToLoad = new HashMap<>();
+ snapshotsToLoad.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG");
+ snapshotsToLoad.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION");
+ snapshotsToLoad.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE");
+ snapshotsToLoad.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS");
+
+ setupCluster(createBlockUpgradeMutex, "snapshots47.tar.gz",
"snapshots4_7/", snapshotsToLoad, "true");
+ }
+
+ public static synchronized void setupCluster(boolean
createBlockUpgradeMutex, String tarName, String snapshotDir, HashMap<String,
String> snapshotsToLoad, String nameSpaceMapping) throws Exception {
+ SNAPSHOT_DIR = snapshotDir;
+ SNAPSHOTS_TO_LOAD = snapshotsToLoad;
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
- serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
- QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
- serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
nameSpaceMapping);
Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
- clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
nameSpaceMapping);
// Start minicluster without Phoenix first
checkClusterInitialized(new
ReadOnlyProps(serverProps.entrySet().iterator()));
@@ -121,7 +124,7 @@ public abstract class LoadSystemTableSnapshotBase extends
BaseTest {
URL folderUrl =
LoadSystemTableSnapshotBase.class.getClassLoader().getResource(SNAPSHOT_DIR);
// extract the tar
- File archive = new File(folderUrl.getFile() + "snapshots47.tar.gz");
+ File archive = new File(folderUrl.getFile() + tarName);
File destination = new File(folderUrl.getFile());
decompress(archive.toString(), destination);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 9109e837f5..3436026410 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -63,6 +63,11 @@ import org.junit.experimental.categories.Category;
@Category(NeedsOwnMiniClusterTest.class)
public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
+ private static final Set<String> HBASE_SYSTEM_TABLES = new
HashSet<>(Arrays.asList(
+ "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL"));
+ private static final Set<String> HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES = new
HashSet<>(
+ Arrays.asList("SYSTEM:PHOENIX_INDEX_TOOL_RESULT",
"SYSTEM:PHOENIX_INDEX_TOOL"));
+
private static final Set<String> PHOENIX_SYSTEM_TABLES =
new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE",
"SYSTEM.STATS",
"SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK",
"SYSTEM.TASK",
@@ -191,7 +196,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
});
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() +
HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
try {
@@ -209,7 +214,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
}
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() +
HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
}
@@ -227,7 +232,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
});
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() +
HBASE_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
user2.doAs(new PrivilegedExceptionAction<Void>() {
@@ -240,7 +245,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
});
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() +
HBASE_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
try {
@@ -260,7 +265,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
}
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() +
HBASE_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
user2.doAs(new PrivilegedExceptionAction<Void>() {
@@ -273,7 +278,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
});
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() +
HBASE_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES));
user3.doAs(new PrivilegedExceptionAction<Void>() {
@@ -285,7 +290,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
});
hbaseTables = getHBaseTables();
- assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
+ assertTrue(hbaseTables.size() ==
PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() +
HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size());
assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java
index 4541a51054..4f3b2ea625 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java
@@ -67,7 +67,8 @@ public class SkipSystemTablesExistenceCheckIT {
private static final Set<String> PHOENIX_SYSTEM_TABLES =
new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE",
"SYSTEM.STATS",
"SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK",
"SYSTEM.TASK",
- "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+ "SYSTEM.TRANSFORM","SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM",
+ "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL"));
private static class PhoenixSystemTablesCreationTestDriver extends
PhoenixTestDriver {
private static ConnectionQueryServices cqs;
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
index 819feebcaa..52fa0d953a 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
@@ -70,12 +70,15 @@ import org.apache.phoenix.util.UpgradeUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.junit.runners.MethodSorters;
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@Category(NeedsOwnMiniClusterTest.class)
public class SystemTablesCreationOnConnectionIT {
@@ -104,12 +107,14 @@ public class SystemTablesCreationOnConnectionIT {
private static final Set<String> PHOENIX_SYSTEM_TABLES =
new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE",
"SYSTEM.STATS",
"SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK",
"SYSTEM.TASK",
- "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+ "SYSTEM.TRANSFORM","SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM",
+ "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL"));
- private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES =
+ private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES =
new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE",
"SYSTEM:STATS",
"SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK",
"SYSTEM:TASK",
- "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+ "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM",
+ "SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL"));
private static class PhoenixSysCatCreationServices extends
ConnectionQueryServicesImpl {
@@ -781,7 +786,6 @@ public class SystemTablesCreationOnConnectionIT {
hbaseTables = getHBaseTables();
assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
assertEquals(0, countUpgradeAttempts);
- assertFalse(isSystemNamespaceCreated());
return driver;
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java
new file mode 100644
index 0000000000..ed36b1c840
--- /dev/null
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+
+import org.apache.phoenix.mapreduce.index.IndexToolTableUtil;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+import static
org.apache.phoenix.mapreduce.index.IndexToolTableUtil.RESULT_TABLE_NAME;
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+
+public class UpgradeIndexToolTablesIT extends LoadSystemTableSnapshotBase {
+ protected String nameSpaceMapping = "true";
+
+ @Before
+ public synchronized void doSetup() throws Exception {
+ setupCluster(nameSpaceMapping);
+ }
+
+ public synchronized void setupCluster(String nameSpaceMappingEnabled)
throws Exception {
+ HashMap<String, String> snapshotsToLoad = new HashMap<>();
+ snapshotsToLoad.put("phoenixtoolresultsnapshot",
"PHOENIX_INDEX_TOOL_RESULT");
+ setupCluster(false, "indexToolsnapshot.tar.gz",
"indexToolResultSnapshot/", snapshotsToLoad, nameSpaceMappingEnabled);
+ }
+
+ @Test
+ public void testPhoenixUpgradeIndexToolTables() throws Exception {
+ try (Admin admin = utility.getAdmin()) {
+ // we load the RESULT_TABLE_NAME from snapshot
+
assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME)));
+
assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_FULL_NAME)));
+ // we don't load the OUTPUT_TABLE_NAME
+
assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME)));
+
assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME)));
+ }
+
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
nameSpaceMapping);
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
nameSpaceMapping);
+
+
+ //Now we can start Phoenix
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
new ReadOnlyProps(clientProps.entrySet()
+ .iterator()));
+ assertTrue(true);
+
+
+ // Check the IndexTool Tables after upgrade
+ try (Admin admin = utility.getAdmin()) {
+
assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME)));
+
assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME)));
+
assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_FULL_NAME)));
+
assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_FULL_NAME)));
+ }
+
+ String tableName = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME,
RESULT_TABLE_NAME);
+ if (nameSpaceMapping.equals("true")) {
+ assertEquals(IndexToolTableUtil.RESULT_TABLE_FULL_NAME,
tableName.replace(QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR));
+ } else {
+ assertEquals(IndexToolTableUtil.RESULT_TABLE_FULL_NAME, tableName);
+ }
+
+ }
+
+}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java
new file mode 100644
index 0000000000..f339550d61
--- /dev/null
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.junit.Before;
+
+
+public class UpgradeIndexToolTablesNameSpaceMappingDisabledIT extends
UpgradeIndexToolTablesIT {
+
+ @Override
+ @Before
+ public synchronized void doSetup() throws Exception {
+ nameSpaceMapping = "false";
+ setupCluster(nameSpaceMapping);
+ }
+}
diff --git
a/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz
b/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz
new file mode 100644
index 0000000000..9d75746276
Binary files /dev/null and
b/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz
differ
