Jacob Isaac created PHOENIX-7590:
------------------------------------
Summary: Threads deadlock when dropping column on view with
partial indexes
Key: PHOENIX-7590
URL: https://issues.apache.org/jira/browse/PHOENIX-7590
Project: Phoenix
Issue Type: Bug
Affects Versions: 5.2.0
Reporter: Jacob Isaac
To reproduce the scenario run the following test -
{{@Test}}
{{public void testDropColumnForMultiTenantTableWithIndex() throws Exception {}}
{{String table = generateUniqueName();}}
{{String view = generateUniqueName();}}
{{String index = generateUniqueName();}}
{{try (Connection conn = DriverManager.getConnection(getUrl())) {}}
{{conn.setAutoCommit(true);}}
{{try (Statement stmt = conn.createStatement()) {}}
{{stmt.execute("CREATE TABLE IF NOT EXISTS " + table +}}
{{" (ID CHAR(10) NOT NULL,KEY_PREFIX CHAR(3) NOT NULL, NON_PK3 CHAR(15)," +}}
{{" CONSTRAINT PK PRIMARY KEY (ID,KEY_PREFIX)) MULTI_TENANT=TRUE");}}
{{// create view}}
{{stmt.execute(}}
{{"CREATE VIEW IF NOT EXISTS " + view +}}
{{" (PK1 DATE NOT NULL,PK2 CHAR(15) NOT NULL," +}}
{{"NON_PK1 CHAR(15),NON_PK2 CHAR(15) CONSTRAINT PKVIEW PRIMARY KEY (PK1,PK2)) "
+}}
{{"AS SELECT * FROM " + table + " WHERE KEY_PREFIX = '123'");}}
{{stmt.execute("CREATE INDEX " + index + " ON " + view +}}
{{" (PK2, PK1) INCLUDE (NON_PK1, NON_PK2) WHERE KEY_PREFIX = '123'");}}
{{// drop column}}
{{stmt.execute("ALTER VIEW " + view + " DROP COLUMN NON_PK3");}}
{{}}}
{{}}}
{{}}}
{{sample log output}}
{{2025-04-25T09:57:27,361 DEBUG [master/localhost:0.Chore.1]
hbase.ScheduledChore(165): CatalogJanitor-localhost:51389 execution time: 32
ms.}}
{{2025-04-25T09:57:27,373 ERROR
[RpcServer.Metadata.Fifo.handler=1,queue=0,port=51391]
coprocessor.MetaDataEndpointImpl(767): getTable failed}}
{{java.io.IOException: Timed out waiting for lock for row: \x00\x00N000002 in
region 2d1454d78354b8b37e3da10a834b2b1d}}
{{ at
org.apache.hadoop.hbase.regionserver.HRegion.getRowLockInternal(HRegion.java:6496)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.regionserver.HRegion.lambda$getRowLock$21(HRegion.java:6526)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
~[hbase-common-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.regionserver.HRegion.getRowLock(HRegion.java:6526)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.regionserver.HRegion.getRowLock(HRegion.java:6433)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.acquireLock(MetaDataEndpointImpl.java:3245)
~[classes/:?]}}
{{ at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.doGetTable(MetaDataEndpointImpl.java:3972)
~[classes/:?]}}
{{ at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.getTable(MetaDataEndpointImpl.java:717)
~[classes/:?]}}
{{ at
org.apache.phoenix.coprocessor.generated.MetaDataProtos$MetaDataService.callMethod(MetaDataProtos.java:17524)
~[classes/:?]}}
{{ at
org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7930)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:2538)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:2512)
~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45014)
~[hbase-protocol-shaded-2.5.11-hadoop3.jar:2.5.11-hadoop3]}}
{{ at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:415)
~[hbase-server-2.5.11-hadoop3.jar:?]}}
{{ at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
~[hbase-server-2.5.11-hadoop3.jar:?]}}
{{ at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
~[hbase-server-2.5.11-hadoop3.jar:?]}}
{{ at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
~[hbase-server-2.5.11-hadoop3.jar:?]}}
{{2025-04-25T09:57:27,379 DEBUG
[RpcServer.Metadata.Fifo.handler=1,queue=0,port=51391]
ipc.MetricsHBaseServer(144): Unknown exception type}}
{{ja}}
{{jstack reveals -}}
"RpcServer.Metadata.Fifo.handler=0,queue=0,port=50960" #253 daemon prio=5
os_prio=31 tid=0x00007f9391451000 nid=0x2412b waiting on condition
[0x0000000317365000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000006ce6b4000> (a java.util.concurrent.FutureTask)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)
at java.util.concurrent.FutureTask.get(FutureTask.java:191)
at
org.apache.hadoop.hbase.client.HTable.lambda$coprocessorService$23(HTable.java:1151)
at org.apache.hadoop.hbase.client.HTable$$Lambda$667/1170482099.run(Unknown
Source)
at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
at org.apache.hadoop.hbase.client.HTable.coprocessorService(HTable.java:1128)
at org.apache.hadoop.hbase.client.HTable.coprocessorService(HTable.java:1114)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.metaDataCoprocessorExec(ConnectionQueryServicesImpl.java:2214)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.metaDataCoprocessorExec(ConnectionQueryServicesImpl.java:2192)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.getTable(ConnectionQueryServicesImpl.java:2465)
at
org.apache.phoenix.query.DelegateConnectionQueryServices.getTable(DelegateConnectionQueryServices.java:152)
at org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:694)
at org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:576)
at
org.apache.phoenix.compile.FromCompiler$BaseColumnResolver.createTableRef(FromCompiler.java:801)
at
org.apache.phoenix.compile.FromCompiler$SingleTableColumnResolver.<init>(FromCompiler.java:455)
at
org.apache.phoenix.compile.FromCompiler.getResolverForQuery(FromCompiler.java:234)
at
org.apache.phoenix.compile.FromCompiler.getResolverForQuery(FromCompiler.java:212)
at org.apache.phoenix.util.ParseNodeUtil.rewrite(ParseNodeUtil.java:177)
at
org.apache.phoenix.jdbc.PhoenixStatement$ExecutableSelectStatement.compilePlan(PhoenixStatement.java:841)
at
org.apache.phoenix.jdbc.PhoenixStatement$ExecutableSelectStatement.compilePlan(PhoenixStatement.java:814)
at
org.apache.phoenix.jdbc.PhoenixStatement.compileQuery(PhoenixStatement.java:2392)
at
org.apache.phoenix.jdbc.PhoenixPreparedStatement.compileQuery(PhoenixPreparedStatement.java:157)
at
org.apache.phoenix.schema.PTableImpl.buildIndexWhereExpression(PTableImpl.java:2535)
at
org.apache.phoenix.schema.PTableImpl.getIndexWhereExpression(PTableImpl.java:2548)
at org.apache.phoenix.index.IndexMaintainer.<init>(IndexMaintainer.java:699)
at org.apache.phoenix.index.IndexMaintainer.create(IndexMaintainer.java:161)
at org.apache.phoenix.schema.PTableImpl.getIndexMaintainer(PTableImpl.java:1809)
- locked <0x00000006cf120d00> (a org.apache.phoenix.schema.PTableImpl)
at org.apache.phoenix.schema.PTableImpl.getIndexMaintainer(PTableImpl.java:1802)
- locked <0x00000006cf120d00> (a org.apache.phoenix.schema.PTableImpl)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.dropIndexes(MetaDataEndpointImpl.java:4130)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.mutateColumn(MetaDataEndpointImpl.java:3680)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.dropColumn(MetaDataEndpointImpl.java:4092)
at
org.apache.phoenix.coprocessor.generated.MetaDataProtos$MetaDataService.callMethod(MetaDataProtos.java:17574)
at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7930)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:2538)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:2512)
at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45014)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:415)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Locked ownable synchronizers:
- <0x00000006ce8f3e88> (a
java.util.concurrent.locks.ReentrantReadWriteLock$FairSync)
"RpcServer.Metadata.Fifo.handler=2,queue=0,port=50960" #255 daemon prio=5
os_prio=31 tid=0x00007f935f0fa000 nid=0x26b03 waiting on condition
[0x000000031756c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000006ce8f3e88> (a
java.util.concurrent.locks.ReentrantReadWriteLock$FairSync)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at
java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:871)
at
org.apache.hadoop.hbase.regionserver.HRegion.getRowLockInternal(HRegion.java:6489)
at
org.apache.hadoop.hbase.regionserver.HRegion.lambda$getRowLock$21(HRegion.java:6526)
at
org.apache.hadoop.hbase.regionserver.HRegion$$Lambda$418/1918809096.call(Unknown
Source)
at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
at org.apache.hadoop.hbase.regionserver.HRegion.getRowLock(HRegion.java:6526)
at org.apache.hadoop.hbase.regionserver.HRegion.getRowLock(HRegion.java:6433)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.acquireLock(MetaDataEndpointImpl.java:3245)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.doGetTable(MetaDataEndpointImpl.java:3972)
at
org.apache.phoenix.coprocessor.MetaDataEndpointImpl.getTable(MetaDataEndpointImpl.java:717)
at
org.apache.phoenix.coprocessor.generated.MetaDataProtos$MetaDataService.callMethod(MetaDataProtos.java:17524)
at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7930)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:2538)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:2512)
at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45014)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:415)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Locked ownable synchronizers:
- None
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
