This is an automated email from the ASF dual-hosted git repository.
luochen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ec22e7a [NO ISSUE][TXN] Increase lock table size
ec22e7a is described below
commit ec22e7aba36ccdfb150865498af36624193fb810
Author: luochen <[email protected]>
AuthorDate: Mon Oct 5 17:05:14 2020 -0700
[NO ISSUE][TXN] Increase lock table size
- user model changes: no
- storage format changes: no
- interface changes: yes.
Details:
- Introduce a new parameter txn.lock.table.size to configure the lock table
size. The default value is set at 1024*1024 to avoid hash collisions.
- Modify conf files used in the test cases to use a small lock table size
(1000) to avoid memory overflow.
Change-Id: I19e2af32af62685bcc613758fa468e58908dbe97
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8003
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../main/java/org/apache/asterix/app/nc/TransactionSubsystem.java | 3 ++-
asterixdb/asterix-app/src/main/resources/cc-rep.conf | 3 ++-
asterixdb/asterix-app/src/main/resources/cc.conf | 2 +-
asterixdb/asterix-app/src/main/resources/cc2.conf | 1 +
asterixdb/asterix-app/src/main/resources/cc3.conf | 1 +
asterixdb/asterix-app/src/main/resources/cc4.conf | 1 +
asterixdb/asterix-app/src/test/resources/cc-compression.conf | 1 +
asterixdb/asterix-app/src/test/resources/cc-multipart.conf | 1 +
asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf | 3 ++-
asterixdb/asterix-app/src/test/resources/cc-single.conf | 1 +
.../asterix-app/src/test/resources/cc-small-txn-log-partition.conf | 3 ++-
asterixdb/asterix-app/src/test/resources/cc-ssl.conf | 3 ++-
asterixdb/asterix-app/src/test/resources/cc-storage.conf | 3 ++-
asterixdb/asterix-app/src/test/resources/cc.conf | 1 +
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../org/apache/asterix/common/config/TransactionProperties.java | 5 +++++
.../management/service/locking/ConcurrentLockManager.java | 6 +++---
19 files changed, 31 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
index 79c87c0..938b16c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
@@ -65,7 +65,8 @@ public class TransactionSubsystem implements
ITransactionSubsystem {
this.id = appCtx.getServiceContext().getNodeId();
this.txnProperties = appCtx.getTransactionProperties();
this.transactionManager = new TransactionManager(this);
- this.lockManager = new
ConcurrentLockManager(txnProperties.getLockManagerShrinkTimer());
+ this.lockManager =
+ new
ConcurrentLockManager(txnProperties.getLockManagerShrinkTimer(),
txnProperties.getLockTableSize());
final ReplicationProperties repProperties =
appCtx.getReplicationProperties();
final boolean replicationEnabled =
repProperties.isReplicationEnabled();
final CheckpointProperties checkpointProperties = new
CheckpointProperties(txnProperties, id);
diff --git a/asterixdb/asterix-app/src/main/resources/cc-rep.conf
b/asterixdb/asterix-app/src/main/resources/cc-rep.conf
index 818b835..6e20dd7 100644
--- a/asterixdb/asterix-app/src/main/resources/cc-rep.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc-rep.conf
@@ -48,4 +48,5 @@ heartbeat.max.misses=25
[common]
log.level = INFO
replication.enabled=true
-replication.strategy=all
\ No newline at end of file
+replication.strategy=all
+txn.lock.table.size=1000
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf
b/asterixdb/asterix-app/src/main/resources/cc.conf
index d5da6d4..a8ff2a9 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -62,4 +62,4 @@ compiler.sort.parallel=false
compiler.internal.sanitycheck=true
messaging.frame.size=4096
messaging.frame.count=512
-
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf
b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 016c485..e5da5e3 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -57,3 +57,4 @@ compiler.windowmemory=192KB
compiler.parallelism=-1
messaging.frame.size=4096
messaging.frame.count=512
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf
b/asterixdb/asterix-app/src/main/resources/cc3.conf
index d2a8556..6d69b75 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -57,3 +57,4 @@ compiler.windowmemory=192KB
compiler.parallelism=3
messaging.frame.size=4096
messaging.frame.count=512
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf
b/asterixdb/asterix-app/src/main/resources/cc4.conf
index 6a66d25..f5025a5 100644
--- a/asterixdb/asterix-app/src/main/resources/cc4.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc4.conf
@@ -54,3 +54,4 @@ compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
compiler.parallelism=-1
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-compression.conf
b/asterixdb/asterix-app/src/test/resources/cc-compression.conf
index a3047a0..60b0197 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-compression.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-compression.conf
@@ -56,3 +56,4 @@ compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
storage.compression.block=snappy
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index 5543e51..b33221d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -54,3 +54,4 @@ compiler.joinmemory=256KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
b/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
index db4ca20..7146c22 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
@@ -58,4 +58,5 @@ trust.store.path=security/root/root.truststore
log.level = INFO
replication.enabled=true
replication.strategy=all
-ssl.enabled=true
\ No newline at end of file
+ssl.enabled=true
+txn.lock.table.size=1000
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single.conf
b/asterixdb/asterix-app/src/test/resources/cc-single.conf
index 794b5b0..875af51 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single.conf
@@ -48,3 +48,4 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
+txn.lock.table.size=1000
diff --git
a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index 562dedd..4091625 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -55,4 +55,5 @@ messaging.frame.count=512
txn.log.partitionsize=2MB
txn.log.buffer.pagesize=128KB
txn.log.checkpoint.pollfrequency=2147483647
-txn.log.checkpoint.history=0
\ No newline at end of file
+txn.log.checkpoint.history=0
+txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
index 1c0a68f..d67da13 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
@@ -67,4 +67,5 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
-ssl.enabled=true
\ No newline at end of file
+ssl.enabled=true
+txn.lock.table.size=1000
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc-storage.conf
b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
index 9e869c7..bc50c13 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
@@ -55,4 +55,5 @@ messaging.frame.size=4096
messaging.frame.count=512
storage.compression.block=snappy
txn.log.checkpoint.pollfrequency=10
-txn.dataset.checkpoint.interval=10
\ No newline at end of file
+txn.dataset.checkpoint.interval=10
+txn.lock.table.size=1000
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf
b/asterixdb/asterix-app/src/test/resources/cc.conf
index 158eaa0..119b53a 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -58,3 +58,4 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
+txn.lock.table.size=1000
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index cc96921..fc04b37 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -46,6 +46,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
+ "txn\.lock\.table\.size" : 1000,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index b9c2894..c4d58ed 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -46,6 +46,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
+ "txn\.lock\.table\.size" : 1000,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 68d3079..28b9e25 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -46,6 +46,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
+ "txn\.lock\.table\.size" : 1000,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
index 0de2c3d..1da9e44 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
@@ -73,6 +73,7 @@ public class TransactionProperties extends AbstractProperties
{
POSITIVE_INTEGER,
10000,
"Interval (in milliseconds) for checking lock " + "timeout"),
+ TXN_LOCK_TABLE_SIZE(POSITIVE_INTEGER, 1024 * 1024, "The number of
slots in the lock table."),
TXN_COMMITPROFILER_ENABLED(BOOLEAN, false, "Enable output of commit
profiler logs"),
TXN_COMMITPROFILER_REPORTINTERVAL(POSITIVE_INTEGER, 5, "Interval (in
seconds) to report commit profiler logs"),
TXN_JOB_RECOVERY_MEMORYSIZE(
@@ -159,6 +160,10 @@ public class TransactionProperties extends
AbstractProperties {
return accessor.getInt(Option.TXN_LOCK_SHRINKTIMER);
}
+ public int getLockTableSize() {
+ return accessor.getInt(Option.TXN_LOCK_TABLE_SIZE);
+ }
+
public int getTimeoutWaitThreshold() {
return accessor.getInt(Option.TXN_LOCK_TIMEOUT_WAITTHRESHOLD);
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
index 3f2412f..ddab169 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
@@ -74,6 +74,7 @@ public class ConcurrentLockManager implements ILockManager,
ILifeCycleComponent
WAIT(true, false),
CONV(true, true) // convert (upgrade) a lock (e.g. from S to X)
;
+
boolean wait;
boolean modify;
@@ -92,9 +93,8 @@ public class ConcurrentLockManager implements ILockManager,
ILifeCycleComponent
{ LockAction.ERR, LockAction.WAIT, LockAction.WAIT,
LockAction.WAIT, LockAction.WAIT } // X
};
- public ConcurrentLockManager(final int lockManagerShrinkTimer) throws
ACIDException {
- this(lockManagerShrinkTimer,
Runtime.getRuntime().availableProcessors() * 2, 1024);
- // TODO increase table size?
+ public ConcurrentLockManager(final int lockManagerShrinkTimer, int
tableSize) throws ACIDException {
+ this(lockManagerShrinkTimer,
Runtime.getRuntime().availableProcessors() * 2, tableSize);
}
public ConcurrentLockManager(final int lockManagerShrinkTimer, final int
noArenas, final int tableSize)
