This is an automated email from the ASF dual-hosted git repository.
leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6c7271f60a [HUDI-5377] Write call stack information to lock file
(#7440)
6c7271f60a is described below
commit 6c7271f60a4bd398c469a44dd5efb9d31f068a2b
Author: HunterXHunter <[email protected]>
AuthorDate: Sun Dec 18 11:46:25 2022 +0800
[HUDI-5377] Write call stack information to lock file (#7440)
---
.../lock/FileSystemBasedLockProvider.java | 44 +++++++++++++-
.../hudi/client/transaction/lock/LockInfo.java | 67 ++++++++++++++++++++++
.../hudi/client/transaction/lock/LockManager.java | 2 +-
.../org/apache/hudi/common/lock/LockProvider.java | 4 ++
4 files changed, 113 insertions(+), 4 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java
index 4135ef9acd..efa644f4b0 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java
@@ -20,6 +20,8 @@
package org.apache.hudi.client.transaction.lock;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.config.LockConfiguration;
@@ -27,6 +29,7 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.lock.LockProvider;
import org.apache.hudi.common.lock.LockState;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -37,6 +40,7 @@ import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.Serializable;
+import java.text.SimpleDateFormat;
import java.util.concurrent.TimeUnit;
import static
org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY;
@@ -50,13 +54,14 @@ import static
org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PA
public class FileSystemBasedLockProvider implements LockProvider<String>,
Serializable {
private static final Logger LOG =
LogManager.getLogger(FileSystemBasedLockProvider.class);
-
private static final String LOCK_FILE_NAME = "lock";
-
private final int lockTimeoutMinutes;
private final transient FileSystem fs;
private final transient Path lockFile;
protected LockConfiguration lockConfiguration;
+ private SimpleDateFormat sdf;
+ private LockInfo lockInfo;
+ private String currentOwnerLockInfo;
public FileSystemBasedLockProvider(final LockConfiguration
lockConfiguration, final Configuration configuration) {
checkRequiredProps(lockConfiguration);
@@ -68,6 +73,8 @@ public class FileSystemBasedLockProvider implements
LockProvider<String>, Serial
}
this.lockTimeoutMinutes =
lockConfiguration.getConfig().getInteger(FILESYSTEM_LOCK_EXPIRE_PROP_KEY);
this.lockFile = new Path(lockDirectory + Path.SEPARATOR + LOCK_FILE_NAME);
+ this.lockInfo = new LockInfo();
+ this.sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
this.fs = FSUtils.getFs(this.lockFile.toString(), configuration);
}
@@ -92,6 +99,7 @@ public class FileSystemBasedLockProvider implements
LockProvider<String>, Serial
fs.delete(this.lockFile, true);
LOG.warn("Delete expired lock file: " + this.lockFile);
} else {
+ reloadCurrentOwnerLockInfo();
return false;
}
}
@@ -122,6 +130,11 @@ public class FileSystemBasedLockProvider implements
LockProvider<String>, Serial
return this.lockFile.toString();
}
+ @Override
+ public String getCurrentOwnerLockInfo() {
+ return currentOwnerLockInfo;
+ }
+
private boolean checkIfExpired() {
if (lockTimeoutMinutes == 0) {
return false;
@@ -139,7 +152,32 @@ public class FileSystemBasedLockProvider implements
LockProvider<String>, Serial
private void acquireLock() {
try {
- fs.create(this.lockFile, false).close();
+ if (!fs.exists(this.lockFile)) {
+ FSDataOutputStream fos = fs.create(this.lockFile, false);
+ initLockInfo();
+ fos.writeBytes(lockInfo.toString());
+ fos.close();
+ }
+ } catch (IOException e) {
+ throw new
HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e);
+ }
+ }
+
+ public void initLockInfo() {
+ lockInfo.setLockCreateTime(sdf.format(System.currentTimeMillis()));
+ lockInfo.setLockThreadName(Thread.currentThread().getName());
+ lockInfo.setLockStacksInfo(Thread.currentThread().getStackTrace());
+ }
+
+ public void reloadCurrentOwnerLockInfo() {
+ try {
+ if (fs.exists(this.lockFile)) {
+ FSDataInputStream fis = fs.open(this.lockFile);
+ this.currentOwnerLockInfo = FileIOUtils.readAsUTFString(fis);
+ fis.close();
+ } else {
+ this.currentOwnerLockInfo = "";
+ }
} catch (IOException e) {
throw new
HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e);
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java
new file mode 100644
index 0000000000..8d7df08f22
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.client.transaction.lock;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.hudi.common.util.JsonUtils;
+
+import java.util.ArrayList;
+
+public class LockInfo {
+ private String lockCreateTime;
+ private String lockThreadName;
+ private ArrayList<String> lockStacksInfo;
+
+ public String getLockCreateTime() {
+ return lockCreateTime;
+ }
+
+ public void setLockCreateTime(String lockCreateTime) {
+ this.lockCreateTime = lockCreateTime;
+ }
+
+ public String getLockThreadName() {
+ return lockThreadName;
+ }
+
+ public void setLockThreadName(String lockThreadName) {
+ this.lockThreadName = lockThreadName;
+ }
+
+ public ArrayList<String> getLockStacksInfo() {
+ return lockStacksInfo;
+ }
+
+ public void setLockStacksInfo(StackTraceElement[] stacks) {
+ lockStacksInfo = new ArrayList<>();
+ for (StackTraceElement ste : stacks) {
+ lockStacksInfo.add(String.format("%s.%s (%s:%s)", ste.getClassName(),
ste.getMethodName(), ste.getFileName(), ste.getLineNumber()));
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return
JsonUtils.getObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ return e.toString();
+ }
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
index 2c5a884638..2697700036 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
@@ -76,7 +76,7 @@ public class LockManager implements Serializable,
AutoCloseable {
break;
}
metrics.updateLockNotAcquiredMetric();
- LOG.info("Retrying to acquire lock...");
+ LOG.info("Retrying to acquire lock. Current lock owner information :
" + lockProvider.getCurrentOwnerLockInfo());
Thread.sleep(maxWaitTimeInMs);
} catch (HoodieLockException | InterruptedException e) {
metrics.updateLockNotAcquiredMetric();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java
b/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java
index 7d8e527384..a56a96a91e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java
@@ -50,6 +50,10 @@ public interface LockProvider<T> extends Lock, AutoCloseable
{
throw new IllegalArgumentException();
}
+ default String getCurrentOwnerLockInfo() {
+ return "";
+ }
+
@Override
default void close() {
}
