This is an automated email from the ASF dual-hosted git repository. leesf pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 6c7271f60a [HUDI-5377] Write call stack information to lock file (#7440) 6c7271f60a is described below commit 6c7271f60a4bd398c469a44dd5efb9d31f068a2b Author: HunterXHunter <1356469...@qq.com> AuthorDate: Sun Dec 18 11:46:25 2022 +0800 [HUDI-5377] Write call stack information to lock file (#7440) --- .../lock/FileSystemBasedLockProvider.java | 44 +++++++++++++- .../hudi/client/transaction/lock/LockInfo.java | 67 ++++++++++++++++++++++ .../hudi/client/transaction/lock/LockManager.java | 2 +- .../org/apache/hudi/common/lock/LockProvider.java | 4 ++ 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java index 4135ef9acd..efa644f4b0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java @@ -20,6 +20,8 @@ package org.apache.hudi.client.transaction.lock; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hudi.common.config.LockConfiguration; @@ -27,6 +29,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.lock.LockProvider; import org.apache.hudi.common.lock.LockState; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieWriteConfig; @@ -37,6 +40,7 @@ import org.apache.log4j.Logger; import java.io.IOException; import java.io.Serializable; +import java.text.SimpleDateFormat; import java.util.concurrent.TimeUnit; import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY; @@ -50,13 +54,14 @@ import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PA public class FileSystemBasedLockProvider implements LockProvider<String>, Serializable { private static final Logger LOG = LogManager.getLogger(FileSystemBasedLockProvider.class); - private static final String LOCK_FILE_NAME = "lock"; - private final int lockTimeoutMinutes; private final transient FileSystem fs; private final transient Path lockFile; protected LockConfiguration lockConfiguration; + private SimpleDateFormat sdf; + private LockInfo lockInfo; + private String currentOwnerLockInfo; public FileSystemBasedLockProvider(final LockConfiguration lockConfiguration, final Configuration configuration) { checkRequiredProps(lockConfiguration); @@ -68,6 +73,8 @@ public class FileSystemBasedLockProvider implements LockProvider<String>, Serial } this.lockTimeoutMinutes = lockConfiguration.getConfig().getInteger(FILESYSTEM_LOCK_EXPIRE_PROP_KEY); this.lockFile = new Path(lockDirectory + Path.SEPARATOR + LOCK_FILE_NAME); + this.lockInfo = new LockInfo(); + this.sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); this.fs = FSUtils.getFs(this.lockFile.toString(), configuration); } @@ -92,6 +99,7 @@ public class FileSystemBasedLockProvider implements LockProvider<String>, Serial fs.delete(this.lockFile, true); LOG.warn("Delete expired lock file: " + this.lockFile); } else { + reloadCurrentOwnerLockInfo(); return false; } } @@ -122,6 +130,11 @@ public class FileSystemBasedLockProvider implements LockProvider<String>, Serial return this.lockFile.toString(); } + @Override + public String getCurrentOwnerLockInfo() { + return currentOwnerLockInfo; + } + private boolean checkIfExpired() { if (lockTimeoutMinutes == 0) { return false; @@ -139,7 +152,32 @@ public class FileSystemBasedLockProvider implements LockProvider<String>, Serial private void acquireLock() { try { - fs.create(this.lockFile, false).close(); + if (!fs.exists(this.lockFile)) { + FSDataOutputStream fos = fs.create(this.lockFile, false); + initLockInfo(); + fos.writeBytes(lockInfo.toString()); + fos.close(); + } + } catch (IOException e) { + throw new HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e); + } + } + + public void initLockInfo() { + lockInfo.setLockCreateTime(sdf.format(System.currentTimeMillis())); + lockInfo.setLockThreadName(Thread.currentThread().getName()); + lockInfo.setLockStacksInfo(Thread.currentThread().getStackTrace()); + } + + public void reloadCurrentOwnerLockInfo() { + try { + if (fs.exists(this.lockFile)) { + FSDataInputStream fis = fs.open(this.lockFile); + this.currentOwnerLockInfo = FileIOUtils.readAsUTFString(fis); + fis.close(); + } else { + this.currentOwnerLockInfo = ""; + } } catch (IOException e) { throw new HoodieIOException(generateLogStatement(LockState.FAILED_TO_ACQUIRE), e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java new file mode 100644 index 0000000000..8d7df08f22 --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.client.transaction.lock; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.hudi.common.util.JsonUtils; + +import java.util.ArrayList; + +public class LockInfo { + private String lockCreateTime; + private String lockThreadName; + private ArrayList<String> lockStacksInfo; + + public String getLockCreateTime() { + return lockCreateTime; + } + + public void setLockCreateTime(String lockCreateTime) { + this.lockCreateTime = lockCreateTime; + } + + public String getLockThreadName() { + return lockThreadName; + } + + public void setLockThreadName(String lockThreadName) { + this.lockThreadName = lockThreadName; + } + + public ArrayList<String> getLockStacksInfo() { + return lockStacksInfo; + } + + public void setLockStacksInfo(StackTraceElement[] stacks) { + lockStacksInfo = new ArrayList<>(); + for (StackTraceElement ste : stacks) { + lockStacksInfo.add(String.format("%s.%s (%s:%s)", ste.getClassName(), ste.getMethodName(), ste.getFileName(), ste.getLineNumber())); + } + } + + @Override + public String toString() { + try { + return JsonUtils.getObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(this); + } catch (JsonProcessingException e) { + return e.toString(); + } + } +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java index 2c5a884638..2697700036 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java @@ -76,7 +76,7 @@ public class LockManager implements Serializable, AutoCloseable { break; } metrics.updateLockNotAcquiredMetric(); - LOG.info("Retrying to acquire lock..."); + LOG.info("Retrying to acquire lock. Current lock owner information : " + lockProvider.getCurrentOwnerLockInfo()); Thread.sleep(maxWaitTimeInMs); } catch (HoodieLockException | InterruptedException e) { metrics.updateLockNotAcquiredMetric(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java b/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java index 7d8e527384..a56a96a91e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java @@ -50,6 +50,10 @@ public interface LockProvider<T> extends Lock, AutoCloseable { throw new IllegalArgumentException(); } + default String getCurrentOwnerLockInfo() { + return ""; + } + @Override default void close() { }