This is an automated email from the ASF dual-hosted git repository. nightowl888 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit fd664cd12afcdfbae9d7b3dab4a9bfcd89489221 Author: Shad Storhaug <[email protected]> AuthorDate: Thu Oct 24 20:19:18 2019 +0700 Lucene.Net.Store.NativeFSLockFactory, Lucene.Net.Support.IO.FileSupport: Changed implementations to provoke lock/sharing/alredy exists exceptions during initialization so the values of the current OS can be used for comparison of HResult values at runtime (closes LUCENENET-618) --- src/Lucene.Net/Store/NativeFSLockFactory.cs | 345 +++++++++++++++++++++------- src/Lucene.Net/Support/IO/FileSupport.cs | 81 ++++++- 2 files changed, 338 insertions(+), 88 deletions(-) diff --git a/src/Lucene.Net/Store/NativeFSLockFactory.cs b/src/Lucene.Net/Store/NativeFSLockFactory.cs index b5b65fc..b07b1af 100644 --- a/src/Lucene.Net/Store/NativeFSLockFactory.cs +++ b/src/Lucene.Net/Store/NativeFSLockFactory.cs @@ -54,8 +54,99 @@ namespace Lucene.Net.Store /// <see cref="LockVerifyServer"/> and <see cref="LockStressTest"/>.</para> /// </summary> /// <seealso cref="LockFactory"/> + // LUCENENET specific - this class has been refactored significantly from its Java counterpart + // to take advantage of .NET FileShare locking in the Windows and Linux environments. public class NativeFSLockFactory : FSLockFactory { + private enum FSLockingStrategy + { + FileStreamLockViolation, + FileSharingViolation, + Fallback + } + + // LUCENENET: This controls the locking strategy used for the current operating system and framework + private static FSLockingStrategy LockingStrategy + { + get + { + if (IS_FILESTREAM_LOCKING_PLATFORM && HRESULT_FILE_LOCK_VIOLATION.HasValue) + return FSLockingStrategy.FileStreamLockViolation; + else if (HRESULT_FILE_SHARE_VIOLATION.HasValue) + return FSLockingStrategy.FileSharingViolation; + else + // Fallback implementation for unknown platforms that don't rely on HResult + return FSLockingStrategy.Fallback; + } + } + + + // LUCNENENET NOTE: Lookup the HResult value we are interested in for the current OS + // by provoking the exception during initialization and caching its HResult value for later. + // We optimize for Windows because those HResult values are known and documented, but for + // other platforms, this is the only way we can reliably determine the HResult values + // we are interested in. + // + // Reference: https://stackoverflow.com/q/46380483 + private static bool IS_FILESTREAM_LOCKING_PLATFORM = LoadIsFileStreamLockingPlatform(); + + private const int WIN_HRESULT_FILE_LOCK_VIOLATION = unchecked((int)0x80070021); + private const int WIN_HRESULT_FILE_SHARE_VIOLATION = unchecked((int)0x80070020); + + internal static readonly int? HRESULT_FILE_LOCK_VIOLATION = LoadFileLockViolationHResult(); + internal static readonly int? HRESULT_FILE_SHARE_VIOLATION = LoadFileShareViolationHResult(); + + private static bool LoadIsFileStreamLockingPlatform() + { +#if FEATURE_FILESTREAM_LOCK + return Constants.WINDOWS; // LUCENENET: See: https://github.com/dotnet/corefx/issues/5964 +#else + return false; +#endif + } + + private static int? LoadFileLockViolationHResult() + { + if (Constants.WINDOWS) + return WIN_HRESULT_FILE_LOCK_VIOLATION; + + // Skip provoking the exception unless we know we will use the value + if (IS_FILESTREAM_LOCKING_PLATFORM) + { + return FileSupport.GetFileIOExceptionHResult(provokeException: (fileName) => + { + using (var lockStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)) + { + lockStream.Lock(0, 1); // Create an exclusive lock + using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) + { + // try to find out if the file is locked by writing a byte. Note that we need to flush the stream to find out. + stream.WriteByte(0); + stream.Flush(); // this *may* throw an IOException if the file is locked, but... + // ... closing the stream is the real test + } + } + }); + } + + return null; + } + + private static int? LoadFileShareViolationHResult() + { + if (Constants.WINDOWS) + return WIN_HRESULT_FILE_SHARE_VIOLATION; + + return FileSupport.GetFileIOExceptionHResult(provokeException: (fileName) => + { + using (var lockStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None, 1, FileOptions.None)) + // Try to get an exclusive lock on the file - this should throw an IOException with the current platform's HResult value for FileShare violation + using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Write, FileShare.None, 1, FileOptions.None)) + { + } + }); + } + /// <summary> /// Create a <see cref="NativeFSLockFactory"/> instance, with <c>null</c> (unset) /// lock directory. When you pass this factory to a <see cref="FSDirectory"/> @@ -119,21 +210,25 @@ namespace Lucene.Net.Store // Internal for testing internal virtual Lock NewLock(string path) { - if (Constants.WINDOWS) - return new WindowsNativeFSLock(this, m_lockDir, path); - - // Fallback implementation for unknown platforms that don't rely on HResult - return new NativeFSLock(this, m_lockDir, path); + switch (LockingStrategy) + { + case FSLockingStrategy.FileStreamLockViolation: + return new NativeFSLock(m_lockDir, path); + case FSLockingStrategy.FileSharingViolation: + return new SharingNativeFSLock(m_lockDir, path); + default: + // Fallback implementation for unknown platforms that don't rely on HResult + return new FallbackNativeFSLock(m_lockDir, path); + } } public override void ClearLock(string lockName) { var path = GetCanonicalPathOfLockFile(lockName); - Lock l; // this is the reason why we can't use ConcurrentDictionary: we need the removal and disposal of the lock to be atomic // otherwise it may clash with MakeLock making a lock and ClearLock disposing of it in another thread. lock (_locks) - if (_locks.TryGetValue(path, out l)) + if (_locks.TryGetValue(path, out Lock l)) { _locks.Remove(path); l.Dispose(); @@ -145,29 +240,19 @@ namespace Lucene.Net.Store // LUCENENET NOTE: We use this implementation as a fallback for platforms that we don't // know the HResult numbers for lock and file sharing errors. // - // Note that using SharingAwareNativeFSLock would be ideal for all platforms. However, - // at the time of this writing there is no cross-platform way to identify sharing errors - // or lock errors because the values of HResult depend on the specific OS platform we - // are running on. Unfortunately, researching what these numbers may be even for Linux and - // OSx turned up nothing, and it is also unclear whether different flavors of Linux/Unix will have - // different HResult numbers. The best we can hope for is for people to contribute subclasses - // of SharingAwareNativeFSLock for the most popular platforms and fall back to this (substandard) - // implementation for all of the other platforms. See WindowsNativeFSLock for an example of what - // one of these subclasses should look like. The NativeFSLockFactory.NewLock() factory method - // is the only place where adding the platform-specific logic which class to instantiate is required. + // Note that using NativeFSLock would be ideal for all platforms. However, there is a + // small chance that provoking lock/share exceptions will fail. In that rare case, we + // fallback to this substandard implementation. // // Reference: https://stackoverflow.com/q/46380483 - internal class NativeFSLock : Lock + internal class FallbackNativeFSLock : Lock { - private readonly NativeFSLockFactory outerInstance; - private FileStream channel; private readonly string path; private readonly DirectoryInfo lockDir; - public NativeFSLock(NativeFSLockFactory outerInstance, DirectoryInfo lockDir, string path) + public FallbackNativeFSLock(DirectoryInfo lockDir, string path) { - this.outerInstance = outerInstance; this.lockDir = lockDir; this.path = path; } @@ -323,28 +408,30 @@ namespace Lucene.Net.Store public override string ToString() { - return "NativeFSLock@" + path; + return "{nameof(FallbackNativeFSLock)}@{path}"; } } - // Abstract class that can be used to create additional native locks that - // work with OS-specific HResult values. - internal abstract class SharingAwareNativeFSLock : Lock + // Locks the entire file. macOS requires this approach. + internal class SharingNativeFSLock : Lock { - private readonly NativeFSLockFactory outerInstance; - private FileStream channel; private readonly string path; private readonly DirectoryInfo lockDir; - public SharingAwareNativeFSLock(NativeFSLockFactory outerInstance, DirectoryInfo lockDir, string path) + public SharingNativeFSLock(DirectoryInfo lockDir, string path) { - this.outerInstance = outerInstance; this.lockDir = lockDir; this.path = path; } - protected abstract bool IsLockOrShareViolation(IOException e); + /// <summary> + /// Return true if the <see cref="IOException"/> is the result of a share violation + /// </summary> + private bool IsShareViolation(IOException e) + { + return e.HResult == NativeFSLockFactory.HRESULT_FILE_SHARE_VIOLATION; + } private FileStream GetLockFileStream(FileMode mode) { @@ -367,11 +454,7 @@ namespace Lucene.Net.Store throw new IOException("Found regular file where directory expected: " + lockDir.FullName); } -#if FEATURE_FILESTREAM_LOCK - return new FileStream(path, mode, FileAccess.Write, FileShare.ReadWrite); -#else return new FileStream(path, mode, FileAccess.Write, FileShare.None, 1, mode == FileMode.Open ? FileOptions.None : FileOptions.DeleteOnClose); -#endif } public override bool Obtain() @@ -385,12 +468,13 @@ namespace Lucene.Net.Store // Our instance is already locked: return false; } - -#if FEATURE_FILESTREAM_LOCK - FileStream stream = null; try { - stream = GetLockFileStream(FileMode.OpenOrCreate); + channel = GetLockFileStream(FileMode.OpenOrCreate); + } + catch (IOException e) when (IsShareViolation(e)) + { + // no failure reason to be recorded, since this is the expected error if a lock exists } catch (IOException e) { @@ -410,29 +494,137 @@ namespace Lucene.Net.Store // if it fails to get the lock. FailureReason = e; } + return channel != null; + } + } - if (stream != null) + protected override void Dispose(bool disposing) + { + if (disposing) + { + lock (this) { + // whether or not we have created a file, we need to remove + // the lock instance from the dictionary that tracks them. try { - stream.Lock(0, 1); - // only assign the channel if the lock succeeds - channel = stream; + lock (NativeFSLockFactory._locks) + NativeFSLockFactory._locks.Remove(path); } - catch (Exception e) + finally { - FailureReason = e; - IOUtils.DisposeWhileHandlingException(stream); + if (channel != null) + { + try + { + IOUtils.DisposeWhileHandlingException(channel); + } + finally + { + channel = null; + } + } } } -#else + } + } + + public override bool IsLocked() + { + lock (this) + { + // First a shortcut, if a lock reference in this instance is available + if (channel != null) + { + return true; + } + try { - channel = GetLockFileStream(FileMode.OpenOrCreate); + using (var stream = GetLockFileStream(FileMode.Open)) + { + } + return false; } - catch (IOException e) when (IsLockOrShareViolation(e)) + catch (IOException e) when (IsShareViolation(e)) { - // no failure reason to be recorded, since this is the expected error if a lock exists + return true; + } + catch (FileNotFoundException) + { + // if the file doesn't exists, there can be no lock active + return false; + } + } + } + + public override string ToString() + { + return $"{nameof(SharingNativeFSLock)}@{path}"; + } + } + + // Uses FileStream locking of file pages. + internal class NativeFSLock : Lock + { + private FileStream channel; + private readonly string path; + private readonly DirectoryInfo lockDir; + + public NativeFSLock(DirectoryInfo lockDir, string path) + { + this.lockDir = lockDir; + this.path = path; + } + + /// <summary> + /// Return true if the <see cref="IOException"/> is the result of a lock violation + /// </summary> + private bool IsLockViolation(IOException e) + { + return e.HResult == NativeFSLockFactory.HRESULT_FILE_LOCK_VIOLATION; + } + + private FileStream GetLockFileStream(FileMode mode) + { + if (!System.IO.Directory.Exists(lockDir.FullName)) + { + try + { + System.IO.Directory.CreateDirectory(lockDir.FullName); + } + catch (Exception e) + { + // note that several processes might have been trying to create the same directory at the same time. + // if one succeeded, the directory will exist and the exception can be ignored. In all other cases we should report it. + if (!System.IO.Directory.Exists(lockDir.FullName)) + throw new IOException("Cannot create directory: " + lockDir.FullName, e); + } + } + else if (File.Exists(lockDir.FullName)) + { + throw new IOException("Found regular file where directory expected: " + lockDir.FullName); + } + + return new FileStream(path, mode, FileAccess.Write, FileShare.ReadWrite); + } + + public override bool Obtain() + { + lock (this) + { + FailureReason = null; + + if (channel != null) + { + // Our instance is already locked: + return false; + } + + FileStream stream = null; + try + { + stream = GetLockFileStream(FileMode.OpenOrCreate); } catch (IOException e) { @@ -452,7 +644,21 @@ namespace Lucene.Net.Store // if it fails to get the lock. FailureReason = e; } -#endif + + if (stream != null) + { + try + { + stream.Lock(0, 1); + // only assign the channel if the lock succeeds + channel = stream; + } + catch (Exception e) + { + FailureReason = e; + IOUtils.DisposeWhileHandlingException(stream); + } + } return channel != null; } } @@ -482,7 +688,6 @@ namespace Lucene.Net.Store { channel = null; } -#if FEATURE_FILESTREAM_LOCK // try to delete the file if we created it, but it's not an error if we can't. try { @@ -491,7 +696,6 @@ namespace Lucene.Net.Store catch { } -#endif } } } @@ -512,16 +716,14 @@ namespace Lucene.Net.Store { using (var stream = GetLockFileStream(FileMode.Open)) { -#if FEATURE_FILESTREAM_LOCK // try to find out if the file is locked by writing a byte. Note that we need to flush the stream to find out. stream.WriteByte(0); stream.Flush(); // this *may* throw an IOException if the file is locked, but... // ... closing the stream is the real test -#endif } return false; } - catch (IOException e) when (IsLockOrShareViolation(e)) + catch (IOException e) when (IsLockViolation(e)) { return true; } @@ -535,36 +737,17 @@ namespace Lucene.Net.Store public override string ToString() { - return "NativeFSLock@" + path; + return $"{nameof(NativeFSLock)}@{path}"; } } - - // LUCENENET: Lock that uses HResult native to Windows - internal class WindowsNativeFSLock : SharingAwareNativeFSLock +#if !FEATURE_FILESTREAM_LOCK + internal static class FileStreamExtensions { -#if FEATURE_FILESTREAM_LOCK - private const int ERROR_LOCK_VIOLATION = 0x21; -#else - private const int ERROR_SHARE_VIOLATION = 0x20; -#endif - - public WindowsNativeFSLock(NativeFSLockFactory outerInstance, DirectoryInfo lockDir, string path) - : base(outerInstance, lockDir, path) + // Dummy lock method to ensure we can compile even if the feature is unavailable + public static void Lock(this FileStream stream, long position, long length) { } - - /// <summary> - /// Return true if the <see cref="IOException"/> is the result of a lock violation - /// </summary> - protected override bool IsLockOrShareViolation(IOException e) - { - var result = e.HResult & 0xFFFF; -#if FEATURE_FILESTREAM_LOCK - return result == ERROR_LOCK_VIOLATION; -#else - return result == ERROR_SHARE_VIOLATION; -#endif - } } +#endif } \ No newline at end of file diff --git a/src/Lucene.Net/Support/IO/FileSupport.cs b/src/Lucene.Net/Support/IO/FileSupport.cs index 35734e6..c988016 100644 --- a/src/Lucene.Net/Support/IO/FileSupport.cs +++ b/src/Lucene.Net/Support/IO/FileSupport.cs @@ -29,7 +29,62 @@ namespace Lucene.Net.Support.IO /// </summary> public static class FileSupport { - private static int ERROR_FILE_EXISTS = 0x0050; + // LUCNENENET NOTE: Lookup the HResult value we are interested in for the current OS + // by provoking the exception during initialization and caching its HResult value for later. + // We optimize for Windows because those HResult values are known and documented, but for + // other platforms, this is the only way we can reliably determine the HResult values + // we are interested in. + // + // Reference: https://stackoverflow.com/q/46380483 + private const int WIN_HRESULT_FILE_ALREADY_EXISTS = unchecked((int)0x80070050); + private static readonly int? HRESULT_FILE_ALREADY_EXISTS = LoadFileAlreadyExistsHResult(); + + private static int? LoadFileAlreadyExistsHResult() + { + if (Constants.WINDOWS) + return WIN_HRESULT_FILE_ALREADY_EXISTS; + + return GetFileIOExceptionHResult(provokeException: (fileName) => + { + //Try to create the file again -this should throw an IOException with the correct HResult for the current platform + using (var stream = new FileStream(fileName, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) { } + }); + } + + internal static int? GetFileIOExceptionHResult(Action<string> provokeException) + { + string fileName; + try + { + // This could throw, but we don't care about this HResult value. + fileName = Path.GetTempFileName(); + } + catch + { + return null; // We couldn't create a temp file + } + try + { + provokeException(fileName); + } + catch (IOException ex) when (ex.HResult != 0) // Assume 0 means the platform is not completely implemented, thus unknown + { + return ex.HResult; + } + catch + { + return null; // Unknown exception + } + finally + { + try + { + File.Delete(fileName); + } + catch { } + } + return null; // Should never get here + } /// <summary> /// Creates a new empty file in a random subdirectory of <see cref="Path.GetTempPath()"/>, using the given prefix and @@ -144,14 +199,26 @@ namespace Lucene.Net.Support.IO return new FileInfo(fileName); } - private static bool IsFileAlreadyExistsException(IOException e, string fileName) + /// <summary> + /// Tests whether the passed in <see cref="Exception"/> is an <see cref="IOException"/> + /// corresponding to the underlying operating system's "File Already Exists" violation. + /// This works by forcing the exception to occur during initialization and caching the + /// <see cref="Exception.HResult"/> value for the current OS. + /// </summary> + /// <param name="ex">An exception, for comparison.</param> + /// <param name="filePath">The path of the file to check. This is used as a fallback in case the + /// current OS doesn't have an HResult (an edge case).</param> + /// <returns><c>true</c> if the exception passed is an <see cref="IOException"/> with an + /// <see cref="Exception.HResult"/> corresponding to the operating system's "File Already Exists" violation, which + /// occurs when an attempt is made to create a file that already exists.</returns> + public static bool IsFileAlreadyExistsException(Exception ex, string filePath) { - // On Windows, we can rely on the constant, but we need to fallback - // to doing a physical file check to be portable across platforms. - if (Constants.WINDOWS) - return (e.HResult & 0xFFFF) == ERROR_FILE_EXISTS; + if (!typeof(IOException).Equals(ex)) + return false; + else if (HRESULT_FILE_ALREADY_EXISTS.HasValue) + return ex.HResult == HRESULT_FILE_ALREADY_EXISTS; else - return File.Exists(fileName); + return File.Exists(filePath); } /// <summary>
