Repository: lucenenet Updated Branches: refs/heads/master 03bc1cbff -> 8eada7984
Revert "Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles variable and all references to them" to prevent widespread 'file in use by another process' error. This reverts commit bc295b0e79c153d9dfa7e9f99a144db135a014f5. Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/681430e4 Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/681430e4 Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/681430e4 Branch: refs/heads/master Commit: 681430e44ce316e18b788598ed002f6f14dd3822 Parents: 03bc1cb Author: Shad Storhaug <[email protected]> Authored: Thu Sep 14 04:12:33 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Thu Sep 14 04:12:33 2017 +0700 ---------------------------------------------------------------------- src/Lucene.Net/Store/FSDirectory.cs | 55 +++++------ src/Lucene.Net/Util/IOUtils.cs | 153 +++++++++++++++---------------- 2 files changed, 96 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Store/FSDirectory.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs index 8c65d9c..5487b96 100644 --- a/src/Lucene.Net/Store/FSDirectory.cs +++ b/src/Lucene.Net/Store/FSDirectory.cs @@ -92,10 +92,7 @@ namespace Lucene.Net.Store public const int DEFAULT_READ_CHUNK_SIZE = 8192; protected readonly DirectoryInfo m_directory; // The underlying filesystem directory - - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed + protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>(); // Files written, but not yet sync'ed #pragma warning disable 612, 618 private int chunkSize = DEFAULT_READ_CHUNK_SIZE; #pragma warning restore 612, 618 @@ -313,9 +310,7 @@ namespace Lucene.Net.Store { throw new IOException("Cannot delete " + file, e); } - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //m_staleFiles.Remove(name); + m_staleFiles.Remove(name); } /// <summary> @@ -358,35 +353,30 @@ namespace Lucene.Net.Store protected virtual void OnIndexOutputClosed(FSIndexOutput io) { - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //m_staleFiles.Add(io.name); + m_staleFiles.Add(io.name); } public override void Sync(ICollection<string> names) { EnsureOpen(); + ISet<string> toSync = new HashSet<string>(names); + toSync.IntersectWith(m_staleFiles); - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. Therefore, there is nothing else to do in this method. - //ISet<string> toSync = new HashSet<string>(names); - //toSync.IntersectWith(m_staleFiles); - - //// LUCENENET specific: Fsync breaks concurrency here. - //// Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2 - ////foreach (var name in toSync) - ////{ - //// Fsync(name); - ////} - - //// fsync the directory itsself, but only if there was any file fsynced before - //// (otherwise it can happen that the directory does not yet exist)! - //if (toSync.Count > 0) + // LUCENENET specific: Fsync breaks concurrency here. + // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2 + //foreach (var name in toSync) //{ - // IOUtils.Fsync(m_directory.FullName, true); + // Fsync(name); //} - //m_staleFiles.ExceptWith(toSync); + // fsync the directory itsself, but only if there was any file fsynced before + // (otherwise it can happen that the directory does not yet exist)! + if (toSync.Count > 0) + { + IOUtils.Fsync(m_directory.FullName, true); + } + + m_staleFiles.ExceptWith(toSync); } public override string GetLockID() @@ -545,12 +535,9 @@ namespace Lucene.Net.Store // LUCENENET NOTE: FileStream doesn't have a way to set length } - // LUCENENET specific: Fsync is pointless in .NET, since we are - // calling FileStream.Flush(true) before the stream is disposed - // which means we never need it at the point in Java where it is called. - //protected virtual void Fsync(string name) - //{ - // IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false); - //} + protected virtual void Fsync(string name) + { + IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Util/IOUtils.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs index 27daafa..5907b60 100644 --- a/src/Lucene.Net/Util/IOUtils.cs +++ b/src/Lucene.Net/Util/IOUtils.cs @@ -522,83 +522,80 @@ namespace Lucene.Net.Util } } - // LUCENENET specific: Fsync is pointless in .NET, since we are - // calling FileStream.Flush(true) before the stream is disposed - // which means we never need it at the point in Java where it is called. - // /// <summary> - // /// Ensure that any writes to the given file is written to the storage device that contains it. </summary> - // /// <param name="fileToSync"> The file to fsync </param> - // /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s, - // /// because not all file systems and operating systems allow to fsync on a directory) </param> - // public static void Fsync(string fileToSync, bool isDir) - // { - // // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version - // // they catch this in IOException branch and return if the call is for the directory. - // // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by - // // IOException block. No need to even attempt to fsync, just return if the call is for directory - // if (isDir) - // { - // return; - // } - - // var retryCount = 1; - // while (true) - // { - // FileStream file = null; - // bool success = false; - // try - // { - // // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect. - // // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ - // file = new FileStream(fileToSync, - // FileMode.Open, // We shouldn't create a file when syncing. - // // Java version uses FileChannel which doesn't create the file if it doesn't already exist, - // // so there should be no reason for attempting to create it in Lucene.Net. - // FileAccess.Write, - // FileShare.ReadWrite); - // //FileSupport.Sync(file); - // file.Flush(true); - // success = true; - // } - //#pragma warning disable 168 - // catch (IOException e) - //#pragma warning restore 168 - // { - // if (retryCount == 5) - // { - // throw; - // } - //#if !NETSTANDARD - // try - // { - //#endif - // // Pause 5 msec - // Thread.Sleep(5); - //#if !NETSTANDARD - // } - // catch (ThreadInterruptedException ie) - // { - // var ex = new ThreadInterruptedException(ie.ToString(), ie); - // //ex.AddSuppressed(exc); - // throw ex; - // } - //#endif - // } - // finally - // { - // if (file != null) - // { - // file.Dispose(); - // } - // } - - // if (success) - // { - // return; - // } - - // retryCount++; - // } - // } + /// <summary> + /// Ensure that any writes to the given file is written to the storage device that contains it. </summary> + /// <param name="fileToSync"> The file to fsync </param> + /// <param name="isDir"> If <c>true</c>, the given file is a directory (we open for read and ignore <see cref="IOException"/>s, + /// because not all file systems and operating systems allow to fsync on a directory) </param> + public static void Fsync(string fileToSync, bool isDir) + { + // Fsync does not appear to function properly for Windows and Linux platforms. In Lucene version + // they catch this in IOException branch and return if the call is for the directory. + // In Lucene.Net the exception is UnauthorizedAccessException and is not handled by + // IOException block. No need to even attempt to fsync, just return if the call is for directory + if (isDir) + { + return; + } + + var retryCount = 1; + while (true) + { + FileStream file = null; + bool success = false; + try + { + // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect. + // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ + file = new FileStream(fileToSync, + FileMode.Open, // We shouldn't create a file when syncing. + // Java version uses FileChannel which doesn't create the file if it doesn't already exist, + // so there should be no reason for attempting to create it in Lucene.Net. + FileAccess.Write, + FileShare.ReadWrite); + //FileSupport.Sync(file); + file.Flush(true); + success = true; + } +#pragma warning disable 168 + catch (IOException e) +#pragma warning restore 168 + { + if (retryCount == 5) + { + throw; + } +#if !NETSTANDARD + try + { +#endif + // Pause 5 msec + Thread.Sleep(5); +#if !NETSTANDARD + } + catch (ThreadInterruptedException ie) + { + var ex = new ThreadInterruptedException(ie.ToString(), ie); + //ex.AddSuppressed(exc); + throw ex; + } +#endif + } + finally + { + if (file != null) + { + file.Dispose(); + } + } + + if (success) + { + return; + } + + retryCount++; + } + } } } \ No newline at end of file
