Lucene.Net.Replicator: Cleaned up documentation and reverted code order back to the same as Lucene.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/67882465 Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/67882465 Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/67882465 Branch: refs/heads/replicator Commit: 678824651ae4dc02d9c51850b9f7ff6383dda78d Parents: e330530 Author: Shad Storhaug <[email protected]> Authored: Thu Aug 17 01:46:02 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Thu Aug 17 01:46:02 2017 +0700 ---------------------------------------------------------------------- .../AspNetCoreReplicationRequest.cs | 26 +- .../AspNetCoreReplicationResponse.cs | 22 +- .../AspNetCoreReplicationServiceExtentions.cs | 19 +- .../Http/HttpClientBase.cs | 91 ++-- .../Http/HttpReplicator.cs | 48 +-- .../Http/ReplicationService.cs | 25 +- src/Lucene.Net.Replicator/Http/package.html | 28 -- .../IReplicationHandler.cs | 32 -- .../ISourceDirectoryFactory.cs | 27 -- .../IndexAndTaxonomyReplicationHandler.cs | 19 +- .../IndexAndTaxonomyRevision.cs | 177 ++++---- .../IndexInputInputStream.cs | 8 +- .../IndexReplicationHandler.cs | 243 ++++++----- src/Lucene.Net.Replicator/IndexRevision.cs | 80 ++-- src/Lucene.Net.Replicator/LocalReplicator.cs | 326 +++++++------- .../Lucene.Net.Replicator.csproj | 8 +- .../PerSessionDirectoryFactory.cs | 4 +- src/Lucene.Net.Replicator/ReplicationClient.cs | 431 ++++++++++--------- src/Lucene.Net.Replicator/Replicator.cs | 21 +- src/Lucene.Net.Replicator/Revision.cs | 20 +- src/Lucene.Net.Replicator/RevisionFile.cs | 27 +- .../SessionExpiredException.cs | 26 +- src/Lucene.Net.Replicator/SessionToken.cs | 33 +- 23 files changed, 875 insertions(+), 866 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationRequest.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationRequest.cs b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationRequest.cs index 7a9fba2..4bcdba5 100644 --- a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationRequest.cs +++ b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationRequest.cs @@ -1,14 +1,26 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Lucene.Net.Replicator.Http; -using Lucene.Net.Replicator.Http.Abstractions; +using Lucene.Net.Replicator.Http.Abstractions; using Microsoft.AspNetCore.Http; +using System.Linq; namespace Lucene.Net.Replicator.AspNetCore { + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /// <summary> /// Abstraction for remote replication requests, allows easy integration into any hosting frameworks. /// </summary> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationResponse.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationResponse.cs b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationResponse.cs index e671101..8e73b28 100644 --- a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationResponse.cs +++ b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationResponse.cs @@ -1,10 +1,26 @@ -using System.IO; -using Lucene.Net.Replicator.Http; -using Lucene.Net.Replicator.Http.Abstractions; +using Lucene.Net.Replicator.Http.Abstractions; using Microsoft.AspNetCore.Http; +using System.IO; namespace Lucene.Net.Replicator.AspNetCore { + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /// <summary> /// Implementation of the <see cref="IReplicationResponse"/> abstraction for the AspNetCore framework. /// </summary> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationServiceExtentions.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationServiceExtentions.cs b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationServiceExtentions.cs index f772bfd..967ea0f 100644 --- a/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationServiceExtentions.cs +++ b/src/Lucene.Net.Replicator.AspNetCore/AspNetCoreReplicationServiceExtentions.cs @@ -3,11 +3,28 @@ using Microsoft.AspNetCore.Http; namespace Lucene.Net.Replicator.AspNetCore { + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + //Note: LUCENENET specific public static class AspNetCoreReplicationServiceExtentions { /// <summary> - /// Extensiont method that mirrors the signature of <see cref="ReplicationService.Perform"/> using AspNetCore as implementation. + /// Extension method that mirrors the signature of <see cref="ReplicationService.Perform"/> using AspNetCore as implementation. /// </summary> public static void Perform(this ReplicationService self, HttpRequest request, HttpResponse response) { http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/Http/HttpClientBase.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/Http/HttpClientBase.cs b/src/Lucene.Net.Replicator/Http/HttpClientBase.cs index 139fffd..8f46b98 100644 --- a/src/Lucene.Net.Replicator/Http/HttpClientBase.cs +++ b/src/Lucene.Net.Replicator/Http/HttpClientBase.cs @@ -31,7 +31,7 @@ namespace Lucene.Net.Replicator.Http /// Base class for Http clients. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public abstract class HttpClientBase : IDisposable { @@ -49,49 +49,32 @@ namespace Lucene.Net.Replicator.Http protected string Url { get; private set; } private readonly HttpClient httpc; - - /// <summary> - /// Gets or Sets the connection timeout for this client, in milliseconds. This setting - /// is used to modify <see cref="HttpClient.Timeout"/>. - /// </summary> - public int ConnectionTimeout - { - get { return (int) httpc.Timeout.TotalMilliseconds; } - set { httpc.Timeout = TimeSpan.FromMilliseconds(value); } - } - - /// <summary> - /// Returns true if this instance was <see cref="Dispose(bool)"/>ed, otherwise - /// returns false. Note that if you override <see cref="Dispose(bool)"/>, you must call - /// <see cref="Dispose(bool)"/> on the base class, in order for this instance to be properly disposed. - /// </summary> - public bool IsDisposed { get; private set; } + private volatile bool isDisposed = false; /// <summary> /// Creates a new <see cref="HttpClientBase"/> with the given host, port and path. /// </summary> /// <remarks> - /// The host, port and path parameters are normalized to <code>http://{host}:{port}{path}</code>, - /// if path is <code>null</code> or <code>empty</code> it defaults to <code>/</code>. - /// <p> - /// A <see cref="HttpMessageHandler"/> is taken as an optional parameter as well, if this is not provided it defaults to null. + /// The host, port and path parameters are normalized to <c>http://{host}:{port}{path}</c>, + /// if path is <c>null</c> or <c>empty</c> it defaults to <c>/</c>. + /// <para/> + /// A <see cref="HttpMessageHandler"/> is taken as an optional parameter as well, if this is not provided it defaults to <c>null</c>. /// In this case the internal <see cref="HttpClient"/> will default to use a <see cref="HttpClientHandler"/>. - /// </p> /// </remarks> /// <param name="host">The host that the client should retrieve data from.</param> /// <param name="port">The port to be used to connect on.</param> /// <param name="path">The path to the replicator on the host.</param> - /// <param name="messageHandler">Optional, The HTTP handler stack to use for sending requests, defaults to null.</param> + /// <param name="messageHandler">Optional, The HTTP handler stack to use for sending requests, defaults to <c>null</c>.</param> protected HttpClientBase(string host, int port, string path, HttpMessageHandler messageHandler = null) : this(NormalizedUrl(host, port, path), messageHandler) { } /// <summary> - /// Creates a new <see cref="HttpClientBase"/> with the given url. + /// Creates a new <see cref="HttpClientBase"/> with the given <paramref name="url"/>. /// </summary> /// <remarks> - /// A <see cref="HttpMessageHandler"/> is taken as an optional parameter as well, if this is not provided it defaults to null. + /// A <see cref="HttpMessageHandler"/> is taken as an optional parameter as well, if this is not provided it defaults to <c>null</c>. /// In this case the internal <see cref="HttpClient"/> will default to use a <see cref="HttpClientHandler"/>. /// </remarks> /// <param name="url">The full url, including with host, port and path.</param> @@ -103,11 +86,11 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Creates a new <see cref="HttpClientBase"/> with the given url and HttpClient. + /// Creates a new <see cref="HttpClientBase"/> with the given <paramref name="url"/> and <see cref="HttpClient"/>. /// </summary> /// <remarks> - /// This allows full controll over how the HttpClient is created, - /// prefer the <see cref="HttpClientBase(string, HttpMessageHandler)"/> over this unless you know you need the control of the HttpClient. + /// This allows full controll over how the <see cref="HttpClient"/> is created, + /// prefer the <see cref="HttpClientBase(string, HttpMessageHandler)"/> over this unless you know you need the control of the <see cref="HttpClient"/>. /// </remarks> /// <param name="url"></param> /// <param name="client">The <see cref="HttpClient"/> to use make remote http calls.</param> @@ -116,21 +99,34 @@ namespace Lucene.Net.Replicator.Http { Url = url; httpc = client; - IsDisposed = false; + ConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + } + + /// <summary> + /// Gets or Sets the connection timeout for this client, in milliseconds. This setting + /// is used to modify <see cref="HttpClient.Timeout"/>. + /// </summary> + public int ConnectionTimeout + { + get { return (int)httpc.Timeout.TotalMilliseconds; } + set { httpc.Timeout = TimeSpan.FromMilliseconds(value); } } /// <summary> - /// Throws <see cref="ObjectDisposedException"/> if this client is already closed. + /// Throws <see cref="ObjectDisposedException"/> if this client is already disposed. /// </summary> - /// <exception cref="ObjectDisposedException">client is already closed.</exception> + /// <exception cref="ObjectDisposedException">client is already disposed.</exception> protected void EnsureOpen() { if (IsDisposed) { - throw new ObjectDisposedException("HttpClient already closed"); + throw new ObjectDisposedException("HttpClient already disposed"); } } + /// <summary> + /// Create a URL out of the given parameters, translate an empty/null path to '/' + /// </summary> private static string NormalizedUrl(string host, int port, string path) { if (string.IsNullOrEmpty(path)) @@ -139,7 +135,7 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Verifies the response status and if not successfull throws an exception. + /// <b>Internal:</b> Verifies the response status and if not successful throws an exception. /// </summary> /// <exception cref="IOException">IO Error happened at the server, check inner exception for details.</exception> /// <exception cref="HttpRequestException">Unknown error received from the server.</exception> @@ -199,6 +195,10 @@ namespace Lucene.Net.Replicator.Http throw new HttpRequestException(string.Format("unknown exception: {0} {1}", response.StatusCode, response.ReasonPhrase), exception); } + /// <summary> + /// <b>Internal:</b> Execute a request and return its result. + /// The <paramref name="parameters"/> argument is treated as: name1,value1,name2,value2,... + /// </summary> protected HttpResponseMessage ExecutePost(string request, object entity, params string[] parameters) { EnsureOpen(); @@ -215,6 +215,10 @@ namespace Lucene.Net.Replicator.Http return response; } + /// <summary> + /// <b>Internal:</b> Execute a request and return its result. + /// The <paramref name="parameters"/> argument is treated as: name1,value1,name2,value2,... + /// </summary> protected HttpResponseMessage ExecuteGet(string request, params string[] parameters) { EnsureOpen(); @@ -236,7 +240,7 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Internal utility: input stream of the provided response + /// Internal utility: input stream of the provided response. /// </summary> /// <exception cref="IOException"></exception> public Stream ResponseInputStream(HttpResponseMessage response) @@ -244,8 +248,10 @@ namespace Lucene.Net.Replicator.Http return ResponseInputStream(response, false); } + // TODO: can we simplify this Consuming !?!?!? /// <summary> - /// Internal utility: input stream of the provided response + /// Internal utility: input stream of the provided response, which optionally + /// consumes the response's resources when the input stream is exhausted. /// </summary> /// <exception cref="IOException"></exception> public Stream ResponseInputStream(HttpResponseMessage response, bool consume) @@ -254,7 +260,14 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Calls the overload <see cref="DoAction{T}(HttpResponseMessage, Boolean, Func{T})"/> passing <code>true</code> to consume. + /// Returns <c>true</c> if this instance was <see cref="Dispose(bool)"/>ed, otherwise + /// returns <c>false</c>. Note that if you override <see cref="Dispose(bool)"/>, you must call + /// <see cref="Dispose(bool)"/> on the base class, in order for this instance to be properly disposed. + /// </summary> + public bool IsDisposed { get { return isDisposed; } } + + /// <summary> + /// Calls the overload <see cref="DoAction{T}(HttpResponseMessage, bool, Func{T})"/> passing <c>true</c> to consume. /// </summary> protected T DoAction<T>(HttpResponseMessage response, Func<T> call) { @@ -264,7 +277,7 @@ namespace Lucene.Net.Replicator.Http /// <summary> /// Do a specific action and validate after the action that the status is still OK, /// and if not, attempt to extract the actual server side exception. Optionally - /// release the response at exit, depending on <code>consume</code> parameter. + /// release the response at exit, depending on <paramref name="consume"/> parameter. /// </summary> protected T DoAction<T>(HttpResponseMessage response, bool consume, Func<T> call) { @@ -302,7 +315,7 @@ namespace Lucene.Net.Replicator.Http { httpc.Dispose(); } - IsDisposed = true; + isDisposed = true; } /// <summary> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/Http/HttpReplicator.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/Http/HttpReplicator.cs b/src/Lucene.Net.Replicator/Http/HttpReplicator.cs index ab519c3..052450a 100644 --- a/src/Lucene.Net.Replicator/Http/HttpReplicator.cs +++ b/src/Lucene.Net.Replicator/Http/HttpReplicator.cs @@ -26,7 +26,7 @@ namespace Lucene.Net.Replicator.Http /// An HTTP implementation of <see cref="IReplicator"/>. Assumes the API supported by <see cref="ReplicationService"/>. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public class HttpReplicator : HttpClientBase, IReplicator { @@ -50,7 +50,7 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Creates a new <see cref="HttpReplicator"/> with the given url and HttpClient. + /// Creates a new <see cref="HttpReplicator"/> with the given <paramref name="url"/> and <see cref="HttpClient"/>. /// <see cref="HttpClientBase(string, HttpClient)"/> for more details. /// </summary> //Note: LUCENENET Specific @@ -58,15 +58,6 @@ namespace Lucene.Net.Replicator.Http : base(url, client) { } - - /// <summary> - /// Not supported. - /// </summary> - /// <exception cref="NotSupportedException">this replicator implementation does not support remote publishing of revisions</exception> - public void Publish(IRevision revision) - { - throw new NotSupportedException("this replicator implementation does not support remote publishing of revisions"); - } /// <summary> /// Checks for updates at the remote host. @@ -75,9 +66,9 @@ namespace Lucene.Net.Replicator.Http { string[] parameters = null; if (currentVersion != null) - parameters = new [] { ReplicationService.REPLICATE_VERSION_PARAM, currentVersion }; + parameters = new[] { ReplicationService.REPLICATE_VERSION_PARAM, currentVersion }; - HttpResponseMessage response = base.ExecuteGet( ReplicationService.ReplicationAction.UPDATE.ToString(), parameters); + HttpResponseMessage response = base.ExecuteGet(ReplicationService.ReplicationAction.UPDATE.ToString(), parameters); return DoAction(response, () => { using (DataInputStream inputStream = new DataInputStream(ResponseInputStream(response))) @@ -88,25 +79,34 @@ namespace Lucene.Net.Replicator.Http } /// <summary> - /// Releases a session obtained from the remote host. - /// </summary> - public void Release(string sessionId) - { - HttpResponseMessage response = ExecuteGet(ReplicationService.ReplicationAction.RELEASE.ToString(), ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionId); - // do not remove this call: as it is still validating for us! - DoAction<object>(response, () => null); - } - - /// <summary> /// Obtains the given file from it's source at the remote host. /// </summary> public Stream ObtainFile(string sessionId, string source, string fileName) { - HttpResponseMessage response = ExecuteGet(ReplicationService.ReplicationAction.OBTAIN.ToString(), + HttpResponseMessage response = ExecuteGet(ReplicationService.ReplicationAction.OBTAIN.ToString(), ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionId, ReplicationService.REPLICATE_SOURCE_PARAM, source, ReplicationService.REPLICATE_FILENAME_PARAM, fileName); return DoAction(response, false, () => ResponseInputStream(response)); } + + /// <summary> + /// Not supported. + /// </summary> + /// <exception cref="NotSupportedException">this replicator implementation does not support remote publishing of revisions</exception> + public void Publish(IRevision revision) + { + throw new NotSupportedException("this replicator implementation does not support remote publishing of revisions"); + } + + /// <summary> + /// Releases a session obtained from the remote host. + /// </summary> + public void Release(string sessionId) + { + HttpResponseMessage response = ExecuteGet(ReplicationService.ReplicationAction.RELEASE.ToString(), ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionId); + // do not remove this call: as it is still validating for us! + DoAction<object>(response, () => null); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/Http/ReplicationService.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/Http/ReplicationService.cs b/src/Lucene.Net.Replicator/Http/ReplicationService.cs index 090ca4d..95c8b50 100644 --- a/src/Lucene.Net.Replicator/Http/ReplicationService.cs +++ b/src/Lucene.Net.Replicator/Http/ReplicationService.cs @@ -26,24 +26,22 @@ namespace Lucene.Net.Replicator.Http /// <summary> /// A server-side service for handling replication requests. The service assumes - /// requests are sent in the format <code>/<context>/<shard>/<action></code> where - /// <ul> - /// <li><code>context</code> is the servlet context, e.g. <see cref="REPLICATION_CONTEXT"/></li> - /// <li><code>shard</code> is the ID of the shard, e.g. "s1"</li> - /// <li><code>action</code> is one of <see cref="ReplicationAction"/> values</li> - /// </ul> + /// requests are sent in the format <c>/<context>/<shard>/<action></c> where + /// <list type="bullet"> + /// <item><description><c>context</c> is the servlet context, e.g. <see cref="REPLICATION_CONTEXT"/></description></item> + /// <item><description><c>shard</c> is the ID of the shard, e.g. "s1"</description></item> + /// <item><description><c>action</c> is one of <see cref="ReplicationAction"/> values</description></item> + /// </list> /// For example, to check whether there are revision updates for shard "s1" you - /// should send the request: <code>http://host:port/replicate/s1/update</code>. + /// should send the request: <c>http://host:port/replicate/s1/update</c>. /// </summary> /// <remarks> /// This service is written using abstractions over requests and responses which makes it easy /// to integrate into any hosting framework. - /// <p> + /// <para/> /// See the Lucene.Net.Replicator.AspNetCore for an example of an implementation for the AspNetCore Framework. - /// </p> - /// </remarks> - /// <remarks> - /// Lucene.Experimental + /// <para/> + /// @lucene.experimental /// </remarks> public class ReplicationService { @@ -83,6 +81,7 @@ namespace Lucene.Net.Replicator.Http /// <summary> /// Json Serializer Settings to use when serializing and deserializing errors. /// </summary> + // LUCENENET specific public static readonly JsonSerializerSettings JSON_SERIALIZER_SETTINGS = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.All @@ -125,6 +124,7 @@ namespace Lucene.Net.Replicator.Http return param; } + // LUCENENET specific - copy method not used /// <summary> /// Executes the replication task. @@ -203,6 +203,5 @@ namespace Lucene.Net.Replicator.Http response.Flush(); } } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/Http/package.html ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/Http/package.html b/src/Lucene.Net.Replicator/Http/package.html deleted file mode 100644 index fce050b..0000000 --- a/src/Lucene.Net.Replicator/Http/package.html +++ /dev/null @@ -1,28 +0,0 @@ -<html> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<head> -<title>HTTP replication implementation</title> -</head> - -<body> -<h1>HTTP replication implementation</h1> -</body> - -</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IReplicationHandler.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IReplicationHandler.cs b/src/Lucene.Net.Replicator/IReplicationHandler.cs deleted file mode 100644 index d3e6504..0000000 --- a/src/Lucene.Net.Replicator/IReplicationHandler.cs +++ /dev/null @@ -1,32 +0,0 @@ -//STATUS: DRAFT - 4.8.0 - -using System.Collections.Generic; -using System.IO; -using Directory = Lucene.Net.Store.Directory; - -namespace Lucene.Net.Replicator -{ - /// <summary>Handler for revisions obtained by the client.</summary> - //Note: LUCENENET specific denesting of interface - public interface IReplicationHandler - { - /// <summary>Returns the current revision files held by the handler.</summary> - string CurrentVersion { get; } - - /// <summary>Returns the current revision version held by the handler.</summary> - IDictionary<string, IList<RevisionFile>> CurrentRevisionFiles { get; } - - /// <summary> - /// Called when a new revision was obtained and is available (i.e. all needed files were successfully copied). - /// </summary> - /// <param name="version">The version of the <see cref="IRevision"/> that was copied</param> - /// <param name="revisionFiles"> the files contained by this <see cref="IRevision"/></param> - /// <param name="copiedFiles">the files that were actually copied</param> - /// <param name="sourceDirectory">a mapping from a source of files to the <see cref="Directory"/> they were copied into</param> - /// <see cref="IOException"/> - void RevisionReady(string version, - IDictionary<string, IList<RevisionFile>> revisionFiles, - IDictionary<string, IList<string>> copiedFiles, - IDictionary<string, Directory> sourceDirectory); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/ISourceDirectoryFactory.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/ISourceDirectoryFactory.cs b/src/Lucene.Net.Replicator/ISourceDirectoryFactory.cs deleted file mode 100644 index 7942b91..0000000 --- a/src/Lucene.Net.Replicator/ISourceDirectoryFactory.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Lucene.Net.Store; - -namespace Lucene.Net.Replicator -{ - /// <summary> - /// Resolves a session and source into a <see cref="Directory"/> to use for copying - /// the session files to. - /// </summary> - //Note: LUCENENET specific denesting of interface - public interface ISourceDirectoryFactory - { - /// <summary> - /// Returns the <see cref="Directory"/> to use for the given session and source. - /// Implementations may e.g. return different directories for different - /// sessions, or the same directory for all sessions. In that case, it is - /// advised to clean the directory before it is used for a new session. - /// </summary> - /// <seealso cref="CleanupSession"/> - Directory GetDirectory(string sessionId, string source); //throws IOException; - - /// <summary> - /// Called to denote that the replication actions for this session were finished and the directory is no longer needed. - /// </summary> - /// <exception cref="System.IO.IOException"></exception> - void CleanupSession(string sessionId); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IndexAndTaxonomyReplicationHandler.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IndexAndTaxonomyReplicationHandler.cs b/src/Lucene.Net.Replicator/IndexAndTaxonomyReplicationHandler.cs index 36bf271..d6a3ce0 100644 --- a/src/Lucene.Net.Replicator/IndexAndTaxonomyReplicationHandler.cs +++ b/src/Lucene.Net.Replicator/IndexAndTaxonomyReplicationHandler.cs @@ -30,23 +30,22 @@ namespace Lucene.Net.Replicator /// A <see cref="IReplicationHandler"/> for replication of an index and taxonomy pair. /// See <see cref="IReplicationHandler"/> for more detail. This handler ensures /// that the search and taxonomy indexes are replicated in a consistent way. - /// - /// <see cref="IndexReplicationHandler"/> /// </summary> /// <remarks> - /// If you intend to recreate a taxonomy index, you should make sure + /// <b>NOTE:</b> If you intend to recreate a taxonomy index, you should make sure /// to reopen an IndexSearcher and TaxonomyReader pair via the provided callback, /// to guarantee that both indexes are in sync. This handler does not prevent /// replicating such index and taxonomy pairs, and if they are reopened by a /// different thread, unexpected errors can occur, as well as inconsistency /// between the taxonomy and index readers. - /// - /// Lucene.Experimental + /// <para/> + /// @lucene.experimental /// </remarks> + /// <seealso cref="IndexReplicationHandler"/> public class IndexAndTaxonomyReplicationHandler : IReplicationHandler { /// <summary> - /// The component used to log messages to the <see cref="InfoStream"/>. + /// The component used to log messages to the <see cref="Util.InfoStream.Default"/> <see cref="Util.InfoStream"/>. /// </summary> public const string INFO_STREAM_COMPONENT = "IndexAndTaxonomyReplicationHandler"; @@ -58,6 +57,10 @@ namespace Lucene.Net.Replicator public string CurrentVersion { get; private set; } public IDictionary<string, IList<RevisionFile>> CurrentRevisionFiles { get; private set; } + + /// <summary> + /// Gets or sets the <see cref="Util.InfoStream"/> to use for logging messages. + /// </summary> public InfoStream InfoStream { get { return infoStream; } @@ -97,10 +100,6 @@ namespace Lucene.Net.Replicator } } - /// <summary> - /// TODO - /// </summary> - /// <exception cref=""></exception> public void RevisionReady(string version, IDictionary<string, IList<RevisionFile>> revisionFiles, IDictionary<string, IList<string>> copiedFiles, http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IndexAndTaxonomyRevision.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IndexAndTaxonomyRevision.cs b/src/Lucene.Net.Replicator/IndexAndTaxonomyRevision.cs index 890f995..63040b0 100644 --- a/src/Lucene.Net.Replicator/IndexAndTaxonomyRevision.cs +++ b/src/Lucene.Net.Replicator/IndexAndTaxonomyRevision.cs @@ -1,6 +1,4 @@ -//STATUS: DRAFT - 4.8.0 - -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -38,10 +36,58 @@ namespace Lucene.Net.Replicator /// guarantee consistency of both on the replicating (client) side. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> + /// <seealso cref="IndexRevision"/> public class IndexAndTaxonomyRevision : IRevision { + /// <summary> + /// A <see cref="DirectoryTaxonomyWriter"/> which sets the underlying + /// <see cref="Index.IndexWriter"/>'s <see cref="IndexDeletionPolicy"/> to + /// <see cref="SnapshotDeletionPolicy"/>. + /// </summary> + public class SnapshotDirectoryTaxonomyWriter : DirectoryTaxonomyWriter + { + /// <summary> + /// Gets the <see cref="SnapshotDeletionPolicy"/> used by the underlying <see cref="Index.IndexWriter"/>. + /// </summary> + public SnapshotDeletionPolicy DeletionPolicy { get; private set; } + /// <summary> + /// Gets the <see cref="Index.IndexWriter"/> used by this <see cref="DirectoryTaxonomyWriter"/>. + /// </summary> + public IndexWriter IndexWriter { get; private set; } + + /// <summary> + /// <see cref="DirectoryTaxonomyWriter(Directory, OpenMode, ITaxonomyWriterCache)"/> + /// </summary> + /// <exception cref="IOException"></exception> + public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode, ITaxonomyWriterCache cache) + : base(directory, openMode, cache) + { + } + + /// <summary> + /// <see cref="DirectoryTaxonomyWriter(Directory, OpenMode)"/> + /// </summary> + /// <exception cref="IOException"></exception> + public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode = OpenMode.CREATE_OR_APPEND) + : base(directory, openMode) + { + } + + protected override IndexWriterConfig CreateIndexWriterConfig(OpenMode openMode) + { + IndexWriterConfig conf = base.CreateIndexWriterConfig(openMode); + conf.IndexDeletionPolicy = DeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy); + return conf; + } + + protected override IndexWriter OpenIndexWriter(Directory directory, IndexWriterConfig config) + { + return IndexWriter = base.OpenIndexWriter(directory, config); + } + } + public const string INDEX_SOURCE = "index"; public const string TAXONOMY_SOURCE = "taxonomy"; @@ -50,11 +96,33 @@ namespace Lucene.Net.Replicator private readonly IndexCommit indexCommit, taxonomyCommit; private readonly SnapshotDeletionPolicy indexSdp, taxonomySdp; - public string Version { get; private set; } - public IDictionary<string, IList<RevisionFile>> SourceFiles { get; private set; } + /// <summary> + /// Returns a map of the revision files from the given <see cref="IndexCommit"/>s of the search and taxonomy indexes. + /// </summary> + /// <exception cref="IOException"></exception> + public static IDictionary<string, IList<RevisionFile>> RevisionFiles(IndexCommit indexCommit, IndexCommit taxonomyCommit) + { + return new Dictionary<string, IList<RevisionFile>>{ + { INDEX_SOURCE, IndexRevision.RevisionFiles(indexCommit).Values.First() }, + { TAXONOMY_SOURCE, IndexRevision.RevisionFiles(taxonomyCommit).Values.First() } + }; + } /// <summary> - /// TODO + /// Returns a <see cref="string"/> representation of a revision's version from the given + /// <see cref="IndexCommit"/>s of the search and taxonomy indexes. + /// </summary> + /// <param name="commit"></param> + /// <returns>a <see cref="string"/> representation of a revision's version from the given <see cref="IndexCommit"/>s of the search and taxonomy indexes.</returns> + public static string RevisionVersion(IndexCommit indexCommit, IndexCommit taxonomyCommit) + { + return string.Format("{0:X}:{1:X}", indexCommit.Generation, taxonomyCommit.Generation); + } + + /// <summary> + /// Constructor over the given <see cref="IndexWriter"/>. Uses the last + /// <see cref="IndexCommit"/> found in the <see cref="Directory"/> managed by the given + /// writer. /// </summary> /// <exception cref="IOException"></exception> public IndexAndTaxonomyRevision(IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxonomyWriter) @@ -105,9 +173,10 @@ namespace Lucene.Net.Replicator return cmp != 0 ? cmp : taxonomyCommit.CompareTo(itr.taxonomyCommit); } - /// <summary> - /// TODO - /// </summary> + public string Version { get; private set; } + + public IDictionary<string, IList<RevisionFile>> SourceFiles { get; private set; } + /// <exception cref="IOException"></exception> public Stream Open(string source, string fileName) { @@ -116,9 +185,6 @@ namespace Lucene.Net.Replicator return new IndexInputStream(commit.Directory.OpenInput(fileName, IOContext.READ_ONCE)); } - /// <summary> - /// TODO - /// </summary> /// <exception cref="IOException"></exception> public void Release() { @@ -140,90 +206,5 @@ namespace Lucene.Net.Replicator taxonomyWriter.IndexWriter.DeleteUnusedFiles(); } } - - /// <summary> - /// Returns a map of the revision files from the given <see cref="IndexCommit"/>s of the search and taxonomy indexes. - /// </summary> - /// <param name="indexCommit"></param> - /// <param name="taxonomyCommit"></param> - /// <returns></returns> - /// <exception cref="IOException"></exception> - public static IDictionary<string, IList<RevisionFile>> RevisionFiles(IndexCommit indexCommit, IndexCommit taxonomyCommit) - { - return new Dictionary<string, IList<RevisionFile>>{ - { INDEX_SOURCE, IndexRevision.RevisionFiles(indexCommit).Values.First() }, - { TAXONOMY_SOURCE, IndexRevision.RevisionFiles(taxonomyCommit).Values.First() } - }; - } - - /// <summary> - /// Returns a String representation of a revision's version from the given - /// <see cref="IndexCommit"/>s of the search and taxonomy indexes. - /// </summary> - /// <param name="commit"></param> - /// <returns>a String representation of a revision's version from the given <see cref="IndexCommit"/>s of the search and taxonomy indexes.</returns> - public static string RevisionVersion(IndexCommit indexCommit, IndexCommit taxonomyCommit) - { - return string.Format("{0:X}:{1:X}", indexCommit.Generation, taxonomyCommit.Generation); - } - - /// <summary> - /// A <seealso cref="DirectoryTaxonomyWriter"/> which sets the underlying - /// <seealso cref="IndexWriter"/>'s <seealso cref="IndexDeletionPolicy"/> to - /// <seealso cref="SnapshotDeletionPolicy"/>. - /// </summary> - public class SnapshotDirectoryTaxonomyWriter : DirectoryTaxonomyWriter - { - /// <summary> - /// Gets the <see cref="SnapshotDeletionPolicy"/> used by the underlying <see cref="IndexWriter"/>. - /// </summary> - public SnapshotDeletionPolicy DeletionPolicy { get; private set; } - /// <summary> - /// Gets the <see cref="IndexWriter"/> used by this <see cref="DirectoryTaxonomyWriter"/>. - /// </summary> - public IndexWriter IndexWriter { get; private set; } - - /// <summary> - /// <see cref="DirectoryTaxonomyWriter(Directory, OpenMode, ITaxonomyWriterCache)"/> - /// </summary> - /// <exception cref="IOException"></exception> - public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode, ITaxonomyWriterCache cache) - : base(directory, openMode, cache) - { - } - - /// <summary> - /// <see cref="DirectoryTaxonomyWriter(Directory, OpenMode)"/> - /// </summary> - /// <exception cref="IOException"></exception> - public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode = OpenMode.CREATE_OR_APPEND) - : base(directory, openMode) - { - } - - /// <summary> - /// - /// </summary> - /// <param name="openMode"></param> - /// <returns></returns> - protected override IndexWriterConfig CreateIndexWriterConfig(OpenMode openMode) - { - IndexWriterConfig conf = base.CreateIndexWriterConfig(openMode); - conf.IndexDeletionPolicy = DeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy); - return conf; - } - - /// <summary> - /// TODO - /// </summary> - /// <param name="directory"></param> - /// <param name="config"></param> - /// <returns></returns> - protected override IndexWriter OpenIndexWriter(Directory directory, IndexWriterConfig config) - { - return IndexWriter = base.OpenIndexWriter(directory, config); - } - } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IndexInputInputStream.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IndexInputInputStream.cs b/src/Lucene.Net.Replicator/IndexInputInputStream.cs index 95f6e1c..df72010 100644 --- a/src/Lucene.Net.Replicator/IndexInputInputStream.cs +++ b/src/Lucene.Net.Replicator/IndexInputInputStream.cs @@ -1,6 +1,4 @@ -//STATUS: INPROGRESS - 4.8.0 - -using System; +using System; using System.IO; using Lucene.Net.Store; @@ -24,10 +22,10 @@ namespace Lucene.Net.Replicator */ /// <summary> - /// + /// A <see cref="Stream"/> which wraps an <see cref="IndexInput"/>. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public class IndexInputStream : Stream { http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IndexReplicationHandler.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IndexReplicationHandler.cs b/src/Lucene.Net.Replicator/IndexReplicationHandler.cs index 7edc95c..5189e44 100644 --- a/src/Lucene.Net.Replicator/IndexReplicationHandler.cs +++ b/src/Lucene.Net.Replicator/IndexReplicationHandler.cs @@ -34,26 +34,24 @@ namespace Lucene.Net.Replicator /// <see cref="IndexWriter"/> to make sure any unused files are deleted. /// </summary> /// <remarks> - /// <para> - /// This handler assumes that <see cref="IndexWriter"/> is not opened by + /// <b>NOTE:</b> This handler assumes that <see cref="IndexWriter"/> is not opened by /// another process on the index directory. In fact, opening an /// <see cref="IndexWriter"/> on the same directory to which files are copied can lead /// to undefined behavior, where some or all the files will be deleted, override /// other files or simply create a mess. When you replicate an index, it is best /// if the index is never modified by <see cref="IndexWriter"/>, except the one that is /// open on the source index, from which you replicate. - /// </para> - /// <para> - /// This handler notifies the application via a provided <see cref="Callable"/> when an + /// <para/> + /// This handler notifies the application via a provided <see cref="T:Func{bool?}"/> when an /// updated index commit was made available for it. - /// </para> - /// - /// Lucene.Experimental + /// <para/> + /// @lucene.experimental /// </remarks> public class IndexReplicationHandler : IReplicationHandler { /// <summary> - /// The component used to log messages to the <see cref="InfoStream"/>. + /// The component used to log messages to the <see cref="Util.InfoStream.Default"/> + /// <see cref="Util.InfoStream"/>. /// </summary> public const string INFO_STREAM_COMPONENT = "IndexReplicationHandler"; @@ -61,104 +59,6 @@ namespace Lucene.Net.Replicator private readonly Func<bool?> callback; private InfoStream infoStream; - public string CurrentVersion { get; private set; } - public IDictionary<string, IList<RevisionFile>> CurrentRevisionFiles { get; private set; } - - public InfoStream InfoStream - { - get { return infoStream; } - set { infoStream = value ?? InfoStream.NO_OUTPUT; } - } - - /// <summary> - /// Constructor with the given index directory and callback to notify when the - /// indexes were updated. - /// </summary> - public IndexReplicationHandler(Directory indexDirectory, Func<bool?> callback) - { - this.InfoStream = InfoStream.Default; - this.callback = callback; - this.indexDirectory = indexDirectory; - - CurrentVersion = null; - CurrentRevisionFiles = null; - - if (DirectoryReader.IndexExists(indexDirectory)) - { - IList<IndexCommit> commits = DirectoryReader.ListCommits(indexDirectory); - IndexCommit commit = commits.Last(); - - CurrentVersion = IndexRevision.RevisionVersion(commit); - CurrentRevisionFiles = IndexRevision.RevisionFiles(commit); - - WriteToInfoStream( - string.Format("constructor(): currentVersion={0} currentRevisionFiles={1}", CurrentVersion, CurrentRevisionFiles), - string.Format("constructor(): commit={0}", commit)); - } - } - - public void RevisionReady(string version, - IDictionary<string, IList<RevisionFile>> revisionFiles, - IDictionary<string, IList<string>> copiedFiles, - IDictionary<string, Directory> sourceDirectory) - { - if (revisionFiles.Count > 1) throw new ArgumentException(string.Format("this handler handles only a single source; got {0}", revisionFiles.Keys)); - - Directory clientDirectory = sourceDirectory.Values.First(); - IList<string> files = copiedFiles.Values.First(); - string segmentsFile = GetSegmentsFile(files, false); - - bool success = false; - try - { - // copy files from the client to index directory - CopyFiles(clientDirectory, indexDirectory, files); - - // fsync all copied files (except segmentsFile) - indexDirectory.Sync(files); - - // now copy and fsync segmentsFile - clientDirectory.Copy(indexDirectory, segmentsFile, segmentsFile, IOContext.READ_ONCE); - indexDirectory.Sync(new[] { segmentsFile }); - - success = true; - } - finally - { - if (!success) - { - files.Add(segmentsFile); // add it back so it gets deleted too - CleanupFilesOnFailure(indexDirectory, files); - } - } - - // all files have been successfully copied + sync'd. update the handler's state - CurrentRevisionFiles = revisionFiles; - CurrentVersion = version; - - WriteToInfoStream(string.Format("revisionReady(): currentVersion={0} currentRevisionFiles={1}", CurrentVersion, CurrentRevisionFiles)); - - // update the segments.gen file - WriteSegmentsGen(segmentsFile, indexDirectory); - - // Cleanup the index directory from old and unused index files. - // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have - // side-effects, e.g. if it hits sudden IO errors while opening the index - // (and can end up deleting the entire index). It is not our job to protect - // against those errors, app will probably hit them elsewhere. - CleanupOldIndexFiles(indexDirectory, segmentsFile); - - // successfully updated the index, notify the callback that the index is - // ready. - if (callback != null) { - try { - callback.Invoke(); - } catch (Exception e) { - throw new IOException(e.Message, e); - } - } - } - //Note: LUCENENET Specific Utility Method private void WriteToInfoStream(params string[] messages) { @@ -171,7 +71,7 @@ namespace Lucene.Net.Replicator /// <summary> /// Returns the last <see cref="IndexCommit"/> found in the <see cref="Directory"/>, or - /// <code>null</code> if there are no commits. + /// <c>null</c> if there are no commits. /// </summary> /// <exception cref="IOException"></exception> public static IndexCommit GetLastCommit(Directory directory) @@ -196,10 +96,9 @@ namespace Lucene.Net.Replicator /// last, after all files. This is important in order to guarantee that if a /// reader sees the new segments_N, all other segment files are already on /// stable storage. - /// <para> + /// <para/> /// The reason why the code fails instead of putting segments_N file last is - /// that this indicates an error in the Revision implementation. - /// </para> + /// that this indicates an error in the <see cref="IRevision"/> implementation. /// </summary> public static string GetSegmentsFile(IList<string> files, bool allowEmpty) { @@ -241,6 +140,17 @@ namespace Lucene.Net.Replicator } } + /// <summary> + /// Cleans up the index directory from old index files. This method uses the + /// last commit found by <see cref="GetLastCommit(Directory)"/>. If it matches the + /// expected <paramref name="segmentsFile"/>, then all files not referenced by this commit point + /// are deleted. + /// </summary> + /// <remarks> + /// <b>NOTE:</b> This method does a best effort attempt to clean the index + /// directory. It suppresses any exceptions that occur, as this can be retried + /// the next time. + /// </remarks> public static void CleanupOldIndexFiles(Directory directory, string segmentsFile) { try @@ -280,7 +190,8 @@ namespace Lucene.Net.Replicator } /// <summary> - /// Copies the provided list of files from the <see cref="source"/> <see cref="Directory"/> to the <see cref="target"/> <see cref="Directory"/>. + /// Copies the provided list of files from the <paramref name="source"/> <see cref="Directory"/> to the + /// <paramref name="target"/> <see cref="Directory"/>, if they are not the same. /// </summary> /// <exception cref="IOException"></exception> public static void CopyFiles(Directory source, Directory target, IList<string> files) @@ -294,7 +205,7 @@ namespace Lucene.Net.Replicator /// <summary> /// Writes <see cref="IndexFileNames.SEGMENTS_GEN"/> file to the directory, reading - /// the generation from the given <code>segmentsFile</code>. If it is <code>null</code>, + /// the generation from the given <paramref name="segmentsFile"/>. If it is <c>null</c>, /// this method deletes segments.gen from the directory. /// </summary> public static void WriteSegmentsGen(string segmentsFile, Directory directory) @@ -314,5 +225,111 @@ namespace Lucene.Net.Replicator // suppress any errors while deleting this file. } } + + /// <summary> + /// Constructor with the given index directory and callback to notify when the + /// indexes were updated. + /// </summary> + public IndexReplicationHandler(Directory indexDirectory, Func<bool?> callback) // LUCENENET TODO: API - shouldn't this be Action ? + { + this.InfoStream = InfoStream.Default; + this.callback = callback; + this.indexDirectory = indexDirectory; + + CurrentVersion = null; + CurrentRevisionFiles = null; + + if (DirectoryReader.IndexExists(indexDirectory)) + { + IList<IndexCommit> commits = DirectoryReader.ListCommits(indexDirectory); + IndexCommit commit = commits.Last(); + + CurrentVersion = IndexRevision.RevisionVersion(commit); + CurrentRevisionFiles = IndexRevision.RevisionFiles(commit); + + WriteToInfoStream( + string.Format("constructor(): currentVersion={0} currentRevisionFiles={1}", CurrentVersion, CurrentRevisionFiles), + string.Format("constructor(): commit={0}", commit)); + } + } + + public string CurrentVersion { get; private set; } + + public IDictionary<string, IList<RevisionFile>> CurrentRevisionFiles { get; private set; } + + public void RevisionReady(string version, + IDictionary<string, IList<RevisionFile>> revisionFiles, + IDictionary<string, IList<string>> copiedFiles, + IDictionary<string, Directory> sourceDirectory) + { + if (revisionFiles.Count > 1) throw new ArgumentException(string.Format("this handler handles only a single source; got {0}", revisionFiles.Keys)); + + Directory clientDirectory = sourceDirectory.Values.First(); + IList<string> files = copiedFiles.Values.First(); + string segmentsFile = GetSegmentsFile(files, false); + + bool success = false; + try + { + // copy files from the client to index directory + CopyFiles(clientDirectory, indexDirectory, files); + + // fsync all copied files (except segmentsFile) + indexDirectory.Sync(files); + + // now copy and fsync segmentsFile + clientDirectory.Copy(indexDirectory, segmentsFile, segmentsFile, IOContext.READ_ONCE); + indexDirectory.Sync(new[] { segmentsFile }); + + success = true; + } + finally + { + if (!success) + { + files.Add(segmentsFile); // add it back so it gets deleted too + CleanupFilesOnFailure(indexDirectory, files); + } + } + + // all files have been successfully copied + sync'd. update the handler's state + CurrentRevisionFiles = revisionFiles; + CurrentVersion = version; + + WriteToInfoStream(string.Format("revisionReady(): currentVersion={0} currentRevisionFiles={1}", CurrentVersion, CurrentRevisionFiles)); + + // update the segments.gen file + WriteSegmentsGen(segmentsFile, indexDirectory); + + // Cleanup the index directory from old and unused index files. + // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have + // side-effects, e.g. if it hits sudden IO errors while opening the index + // (and can end up deleting the entire index). It is not our job to protect + // against those errors, app will probably hit them elsewhere. + CleanupOldIndexFiles(indexDirectory, segmentsFile); + + // successfully updated the index, notify the callback that the index is + // ready. + if (callback != null) + { + try + { + callback.Invoke(); + } + catch (Exception e) + { + throw new IOException(e.ToString(), e); + } + } + } + + /// <summary> + /// Gets or sets the <see cref="Util.InfoStream"/> to use for logging messages. + /// </summary> + public InfoStream InfoStream + { + get { return infoStream; } + set { infoStream = value ?? InfoStream.NO_OUTPUT; } + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/IndexRevision.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/IndexRevision.cs b/src/Lucene.Net.Replicator/IndexRevision.cs index 5708bf1..29ea77c 100644 --- a/src/Lucene.Net.Replicator/IndexRevision.cs +++ b/src/Lucene.Net.Replicator/IndexRevision.cs @@ -1,5 +1,3 @@ -//STATUS: DRAFT - 4.8.0 - using System; using System.IO; using System.Collections.Generic; @@ -37,14 +35,13 @@ namespace Lucene.Net.Replicator /// <see cref="SnapshotDeletionPolicy"/> (this means that the given writer's /// <see cref="IndexWriterConfig.IndexDeletionPolicy"/> should return /// <see cref="SnapshotDeletionPolicy"/>). - /// <p> - /// When this revision is <see cref="Release"/>d, it releases the obtained + /// <para/> + /// When this revision is <see cref="Release()"/>d, it releases the obtained /// snapshot as well as calls <see cref="IndexWriter.DeleteUnusedFiles"/> so that the /// snapshotted files are deleted (if they are no longer needed). - /// </p> /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public class IndexRevision : IRevision { @@ -54,9 +51,45 @@ namespace Lucene.Net.Replicator private readonly IndexCommit commit; private readonly SnapshotDeletionPolicy sdp; - public string Version { get; private set; } public IDictionary<string, IList<RevisionFile>> SourceFiles { get; private set; } + // returns a RevisionFile with some metadata + private static RevisionFile CreateRevisionFile(string fileName, Directory directory) + { + return new RevisionFile(fileName, directory.FileLength(fileName)); + } + + /// <summary> + /// Returns a singleton map of the revision files from the given <see cref="IndexCommit"/>. + /// </summary> + public static IDictionary<string, IList<RevisionFile>> RevisionFiles(IndexCommit commit) + { + List<RevisionFile> revisionFiles = commit.FileNames + .Where(file => !string.Equals(file, commit.SegmentsFileName)) + .Select(file => CreateRevisionFile(file, commit.Directory)) + //Note: segments_N must be last + .Union(new[] {CreateRevisionFile(commit.SegmentsFileName, commit.Directory)}) + .ToList(); + return new Dictionary<string, IList<RevisionFile>> + { + { SOURCE, revisionFiles } + }; + } + + /// <summary> + /// Returns a string representation of a revision's version from the given + /// <see cref="IndexCommit"/> + /// </summary> + public static string RevisionVersion(IndexCommit commit) + { + return commit.Generation.ToString("X"); + } + + /// <summary> + /// Constructor over the given <see cref="IndexWriter"/>. Uses the last + /// <see cref="IndexCommit"/> found in the <see cref="Directory"/> managed by the given + /// writer. + /// </summary> public IndexRevision(IndexWriter writer) { sdp = writer.Config.IndexDeletionPolicy as SnapshotDeletionPolicy; @@ -86,6 +119,8 @@ namespace Lucene.Net.Replicator return commit.CompareTo(or.commit); } + public string Version { get; private set; } + public Stream Open(string source, string fileName) { Debug.Assert(source.Equals(SOURCE), string.Format("invalid source; expected={0} got={1}", SOURCE, source)); @@ -102,36 +137,5 @@ namespace Lucene.Net.Replicator { return "IndexRevision version=" + Version + " files=" + SourceFiles; } - - // returns a RevisionFile with some metadata - private static RevisionFile CreateRevisionFile(string fileName, Directory directory) - { - return new RevisionFile(fileName, directory.FileLength(fileName)); - } - - /** Returns a singleton map of the revision files from the given {@link IndexCommit}. */ - public static IDictionary<string, IList<RevisionFile>> RevisionFiles(IndexCommit commit) - { - List<RevisionFile> revisionFiles = commit.FileNames - .Where(file => !string.Equals(file, commit.SegmentsFileName)) - .Select(file => CreateRevisionFile(file, commit.Directory)) - //Note: segments_N must be last - .Union(new[] {CreateRevisionFile(commit.SegmentsFileName, commit.Directory)}) - .ToList(); - return new Dictionary<string, IList<RevisionFile>> - { - { SOURCE, revisionFiles } - }; - } - - /// <summary> - /// Returns a String representation of a revision's version from the given <see cref="IndexCommit"/> - /// </summary> - /// <param name="commit"></param> - /// <returns></returns> - public static string RevisionVersion(IndexCommit commit) - { - return commit.Generation.ToString("X"); - } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/LocalReplicator.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/LocalReplicator.cs b/src/Lucene.Net.Replicator/LocalReplicator.cs index c32f8b7..981eecb 100644 --- a/src/Lucene.Net.Replicator/LocalReplicator.cs +++ b/src/Lucene.Net.Replicator/LocalReplicator.cs @@ -1,12 +1,9 @@ -//STATUS: DRAFT - 4.8.0 - +using Lucene.Net.Support; using System; using System.Collections.Generic; -using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; -using Lucene.Net.Support; namespace Lucene.Net.Replicator { @@ -34,159 +31,104 @@ namespace Lucene.Net.Replicator /// <see cref="SessionToken"/> through which it can /// <see cref="ObtainFile"/> the files of that /// revision. As long as a revision is being replicated, this replicator - /// guarantees that it will not be <seealso cref="IRevision.Release"/>. - /// <para> + /// guarantees that it will not be <see cref="IRevision.Release"/>. + /// <para/> /// Replication sessions expire by default after - /// <seealso cref="DEFAULT_SESSION_EXPIRATION_THRESHOLD"/>, and the threshold can be - /// configured through <seealso cref="ExpirationThreshold"/>. - /// </para> + /// <seea cref="DEFAULT_SESSION_EXPIRATION_THRESHOLD"/>, and the threshold can be + /// configured through <see cref="ExpirationThreshold"/>. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public class LocalReplicator : IReplicator { - /// <summary>Threshold for expiring inactive sessions. Defaults to 30 minutes.</summary> - public const long DEFAULT_SESSION_EXPIRATION_THRESHOLD = 1000 * 60 * 30; - - private long expirationThreshold = DEFAULT_SESSION_EXPIRATION_THRESHOLD; - private readonly object padlock = new object(); - private volatile RefCountedRevision currentRevision; - private volatile bool disposed = false; + private class RefCountedRevision + { + private readonly AtomicInt32 refCount = new AtomicInt32(1); - private readonly AtomicInt32 sessionToken = new AtomicInt32(0); - private readonly Dictionary<string, ReplicationSession> sessions = new Dictionary<string, ReplicationSession>(); + public IRevision Revision { get; private set; } - /// <summary> - /// Returns the expiration threshold. - /// </summary> - public long ExpirationThreshold - { - get { return expirationThreshold; } - set + public RefCountedRevision(IRevision revision) { - lock (padlock) - { - EnsureOpen(); - expirationThreshold = value; - CheckExpiredSessions(); - } + Revision = revision; } - } - public void Publish(IRevision revision) - { - lock (padlock) + /// <summary/> + /// <exception cref="InvalidOperationException"></exception> + public void DecRef() { - EnsureOpen(); + if (refCount.Get() <= 0) + { + throw new InvalidOperationException("this revision is already released"); + } - if (currentRevision != null) + var rc = refCount.DecrementAndGet(); + if (rc == 0) { - int compare = revision.CompareTo(currentRevision.Revision); - if (compare == 0) + bool success = false; + try { - // same revision published again, ignore but release it - revision.Release(); - return; + Revision.Release(); + success = true; } - - if (compare < 0) + finally { - revision.Release(); - throw new ArgumentException(string.Format("Cannot publish an older revision: rev={0} current={1}", revision, currentRevision), "revision"); + if (!success) + { + // Put reference back on failure + refCount.IncrementAndGet(); + } } } + else if (rc < 0) + { + throw new InvalidOperationException(string.Format("too many decRef calls: refCount is {0} after decrement", rc)); + } + } - RefCountedRevision oldRevision = currentRevision; - currentRevision = new RefCountedRevision(revision); - if(oldRevision != null) - oldRevision.DecRef(); - - CheckExpiredSessions(); + public void IncRef() + { + refCount.IncrementAndGet(); } } - /// <summary> - /// TODO - /// </summary> - /// <returns></returns> - public SessionToken CheckForUpdate(string currentVersion) + private class ReplicationSession { - lock (padlock) - { - EnsureOpen(); - if (currentRevision == null) - return null; // no published revisions yet - - if (currentVersion != null && currentRevision.Revision.CompareTo(currentVersion) <= 0) - return null; // currentVersion is newer or equal to latest published revision + public SessionToken Session { get; private set; } + public RefCountedRevision Revision { get; private set; } - // currentVersion is either null or older than latest published revision - currentRevision.IncRef(); + private long lastAccessTime; - string sessionID = sessionToken.IncrementAndGet().ToString(); - SessionToken token = new SessionToken(sessionID, currentRevision.Revision); - sessions[sessionID] = new ReplicationSession(token, currentRevision); - return token; + public ReplicationSession(SessionToken session, RefCountedRevision revision) + { + Session = session; + Revision = revision; + lastAccessTime = Stopwatch.GetTimestamp(); } - } + public bool IsExpired(long expirationThreshold) + { + return lastAccessTime < Stopwatch.GetTimestamp() - expirationThreshold * Stopwatch.Frequency / 1000; // LUCENENET TODO: CurrentTimeMilliseconds() + } - /// <summary> - /// TODO - /// </summary> - /// <exception cref="InvalidOperationException"></exception> - public void Release(string sessionId) - { - lock (padlock) + public void MarkAccessed() { - EnsureOpen(); - ReleaseSession(sessionId); + lastAccessTime = Stopwatch.GetTimestamp(); // LUCENENET TODO: CurrentTimeMilliseconds() } } - /// <summary> - /// TODO - /// </summary> - public Stream ObtainFile(string sessionId, string source, string fileName) - { - lock (padlock) - { - EnsureOpen(); + /// <summary>Threshold for expiring inactive sessions. Defaults to 30 minutes.</summary> + public const long DEFAULT_SESSION_EXPIRATION_THRESHOLD = 1000 * 60 * 30; - ReplicationSession session = sessions[sessionId]; - if (session != null && session.IsExpired(ExpirationThreshold)) - { - ReleaseSession(sessionId); - session = null; - } - // session either previously expired, or we just expired it - if (session == null) - { - throw new SessionExpiredException(string.Format("session ({0}) expired while obtaining file: source={1} file={2}", sessionId, source, fileName)); - } - sessions[sessionId].MarkAccessed(); - return session.Revision.Revision.Open(source, fileName); - } + private long expirationThreshold = DEFAULT_SESSION_EXPIRATION_THRESHOLD; - } + private readonly object padlock = new object(); - /// <summary> - /// TODO - /// </summary> - public void Dispose() - { - if (disposed) - return; + private volatile RefCountedRevision currentRevision; + private volatile bool disposed = false; - lock (padlock) - { - foreach (ReplicationSession session in sessions.Values) - session.Revision.DecRef(); - sessions.Clear(); - } - disposed = true; - } + private readonly AtomicInt32 sessionToken = new AtomicInt32(0); + private readonly IDictionary<string, ReplicationSession> sessions = new Dictionary<string, ReplicationSession>(); /// <exception cref="InvalidOperationException"></exception> private void CheckExpiredSessions() @@ -215,7 +157,7 @@ namespace Lucene.Net.Replicator /// <summary> /// Ensure that replicator is still open, or throw <see cref="ObjectDisposedException"/> otherwise. /// </summary> - /// <exception cref="ObjectDisposedException">This replicator has already been closed</exception> + /// <exception cref="ObjectDisposedException">This replicator has already been disposed.</exception> protected void EnsureOpen() { lock (padlock) @@ -227,80 +169,124 @@ namespace Lucene.Net.Replicator } } - private class RefCountedRevision + public SessionToken CheckForUpdate(string currentVersion) { - private readonly AtomicInt32 refCount = new AtomicInt32(1); + lock (padlock) + { + EnsureOpen(); + if (currentRevision == null) + return null; // no published revisions yet - public IRevision Revision { get; private set; } + if (currentVersion != null && currentRevision.Revision.CompareTo(currentVersion) <= 0) + return null; // currentVersion is newer or equal to latest published revision - public RefCountedRevision(IRevision revision) + // currentVersion is either null or older than latest published revision + currentRevision.IncRef(); + + string sessionID = sessionToken.IncrementAndGet().ToString(); + SessionToken token = new SessionToken(sessionID, currentRevision.Revision); + sessions[sessionID] = new ReplicationSession(token, currentRevision); + return token; + } + } + + public void Dispose() // LUCENENET TODO: API Dispose pattern + { + if (disposed) + return; + + lock (padlock) { - Revision = revision; + foreach (ReplicationSession session in sessions.Values) + session.Revision.DecRef(); + sessions.Clear(); } + disposed = true; + } - /// <summary/> - /// <exception cref="InvalidOperationException"></exception> - public void DecRef() + /// <summary> + /// Gets or sets the expiration threshold. + /// <para/> + /// If a replication session is inactive this + /// long it is automatically expired, and further attempts to operate within + /// this session will throw a <see cref="SessionExpiredException"/>. + /// </summary> + public long ExpirationThreshold + { + get { return expirationThreshold; } + set { - if (refCount.Get() <= 0) + lock (padlock) { - throw new InvalidOperationException("this revision is already released"); + EnsureOpen(); + expirationThreshold = value; + CheckExpiredSessions(); } + } + } - var rc = refCount.DecrementAndGet(); - if (rc == 0) + public Stream ObtainFile(string sessionId, string source, string fileName) + { + lock (padlock) + { + EnsureOpen(); + + ReplicationSession session = sessions[sessionId]; + if (session != null && session.IsExpired(ExpirationThreshold)) { - bool success = false; - try - { - Revision.Release(); - success = true; - } - finally - { - if (!success) - { - // Put reference back on failure - refCount.IncrementAndGet(); - } - } + ReleaseSession(sessionId); + session = null; } - else if (rc < 0) + // session either previously expired, or we just expired it + if (session == null) { - throw new InvalidOperationException(string.Format("too many decRef calls: refCount is {0} after decrement", rc)); + throw new SessionExpiredException(string.Format("session ({0}) expired while obtaining file: source={1} file={2}", sessionId, source, fileName)); } + sessions[sessionId].MarkAccessed(); + return session.Revision.Revision.Open(source, fileName); } - - public void IncRef() - { - refCount.IncrementAndGet(); - } } - private class ReplicationSession + public void Publish(IRevision revision) { - public SessionToken Session { get; private set; } - public RefCountedRevision Revision { get; private set; } + lock (padlock) + { + EnsureOpen(); - private long lastAccessTime; + if (currentRevision != null) + { + int compare = revision.CompareTo(currentRevision.Revision); + if (compare == 0) + { + // same revision published again, ignore but release it + revision.Release(); + return; + } - public ReplicationSession(SessionToken session, RefCountedRevision revision) - { - Session = session; - Revision = revision; - lastAccessTime = Stopwatch.GetTimestamp(); - } + if (compare < 0) + { + revision.Release(); + throw new ArgumentException(string.Format("Cannot publish an older revision: rev={0} current={1}", revision, currentRevision), "revision"); + } + } - public bool IsExpired(long expirationThreshold) - { - return lastAccessTime < Stopwatch.GetTimestamp() - expirationThreshold * Stopwatch.Frequency / 1000; + RefCountedRevision oldRevision = currentRevision; + currentRevision = new RefCountedRevision(revision); + if (oldRevision != null) + oldRevision.DecRef(); + + CheckExpiredSessions(); } + } - public void MarkAccessed() + /// <exception cref="InvalidOperationException"></exception> + public void Release(string sessionId) + { + lock (padlock) { - lastAccessTime = Stopwatch.GetTimestamp(); + EnsureOpen(); + ReleaseSession(sessionId); } } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/Lucene.Net.Replicator.csproj ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/Lucene.Net.Replicator.csproj b/src/Lucene.Net.Replicator/Lucene.Net.Replicator.csproj index 1b0b90f..5efeb74 100644 --- a/src/Lucene.Net.Replicator/Lucene.Net.Replicator.csproj +++ b/src/Lucene.Net.Replicator/Lucene.Net.Replicator.csproj @@ -49,6 +49,9 @@ <ErrorReport>prompt</ErrorReport> <WarningLevel>4</WarningLevel> </PropertyGroup> + <PropertyGroup> + <DefineConstants>$(DefineConstants);FEATURE_SERIALIZABLE</DefineConstants> + </PropertyGroup> <ItemGroup> <Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <HintPath>..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath> @@ -74,8 +77,6 @@ <Compile Include="IndexInputInputStream.cs" /> <Compile Include="IndexReplicationHandler.cs" /> <Compile Include="IndexRevision.cs" /> - <Compile Include="IReplicationHandler.cs" /> - <Compile Include="ISourceDirectoryFactory.cs" /> <Compile Include="LocalReplicator.cs" /> <Compile Include="PerSessionDirectoryFactory.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> @@ -87,9 +88,6 @@ <Compile Include="SessionToken.cs" /> </ItemGroup> <ItemGroup> - <Content Include="Http\package.html" /> - </ItemGroup> - <ItemGroup> <ProjectReference Include="..\Lucene.Net.Facet\Lucene.Net.Facet.csproj"> <Project>{48f7884a-9454-4e88-8413-9d35992cb440}</Project> <Name>Lucene.Net.Facet</Name> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/67882465/src/Lucene.Net.Replicator/PerSessionDirectoryFactory.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Replicator/PerSessionDirectoryFactory.cs b/src/Lucene.Net.Replicator/PerSessionDirectoryFactory.cs index e65c8cb..3661f71 100644 --- a/src/Lucene.Net.Replicator/PerSessionDirectoryFactory.cs +++ b/src/Lucene.Net.Replicator/PerSessionDirectoryFactory.cs @@ -28,13 +28,13 @@ namespace Lucene.Net.Replicator /// deleted. /// </summary> /// <remarks> - /// Lucene.Experimental + /// @lucene.experimental /// </remarks> public class PerSessionDirectoryFactory : ISourceDirectoryFactory { private readonly string workingDirectory; - /** Constructor with the given sources mapping. */ + /// <summary>Constructor with the given sources mapping.</summary> public PerSessionDirectoryFactory(string workingDirectory) { this.workingDirectory = workingDirectory;
