jenkins-bot has submitted this change and it was merged.
Change subject: Elasticsearch 2.1.1 support
......................................................................
Elasticsearch 2.1.1 support
Change-Id: I30ce16391d74b970e221c631deaef3185e911586
---
A .gitreview
M README.md
M pom.xml
M src/main/assemblies/plugin.xml
M src/main/java/org/wikimedia/elasticsearch/swift/SwiftRepositoryPlugin.java
M
src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java
M src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftService.java
M
src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java
M
src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobStore.java
D src/main/resources/es-plugin.properties
A src/main/resources/plugin-descriptor.properties
11 files changed, 100 insertions(+), 68 deletions(-)
Approvals:
EBernhardson: Looks good to me, approved
jenkins-bot: Verified
diff --git a/.gitreview b/.gitreview
new file mode 100644
index 0000000..291fc3f
--- /dev/null
+++ b/.gitreview
@@ -0,0 +1,6 @@
+[gerrit]
+host=gerrit.wikimedia.org
+port=29418
+project=search/repository-swift
+defaultbranch=master
+defaultrebase=0
diff --git a/README.md b/README.md
index 2f90ae4..d33f025 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@
| 0.7 | 1.4.0 | 2014-11-07 |
| 1.6.0 | 1.6.0 | 2015-06-09 |
| 1.7.0 | 1.7.0 | 2015-07-20 |
+| 2.1.1 | 2.1.1 | 2016-05-12 |
Only the versions in the table above should be used. The in-between releases
were buggy and are not recommended.
@@ -27,7 +28,7 @@
}'
```
-See [Snapshot And
Restore](http://www.elasticsearch.org/guide/en/elasticsearch/reference/1.x/modules-snapshots.html)
for more information
+See [Snapshot And
Restore](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html)
for more information
## Settings
diff --git a/pom.xml b/pom.xml
index 3592a09..b168bb9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
<groupId>org.wikimedia.elasticsearch.swift</groupId>
<artifactId>swift-repository-plugin</artifactId>
- <version>1.7.1-SNAPSHOT</version>
+ <version>2.1.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>SwiftRepositoryPlugin</name>
<description>Repository plugin for Elasticsearch backed by
Swift</description>
@@ -33,11 +33,19 @@
<email>[email protected]</email>
<timezone>America/Los Angeles</timezone>
</developer>
+ <developer>
+ <name>Avik Das</name>
+ <email>[email protected]</email>
+ <timezone>America/Los Angeles</timezone>
+ </developer>
</developers>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <elasticsearch.version>1.7.0</elasticsearch.version>
+ <elasticsearch.version>2.1.1</elasticsearch.version>
+ <elasticsearch.plugin.name>swift-repository</elasticsearch.plugin.name>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ <maven.compiler.source>1.7</maven.compiler.source>
</properties>
<dependencies>
@@ -71,6 +79,12 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.2</version>
<optional>true</optional>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
@@ -109,10 +123,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml
index 9e3ccb3..358363f 100644
--- a/src/main/assemblies/plugin.xml
+++ b/src/main/assemblies/plugin.xml
@@ -5,6 +5,13 @@
<format>zip</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
+ <files>
+ <file>
+
<source>${project.basedir}/src/main/resources/plugin-descriptor.properties</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ </file>
+ </files>
<dependencySets>
<dependencySet>
<outputDirectory>/</outputDirectory>
diff --git
a/src/main/java/org/wikimedia/elasticsearch/swift/SwiftRepositoryPlugin.java
b/src/main/java/org/wikimedia/elasticsearch/swift/SwiftRepositoryPlugin.java
index 0ad94de..236aa03 100644
--- a/src/main/java/org/wikimedia/elasticsearch/swift/SwiftRepositoryPlugin.java
+++ b/src/main/java/org/wikimedia/elasticsearch/swift/SwiftRepositoryPlugin.java
@@ -1,20 +1,20 @@
package org.wikimedia.elasticsearch.swift;
-import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.AbstractPlugin;
+import
org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
+import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoriesModule;
import org.wikimedia.elasticsearch.swift.repositories.SwiftRepository;
-import org.wikimedia.elasticsearch.swift.repositories.SwiftRepositoryModule;
import org.wikimedia.elasticsearch.swift.repositories.SwiftService;
import java.util.Collection;
+import java.util.Collections;
/**
* Our base plugin stuff.
*/
-public class SwiftRepositoryPlugin extends AbstractPlugin {
+public class SwiftRepositoryPlugin extends Plugin {
// Elasticsearch settings
private final Settings settings;
@@ -45,14 +45,11 @@
/**
* Register our services, if needed.
*/
- @Override
+
@SuppressWarnings("rawtypes")
- public Collection<Class<? extends LifecycleComponent>> services() {
- Collection<Class<? extends LifecycleComponent>> services =
Lists.newArrayList();
- if (settings.getAsBoolean("swift.repository.enabled", true)) {
- services.add(SwiftService.class);
- }
- return services;
+ @Override
+ public Collection<Class<? extends LifecycleComponent>> nodeServices() {
+ return Collections.<Class<? extends
LifecycleComponent>>singleton(SwiftService.class);
}
/**
@@ -61,7 +58,7 @@
*/
public void onModule(RepositoriesModule repositoriesModule) {
if (settings.getAsBoolean("swift.repository.enabled", true)) {
- repositoriesModule.registerRepository(SwiftRepository.TYPE,
SwiftRepositoryModule.class);
+ repositoriesModule.registerRepository(SwiftRepository.TYPE,
SwiftRepository.class, BlobStoreIndexShardRepository.class);
}
}
}
diff --git
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java
index 1d4ecd7..2e4c169 100644
---
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java
+++
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java
@@ -45,7 +45,7 @@
* an instance of SwiftService
*/
@Inject
- protected SwiftRepository(RepositoryName name, RepositorySettings
repositorySettings, IndexShardRepository indexShardRepository, SwiftService
swiftService) {
+ public SwiftRepository(RepositoryName name, RepositorySettings
repositorySettings, IndexShardRepository indexShardRepository, SwiftService
swiftService) {
super(name.getName(), repositorySettings, indexShardRepository);
String url = repositorySettings.settings().get("swift_url");
@@ -65,10 +65,10 @@
Account account = SwiftAccountFactory.createAccount(swiftService, url,
username, password, tenantName, authMethod);
blobStore = new SwiftBlobStore(settings, account, container);
-
this.chunkSize =
repositorySettings.settings().getAsBytesSize("chunk_size",
- componentSettings.getAsBytesSize("chunk_size", new
ByteSizeValue(5, ByteSizeUnit.GB)));
- this.compress = repositorySettings.settings().getAsBoolean("compress",
componentSettings.getAsBoolean("compress", false));
+ settings.getAsBytesSize("chunk_size", new ByteSizeValue(5,
ByteSizeUnit.GB)));
+ this.compress = repositorySettings.settings().getAsBoolean("compress",
+ settings.getAsBoolean("compress", false));
this.basePath = BlobPath.cleanPath();
}
diff --git
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftService.java
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftService.java
index cbcb90c..facd0d6 100644
---
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftService.java
+++
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftService.java
@@ -1,7 +1,6 @@
package org.wikimedia.elasticsearch.swift.repositories;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@@ -35,6 +34,7 @@
* The username
* @param password
* The password
+ * @return swift Account
*/
public synchronized Account swiftBasic(String url, String username, String
password) {
if (swiftUser != null) {
@@ -45,7 +45,7 @@
AccountConfig conf = getStandardConfig(url, username, password,
AuthenticationMethod.BASIC);
swiftUser = new AccountFactory(conf).createAccount();
} catch (CommandException ce) {
- throw new ElasticsearchIllegalArgumentException("Unable to
authenticate to Swift Basic " + url + "/" + username + "/" + password, ce);
+ throw new ElasticsearchException("Unable to authenticate to Swift
Basic " + url + "/" + username + "/" + password, ce);
}
return swiftUser;
}
@@ -60,7 +60,7 @@
conf.setTenantName(tenantName);
swiftUser = new AccountFactory(conf).createAccount();
} catch (CommandException ce) {
- throw new ElasticsearchIllegalArgumentException(
+ throw new ElasticsearchException(
"Unable to authenticate to Swift Keystone " + url + "/" +
username + "/" + password + "/" + tenantName, ce);
}
return swiftUser;
@@ -75,7 +75,7 @@
AccountConfig conf = getStandardConfig(url, username, password,
AuthenticationMethod.TEMPAUTH);
swiftUser = new AccountFactory(conf).createAccount();
} catch (CommandException ce) {
- throw new ElasticsearchIllegalArgumentException("Unable to
authenticate to Swift Temp", ce);
+ throw new ElasticsearchException("Unable to authenticate to Swift
Temp", ce);
}
return swiftUser;
}
diff --git
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java
index 3171385..d280aa0 100644
---
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java
+++
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java
@@ -5,17 +5,17 @@
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
-import org.elasticsearch.common.collect.ImmutableMap;
+import org.elasticsearch.common.bytes.BytesReference;
import org.javaswift.joss.model.Directory;
import org.javaswift.joss.model.DirectoryOrObject;
import org.javaswift.joss.model.StoredObject;
+import com.google.common.collect.ImmutableMap;
+
import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
import java.util.Collection;
/**
@@ -56,17 +56,17 @@
* @param blobName A blob to delete
*/
@Override
- public boolean deleteBlob(String blobName) throws IOException {
+ public void deleteBlob(String blobName) throws IOException {
StoredObject object = blobStore.swift().getObject(buildKey(blobName));
if (object.exists()) {
object.delete();
}
- return true;
}
/**
* Get the blobs matching a given prefix
* @param blobNamePrefix The prefix to look for blobs with
+ * @return blobs metadata
*/
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable
String blobNamePrefix) throws IOException {
@@ -101,48 +101,44 @@
/**
* Build a key for a blob name, based on the keyPath
* @param blobName The blob name to build a key for
+ * @return the key
*/
protected String buildKey(String blobName) {
return keyPath + blobName;
}
+ @Override
+ public void move(String sourceBlobname, String destinationBlobname) throws
IOException {
+
+ String source = buildKey(sourceBlobname);
+ String target = buildKey(destinationBlobname);
+ if (blobExists(sourceBlobname)) {
+ blobStore.moveBlobStorage(source, target);
+ }
+ }
+
/**
* Fetch a given blob into a BufferedInputStream
* @param blobName The blob name to read
+ * @return a stream
*/
@Override
- public InputStream openInput(String blobName) throws IOException {
- return new BufferedInputStream(
-
blobStore.swift().getObject(buildKey(blobName)).downloadObjectAsInputStream(),
+ public InputStream readBlob(String blobName) throws IOException {
+ return new
BufferedInputStream(blobStore.swift().getObject(buildKey(blobName)).downloadObjectAsInputStream(),
blobStore.bufferSizeInBytes());
}
@Override
- public OutputStream createOutput(final String blobName) throws IOException
{
- // need to remove old file if already exist
- deleteBlob(blobName);
+ public void writeBlob(String blobName, final BytesReference bytes) throws
IOException {
+ // need to remove old file if already exist
+ deleteBlob(blobName);
+ blobStore.swift().getObject(buildKey(blobName)).uploadObject(new
ByteArrayInputStream(bytes.array(), bytes.arrayOffset(), bytes.length()));
+ }
- final PipedInputStream in = new PipedInputStream();
-
- // We'll need to store this thread and make sure it terminates when the
output stream is closed.
- final Thread transport = new Thread(new Runnable(){
- public void run(){
- blobStore.swift().getObject(buildKey(blobName)).uploadObject(in);
- }
- });
- transport.start();
-
- return new PipedOutputStream(in) {
- @Override
- public void close() throws IOException {
- try {
- // Close output, close the thread
- super.close();
- transport.join();
- } catch(InterruptedException e) {
- throw new IOException("Swift input/output shenanigans.", e);
- }
- }
- };
+ @Override
+ public void writeBlob(String blobName, InputStream in, long blobSize)
throws IOException {
+ // need to remove old file if already exist
+ deleteBlob(blobName);
+ blobStore.swift().getObject(buildKey(blobName)).uploadObject(in);
}
}
diff --git
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobStore.java
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobStore.java
index 61877e6..898ca37 100644
---
a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobStore.java
+++
b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobStore.java
@@ -24,13 +24,12 @@
/**
* Constructor. Sets up the container mostly.
* @param settings Settings for our repository. Only care about buffer
size.
- * @param auth
- * @param container
+ * @param auth swift account info
+ * @param container swift container
*/
public SwiftBlobStore(Settings settings, Account auth, String container) {
super(settings);
this.bufferSizeInBytes = (int)settings.getAsBytesSize("buffer_size",
new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();
-
swift = auth.getContainer(container);
if (!swift.exists()) {
swift.create();
@@ -38,15 +37,26 @@
}
}
+ public boolean moveBlobStorage(String sourceblob,String destinationblob){
+ StoredObject sourceObject = swift.getObject(sourceblob);
+ if(sourceObject.exists()) {
+ StoredObject newObject = swift.getObject(destinationblob);
+ sourceObject.copyObject(swift, newObject);
+ sourceObject.delete();
+ return true;
+ }
+ return false;
+ }
+
/**
- * Get the container
+ * @return the container
*/
public Container swift() {
return swift;
}
/**
- * Get our buffer size
+ * @return buffer size
*/
public int bufferSizeInBytes() {
return bufferSizeInBytes;
diff --git a/src/main/resources/es-plugin.properties
b/src/main/resources/es-plugin.properties
deleted file mode 100644
index 7d2b18b..0000000
--- a/src/main/resources/es-plugin.properties
+++ /dev/null
@@ -1,2 +0,0 @@
-plugin=${project.groupId}.SwiftRepositoryPlugin
-version=${project.version}
diff --git a/src/main/resources/plugin-descriptor.properties
b/src/main/resources/plugin-descriptor.properties
new file mode 100644
index 0000000..c9ca23b
--- /dev/null
+++ b/src/main/resources/plugin-descriptor.properties
@@ -0,0 +1,7 @@
+name=${elasticsearch.plugin.name}
+description=${project.description}
+version=${project.version}
+jvm=true
+classname=org.wikimedia.elasticsearch.swift.SwiftRepositoryPlugin
+java.version=${maven.compiler.target}
+elasticsearch.version=${elasticsearch.version}
--
To view, visit https://gerrit.wikimedia.org/r/288891
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I30ce16391d74b970e221c631deaef3185e911586
Gerrit-PatchSet: 5
Gerrit-Project: search/repository-swift
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
Gerrit-Reviewer: Avik07011988 <[email protected]>
Gerrit-Reviewer: DCausse <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
