eribeiro commented on a change in pull request #864: SOLR-13101 : Shared
storage support in SolrCloud
URL: https://github.com/apache/lucene-solr/pull/864#discussion_r324030543
##########
File path:
solr/core/src/java/org/apache/solr/store/blob/client/LocalStorageClient.java
##########
@@ -0,0 +1,259 @@
+package org.apache.solr.store.blob.client;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Class that handles reads and writes of solr blob files to the local file
system.
+ */
+public class LocalStorageClient implements CoreStorageClient {
+
+ /** The directory on the local file system where blobs will be stored. */
+ public static final String BLOB_STORE_LOCAL_FS_ROOT_DIR_PROPERTY =
"blob.local.dir";
+
+ private final String blobStoreRootDir =
System.getProperty(BLOB_STORE_LOCAL_FS_ROOT_DIR_PROPERTY,
"/tmp/BlobStoreLocal/");
+
+ public LocalStorageClient() throws IOException {
+ File rootDir = new File(blobStoreRootDir);
+ rootDir.mkdirs(); // Might create the directory... or not
+ if (!rootDir.isDirectory()) {
+ throw new IOException("Can't create local Blob root directory " +
rootDir.getAbsolutePath());
+ }
+ }
+
+ private File getCoreRootDir(String blobName) {
+ return new File(BlobClientUtils.concatenatePaths(blobStoreRootDir,
blobName));
+ }
+
+ @Override
+ public String pushStream(String blobName, InputStream is, long
contentLength, String fileNamePrefix) throws BlobException {
+ try {
+ createCoreStorage(blobName);
+ String blobPath = createNewNonExistingBlob(blobName, fileNamePrefix);
+
+ Files.copy(is, Paths.get(getBlobAbsolutePath(blobPath)),
StandardCopyOption.REPLACE_EXISTING);
+
+ assert new File(getBlobAbsolutePath(blobPath)).length() == contentLength;
+
+ return blobPath;
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ /**
+ * Picks a unique name for a new blob for the given core.<p>
+ * The current implementation creates a file, but eventually we just pick up
a random blob name then delegate to S3...
+ * @return the blob file name, including the "path" part of the name
+ */
+ private String createNewNonExistingBlob(String blobName, String
fileNamePrefix) throws BlobException {
+ try {
+ String blobPath = BlobClientUtils.generateNewBlobCorePath(blobName,
fileNamePrefix);
+ final File blobFile = new File(getBlobAbsolutePath(blobPath));
+ if (blobFile.exists()) {
+ // Not expecting this ever to happen. In theory we could just do
"continue" here to try a new
+ // name. For now throwing an exception to make sure we don't run into
this...
+ // continue;
+ throw new IllegalStateException("The random file name chosen using
UUID already exists. Very worrying! " + blobFile.getAbsolutePath());
+ }
+
+ return blobPath;
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ @Override
+ public InputStream pullStream(String blobPath) throws BlobException {
+ try {
+ File blobFile = new File(getBlobAbsolutePath(blobPath));
+ return new FileInputStream(blobFile);
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ @Override
+ public void pushCoreMetadata(String sharedStoreName, String
blobCoreMetadataName, BlobCoreMetadata bcm) throws BlobException {
+ try {
+ createCoreStorage(sharedStoreName);
+ ToFromJson<BlobCoreMetadata> converter = new ToFromJson<>();
+ String json = converter.toJson(bcm);
+
+ // Constant path under which the core metadata is stored in the Blob
store (the only blob stored under a constant path!)
+ String blobMetadataPath =
getBlobAbsolutePath(getBlobMetadataName(sharedStoreName, blobCoreMetadataName));
+ final File blobMetadataFile = new File(blobMetadataPath);
+
+ // Writing to the file assumed atomic, the file cannot be observed
midway. Might not hold here but should be the case
+ // with a real S3 implementation.
+ try (PrintWriter out = new PrintWriter(blobMetadataFile)){
+ out.println(json);
+ }
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ @Override
+ public BlobCoreMetadata pullCoreMetadata(String sharedStoreName, String
blobCoreMetadataName) throws BlobException {
+ try {
+ if (!coreMetadataExists(sharedStoreName, blobCoreMetadataName)) {
+ return null;
+ }
+
+ String blobMetadataPath =
getBlobAbsolutePath(getBlobMetadataName(sharedStoreName, blobCoreMetadataName));
+ File blobMetadataFile = new File(blobMetadataPath);
+
+ String json = new String(Files.readAllBytes(blobMetadataFile.toPath()));
+ ToFromJson<BlobCoreMetadata> converter = new ToFromJson<>();
+ return converter.fromJson(json, BlobCoreMetadata.class);
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ @Override
+ public boolean coreMetadataExists(String sharedStoreName, String
blobCoreMetadataName) throws BlobException {
+ try {
+ String blobMetadataPath =
getBlobAbsolutePath(getBlobMetadataName(sharedStoreName, blobCoreMetadataName));
+ File coreMetadataFile = new File(blobMetadataPath);
+ return coreMetadataFile.exists();
+ } catch (Exception ex) {
+ throw new BlobException(ex);
+ }
+ }
+
+ /**
+ * Prefixes the given path with the blob store root directory on the local
FS
+ */
+ private String getBlobAbsolutePath(String blobPath) {
+ return BlobClientUtils.concatenatePaths(blobStoreRootDir, blobPath);
+ }
+
+ private void createCoreStorage(String blobName) throws Exception {
+ File coreRootDir = getCoreRootDir(blobName);
+ coreRootDir.mkdirs();
+ if (!coreRootDir.isDirectory()) {
+ throw new IOException("Can't create Blob core root directory " +
coreRootDir.getAbsolutePath());
+ }
+ }
+
+ @Override
+ public void deleteCore(String blobName) throws BlobException {
+ try {
+ Path path = Paths.get(getBlobAbsolutePath(blobName));
+ Files.walk(path)
+ // Since this traversal includes the root directory
+ // we need to reverse because we can't delete non-empty directories
+ .sorted(Comparator.reverseOrder())
+ .map(Path::toFile)
+ .forEach(file -> {
+ file.delete();
+ });
+ } catch (Exception ex) {
+ // In case the path doesn't exist, we'll just swallow the exception
because it's not an issue,
+ // especially in test clean up.
+ }
+ }
+
+ /**
+ * On the local FS implementation we'll just delete blob files individually
+ */
+ @Override
+ public void deleteBlobs(Collection<String> paths) throws BlobException {
+ try {
+ for (String blobPath : paths) {
+ final File blobFile = new File(getBlobAbsolutePath(blobPath));
+ blobFile.delete();
Review comment:
`File.delete()` returns `false` if the file cannot be deleted. Maybe it is
the case of using `Files` utility class that throws an exception if the file
cannot be deleted?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org
