Repository: beam Updated Branches: refs/heads/master 29b828235 -> b23610409
[BEAM-2005] Swap to use Lists within MatchResult instead of arrays. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a83524c2 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a83524c2 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a83524c2 Branch: refs/heads/master Commit: a83524c2c092d62953851d7b2885338b71ddaced Parents: 29b8282 Author: Luke Cwik <[email protected]> Authored: Fri Apr 28 16:27:53 2017 -0700 Committer: Luke Cwik <[email protected]> Committed: Fri Apr 28 17:15:52 2017 -0700 ---------------------------------------------------------------------- .../org/apache/beam/sdk/io/FileBasedSource.java | 6 +- .../org/apache/beam/sdk/io/FileSystems.java | 11 ++-- .../org/apache/beam/sdk/io/LocalFileSystem.java | 9 ++- .../org/apache/beam/sdk/io/fs/MatchResult.java | 63 +++++++++----------- .../beam/sdk/util/NumberedShardedFile.java | 5 +- .../apache/beam/sdk/io/LocalFileSystemTest.java | 3 +- .../beam/sdk/io/gcp/storage/GcsFileSystem.java | 4 +- 7 files changed, 45 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java index 7f89c1c..2beb5e0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java @@ -210,14 +210,14 @@ public abstract class FileBasedSource<T> extends OffsetBasedSource<T> { "Error matching the pattern or glob %s: status %s", pattern, result.status()); - Metadata[] allMatches = result.metadata(); + List<Metadata> allMatches = result.metadata(); for (Metadata metadata : allMatches) { totalSize += metadata.sizeBytes(); } LOG.info( "Filepattern {} matched {} files with total size {}", fileOrPatternSpec.get(), - allMatches.length, + allMatches.size(), totalSize); return totalSize; } else { @@ -383,7 +383,7 @@ public abstract class FileBasedSource<T> extends OffsetBasedSource<T> { private static List<Metadata> expandFilePattern(String fileOrPatternSpec) throws IOException { MatchResult matches = Iterables.getOnlyElement(FileSystems.match(Collections.singletonList(fileOrPatternSpec))); - LOG.info("Matched {} files for pattern {}", matches.metadata().length, fileOrPatternSpec); + LOG.info("Matched {} files for pattern {}", matches.metadata().size(), fileOrPatternSpec); return ImmutableList.copyOf(matches.metadata()); } http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java index c41ee48..e4f00ea 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -122,16 +121,16 @@ public class FileSystems { throw new IOException( String.format("Error matching file spec %s: status %s", spec, matchResult.status())); } - Metadata[] metadata = matchResult.metadata(); - if (metadata.length != 1) { + List<Metadata> metadata = matchResult.metadata(); + if (metadata.size() != 1) { throw new IOException( String.format( "Expecting spec %s to match exactly one file, but matched %s: %s", spec, - metadata.length, - Arrays.toString(metadata))); + metadata.size(), + metadata)); } - return metadata[0]; + return metadata.get(0); } /** http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java index 235b77d..b732bee 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java @@ -39,6 +39,7 @@ import java.nio.file.PathMatcher; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import org.apache.beam.sdk.io.fs.CreateOptions; @@ -55,8 +56,6 @@ class LocalFileSystem extends FileSystem<LocalResourceId> { private static final Logger LOG = LoggerFactory.getLogger(LocalFileSystem.class); - private static final Metadata[] EMPTY_METADATA = new Metadata[0]; - LocalFileSystem() { } @@ -180,12 +179,12 @@ class LocalFileSystem extends FileSystem<LocalResourceId> { File file = Paths.get(spec).toFile(); if (file.exists()) { - return MatchResult.create(Status.OK, new Metadata[]{toMetadata(file)}); + return MatchResult.create(Status.OK, ImmutableList.of(toMetadata(file))); } File parent = file.getAbsoluteFile().getParentFile(); if (!parent.exists()) { - return MatchResult.create(Status.NOT_FOUND, EMPTY_METADATA); + return MatchResult.create(Status.NOT_FOUND, Collections.<Metadata>emptyList()); } // Method getAbsolutePath() on Windows platform may return something like @@ -223,7 +222,7 @@ class LocalFileSystem extends FileSystem<LocalResourceId> { Status.NOT_FOUND, new FileNotFoundException(String.format("No files found for spec: %s.", spec))); } else { - return MatchResult.create(Status.OK, result.toArray(new Metadata[result.size()])); + return MatchResult.create(Status.OK, result); } } http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java index ab11d66..642c049 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/MatchResult.java @@ -20,6 +20,7 @@ package org.apache.beam.sdk.io.fs; import com.google.auto.value.AutoValue; import java.io.IOException; import java.io.Serializable; +import java.util.List; /** * The result of {@link org.apache.beam.sdk.io.FileSystem#match}. @@ -31,52 +32,44 @@ public abstract class MatchResult { /** * Returns a {@link MatchResult} given the {@link Status} and {@link Metadata}. */ - public static MatchResult create(final Status status, final Metadata[] metadata) { - return new MatchResult() { - @Override - public Status status() { - return status; - } - - @Override - public Metadata[] metadata() throws IOException { - return metadata; - } - }; + public static MatchResult create(Status status, List<Metadata> metadata) { + return new AutoValue_MatchResult_Success(status, metadata); + } + + @AutoValue + abstract static class Success extends MatchResult { + abstract List<Metadata> getMetadata(); + + @Override + public List<Metadata> metadata() throws IOException { + return getMetadata(); + } } /** * Returns a {@link MatchResult} given the {@link Status} and {@link IOException}. */ public static MatchResult create(final Status status, final IOException e) { - return new MatchResult() { - @Override - public Status status() { - return status; - } - - @Override - public Metadata[] metadata() throws IOException { - throw e; - } - }; + return new AutoValue_MatchResult_Failure(status, e); + } + + @AutoValue + abstract static class Failure extends MatchResult { + abstract IOException getException(); + + @Override + public List<Metadata> metadata() throws IOException { + throw getException(); + } } /** * Returns a {@link MatchResult} with {@link Status#UNKNOWN}. */ public static MatchResult unknown() { - return new MatchResult() { - @Override - public Status status() { - return Status.UNKNOWN; - } - - @Override - public Metadata[] metadata() throws IOException { - throw new IOException("MatchResult status is UNKNOWN, and metadata is not available."); - } - }; + return new AutoValue_MatchResult_Failure( + Status.UNKNOWN, + new IOException("MatchResult status is UNKNOWN, and metadata is not available.")); } /** @@ -87,7 +80,7 @@ public abstract class MatchResult { /** * {@link Metadata} of matched files. */ - public abstract Metadata[] metadata() throws IOException; + public abstract List<Metadata> metadata() throws IOException; /** * {@link Metadata} of a matched file. http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java index e904382..786cdcb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java @@ -37,7 +37,6 @@ import java.io.Reader; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.regex.Matcher; @@ -120,8 +119,8 @@ public class NumberedShardedFile implements ShardedFile { do { try { // Match inputPath which may contains glob - Collection<Metadata> files = Arrays.asList(Iterables.getOnlyElement( - FileSystems.match(ImmutableList.of(filePattern))).metadata()); + Collection<Metadata> files = Iterables.getOnlyElement( + FileSystems.match(ImmutableList.of(filePattern))).metadata(); LOG.debug("Found {} file(s) by matching the path: {}", files.size(), filePattern); http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java index ac4fe61..048908f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java @@ -40,7 +40,6 @@ import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions; import org.apache.beam.sdk.io.fs.MatchResult; @@ -357,7 +356,7 @@ public class LocalFileSystemTest { @Override public Iterable<Metadata> apply(MatchResult matchResult) { try { - return Arrays.asList(matchResult.metadata()); + return matchResult.metadata(); } catch (IOException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/beam/blob/a83524c2/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java index ff71f3c..2663864 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsFileSystem.java @@ -197,7 +197,7 @@ class GcsFileSystem extends FileSystem<GcsResourceId> { } pageToken = objects.getNextPageToken(); } while (pageToken != null); - return MatchResult.create(Status.OK, results.toArray(new Metadata[results.size()])); + return MatchResult.create(Status.OK, results); } /** @@ -226,7 +226,7 @@ class GcsFileSystem extends FileSystem<GcsResourceId> { } else { StorageObject object = objectOrException.storageObject(); assert object != null; // fix a warning; guaranteed by StorageObjectOrIOException semantics. - return MatchResult.create(Status.OK, new Metadata[]{toMetadata(object)}); + return MatchResult.create(Status.OK, ImmutableList.of(toMetadata(object))); } }
