http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java index 8f299bb..7bf3909 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java @@ -18,8 +18,16 @@ package org.apache.drill.exec.server.rest; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.xml.bind.annotation.XmlRootElement; + import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.coord.ClusterCoordinator; @@ -34,14 +42,8 @@ import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.rpc.user.UserResultsListener; import org.apache.drill.exec.vector.ValueVector; -import javax.xml.bind.annotation.XmlRootElement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; @XmlRootElement public class QueryWrapper {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java index 79150d9..c99c49b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java @@ -17,13 +17,8 @@ */ package org.apache.drill.exec.server.rest; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.drill.exec.server.options.OptionManager; -import org.apache.drill.exec.server.options.OptionValue; -import org.apache.drill.exec.server.options.OptionValue.Kind; -import org.apache.drill.exec.work.WorkManager; -import org.glassfish.jersey.server.mvc.Viewable; +import java.util.LinkedList; +import java.util.List; import javax.inject.Inject; import javax.ws.rs.Consumes; @@ -31,12 +26,17 @@ import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; -import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.xml.bind.annotation.XmlRootElement; -import java.util.LinkedList; -import java.util.List; + +import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.Kind; +import org.apache.drill.exec.work.WorkManager; +import org.glassfish.jersey.server.mvc.Viewable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; @Path("/") public class StatusResources { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java index e841430..49f387c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java @@ -17,16 +17,12 @@ */ package org.apache.drill.exec.server.rest; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.store.StoragePlugin; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.sys.PStoreProvider; -import org.glassfish.jersey.server.mvc.Viewable; +import java.io.IOException; +import java.io.StringReader; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; import javax.inject.Inject; import javax.ws.rs.Consumes; @@ -39,12 +35,17 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.xml.bind.annotation.XmlRootElement; -import java.io.IOException; -import java.io.StringReader; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.glassfish.jersey.server.mvc.Viewable; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; @Path("/") public class StorageResources { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ThreadsResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ThreadsResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ThreadsResources.java index 0fc8055..def5acb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ThreadsResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ThreadsResources.java @@ -17,13 +17,13 @@ */ package org.apache.drill.exec.server.rest; -import org.glassfish.jersey.server.mvc.Viewable; - import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import org.glassfish.jersey.server.mvc.Viewable; + @Path("/threads") public class ThreadsResources { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MetricsResources.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebResourceServer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebResourceServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebResourceServer.java index 33b9be9..249cb8c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebResourceServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebResourceServer.java @@ -49,7 +49,7 @@ public class WebResourceServer { String s = "rest/www/" + path; ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = new BufferedInputStream(cl.getResource(s).openStream()); - + String mime = "text/plain"; if (s.endsWith(".js")) { mime = "text/javascript"; @@ -58,14 +58,14 @@ public class WebResourceServer { } else { mime = URLConnection.guessContentTypeFromStream(is); } - + byte[] d = IOUtils.toByteArray(is); - return Response.ok(d).type(mime).build(); + return Response.ok(d).type(mime).build(); } catch (Exception e) { e.printStackTrace(); e.printStackTrace(System.out); } - + return Response.noContent().status(Status.NOT_FOUND).build(); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java index ce109d9..ff6e13c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java @@ -17,8 +17,6 @@ */ package org.apache.drill.exec.service; -import io.netty.channel.nio.NioEventLoopGroup; - import java.io.Closeable; import java.io.IOException; import java.net.InetAddress; @@ -28,7 +26,6 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.rpc.NamedThreadFactory; import org.apache.drill.exec.rpc.TransportCheck; import org.apache.drill.exec.rpc.control.Controller; import org.apache.drill.exec.rpc.control.ControllerImpl; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java index 209ec8f..554b3e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java @@ -20,14 +20,15 @@ package org.apache.drill.exec.store; import java.util.Collection; import java.util.Map; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.vector.ValueVector; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + public abstract class AbstractRecordReader implements RecordReader { private static final String COL_NULL_ERROR = "Columns cannot be null. Use star column to select all fields."; private static final String COL_EMPTY_ERROR = "Readers needs at least a column to read."; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java index f34b1e3..90e3ef4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java @@ -22,18 +22,17 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import net.hydromatic.linq4j.expressions.DefaultExpression; import net.hydromatic.linq4j.expressions.Expression; import net.hydromatic.optiq.Function; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; - import net.hydromatic.optiq.Table; + import org.apache.drill.exec.planner.logical.CreateTableEntry; -import org.apache.drill.exec.planner.logical.DrillTable; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; public abstract class AbstractSchema implements Schema{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSchema.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java index 9cdfe24..b032fce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java @@ -32,7 +32,7 @@ public abstract class AbstractStoragePlugin implements StoragePlugin{ protected AbstractStoragePlugin(){ } - + @Override public boolean supportsRead() { return false; @@ -52,7 +52,7 @@ public abstract class AbstractStoragePlugin implements StoragePlugin{ public AbstractGroupScan getPhysicalScan(JSONOptions selection) throws IOException { return getPhysicalScan(selection, AbstractGroupScan.ALL_COLUMNS); } - + @Override public AbstractGroupScan getPhysicalScan(JSONOptions selection, List<SchemaPath> columns) throws IOException { throw new UnsupportedOperationException(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java index f7d3c67..5e00f08 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java @@ -37,9 +37,9 @@ public class ClassPathFileSystem extends FileSystem{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassPathFileSystem.class); static final String ERROR_MSG = "ClassPathFileSystem is read only."; - + private Path working; - + @Override public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException { throw new IOException(ERROR_MSG); @@ -68,18 +68,18 @@ public class ClassPathFileSystem extends FileSystem{ } return file; } - + @Override public FileStatus getFileStatus(Path arg0) throws IOException { String file = getFileName(arg0); - - try{ + + try{ URL url = Resources.getResource(file); if(url == null){ throw new IOException(String.format("Unable to find path %s.", arg0.toString())); } - - + + return new FileStatus(Resources.asByteSource(url).size(), false, 1, 8096, System.currentTimeMillis(), arg0); }catch(RuntimeException e){ throw new IOException(String.format("Failure trying to load file %s", arg0), e); @@ -130,7 +130,7 @@ public class ClassPathFileSystem extends FileSystem{ public void setWorkingDirectory(Path arg0) { this.working = arg0; } - + public static void main(String[] args) throws Exception{ URI uri = new URI("classpath:///"); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java index 42cdcc3..09495f5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java @@ -17,16 +17,15 @@ */ package org.apache.drill.exec.store; +import java.util.Map; + import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.memory.OutOfMemoryException; +import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; -import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.vector.ValueVector; -import java.util.Map; - public interface RecordReader { public static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024; @@ -34,7 +33,7 @@ public interface RecordReader { /** * Configure the RecordReader with the provided schema and the record batch that should be written to. - * + * * @param output * The place where output for a particular scan should be written. The record reader is responsible for * mutating the set of schema values for that particular record. @@ -53,8 +52,8 @@ public interface RecordReader { /** - * Increment record reader forward, writing into the provided output batch. - * + * Increment record reader forward, writing into the provided output batch. + * * @return The number of additional records added to the output. */ public abstract int next(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordRecorder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordRecorder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordRecorder.java index 17dbdb1..ffaff8f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordRecorder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordRecorder.java @@ -23,14 +23,14 @@ import org.apache.drill.exec.record.RecordBatch; public interface RecordRecorder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordRecorder.class); - + public void setup() throws IOException; - + /** - * + * * @param batch * @return */ public boolean record(RecordBatch batch); - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java index fb58c3d..5d7ea2c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java @@ -32,8 +32,8 @@ public class ResourceInputStream extends ByteArrayInputStream implements Seekabl super(bytes); } - - + + @Override public void readFully(long position, byte[] buffer) throws IOException { @@ -65,7 +65,7 @@ public class ResourceInputStream extends ByteArrayInputStream implements Seekabl System.arraycopy(buf, start, b, off, len); return len; } - + @Override public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { int l = read(position, buffer, offset, length); @@ -77,7 +77,7 @@ public class ResourceInputStream extends ByteArrayInputStream implements Seekabl return pos; } - + @Override public int read(byte[] b) throws IOException { int l = read(pos, b, 0, b.length); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java index 5acbe78..feadabd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.store; import net.hydromatic.optiq.SchemaPlus; -import org.apache.drill.exec.rpc.user.DrillUser; import org.apache.drill.exec.rpc.user.UserSession; public interface SchemaFactory { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index 653d69d..ef5978c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -35,7 +35,7 @@ public interface StoragePlugin extends SchemaFactory { /** * Get the physical scan operator for the particular GroupScan (read) node. - * + * * @param selection * The configured storage engine specific selection. * @return @@ -46,7 +46,7 @@ public interface StoragePlugin extends SchemaFactory { /** * Get the physical scan operator for the particular GroupScan (read) node. - * + * * @param selection * The configured storage engine specific selection. * @param columns http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java index 2d8dee8..2c0d8b8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SubSchemaWrapper.java @@ -17,14 +17,16 @@ */ package org.apache.drill.exec.store; -import com.google.common.collect.ImmutableList; +import java.util.Collection; +import java.util.Set; + import net.hydromatic.optiq.Function; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.Table; + import org.apache.drill.exec.planner.logical.CreateTableEntry; -import java.util.Collection; -import java.util.Set; +import com.google.common.collect.ImmutableList; public class SubSchemaWrapper extends AbstractSchema { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java index ef26d11..8387d49 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java @@ -31,7 +31,7 @@ public class VectorHolder { private int currentLength; private boolean repeated; - + public VectorHolder(int length, ValueVector vector) { this.length = length; this.vector = vector; @@ -39,7 +39,7 @@ public class VectorHolder { repeated = true; } } - + public VectorHolder(ValueVector vector) { this.length = vector.getValueCapacity(); this.vector = vector; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java index 4ac79fd..dda2dfc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java @@ -20,21 +20,19 @@ package org.apache.drill.exec.store.dfs; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; -import com.google.common.collect.Lists; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Range; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.io.compress.CompressionCodecFactory; public class BasicFormatMatcher extends FormatMatcher{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicFormatMatcher.class); @@ -45,7 +43,7 @@ public class BasicFormatMatcher extends FormatMatcher{ protected final FormatPlugin plugin; protected final boolean compressible; protected final CompressionCodecFactory codecFactory; - + public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, List<Pattern> patterns, List<MagicString> magicStrings) { super(); this.patterns = ImmutableList.copyOf(patterns); @@ -55,7 +53,7 @@ public class BasicFormatMatcher extends FormatMatcher{ this.compressible = false; this.codecFactory = null; } - + public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, List<String> extensions, boolean compressible){ List<Pattern> patterns = Lists.newArrayList(); for (String extension : extensions) { @@ -68,7 +66,7 @@ public class BasicFormatMatcher extends FormatMatcher{ this.compressible = compressible; this.codecFactory = new CompressionCodecFactory(fs.getUnderlying().getConf()); } - + @Override public boolean supportDirectoryReads() { return false; @@ -105,12 +103,12 @@ public class BasicFormatMatcher extends FormatMatcher{ return true; } } - + if(matcher.matches(status)) return true; return false; } - - + + @Override @JsonIgnore public FormatPlugin getFormatPlugin() { @@ -119,20 +117,20 @@ public class BasicFormatMatcher extends FormatMatcher{ private class MagicStringMatcher{ - + private List<RangeMagics> ranges; - + public MagicStringMatcher(List<MagicString> magicStrings){ ranges = Lists.newArrayList(); for(MagicString ms : magicStrings){ ranges.add(new RangeMagics(ms)); } } - + public boolean matches(FileStatus status) throws IOException{ if(ranges.isEmpty()) return false; final Range<Long> fileRange = Range.closedOpen( 0L, status.getLen()); - + try(FSDataInputStream is = fs.open(status.getPath()).getInputStream()){ for(RangeMagics rMagic : ranges){ Range<Long> r = rMagic.range; @@ -141,18 +139,18 @@ public class BasicFormatMatcher extends FormatMatcher{ byte[] bytes = new byte[len]; is.readFully(r.lowerEndpoint(), bytes); for(byte[] magic : rMagic.magics){ - if(Arrays.equals(magic, bytes)) return true; + if(Arrays.equals(magic, bytes)) return true; } - + } } return false; } - + private class RangeMagics{ Range<Long> range; byte[][] magics; - + public RangeMagics(MagicString ms){ this.range = Range.closedOpen( ms.getOffset(), (long) ms.getBytes().length); this.magics = new byte[][]{ms.getBytes()}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java index 4fbe87c..76f6be4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import com.google.common.base.Preconditions; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; @@ -29,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; /** @@ -52,7 +52,7 @@ public class FileSelection { this.files = files; this.selectionRoot = selectionRoot; } - + public FileSelection(List<String> files, boolean dummy){ this.files = files; } @@ -108,7 +108,7 @@ public class FileSelection { } return files; } - + private void init(DrillFileSystem fs) throws IOException { if (files != null && statuses == null) { statuses = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java index 3b74b72..4a3eba9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java @@ -22,25 +22,20 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.ImmutableList; - import net.hydromatic.optiq.Function; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.Table; import org.apache.drill.exec.planner.logical.CreateTableEntry; -import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.rpc.user.DrillUser; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.SchemaFactory; import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; - /** * This is the top level schema that responds to root level path requests. Also supports http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java index b40502f..0d0d46a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java @@ -35,24 +35,24 @@ import com.google.common.collect.Maps; public class FormatCreator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatCreator.class); - - + + static final ConstructorChecker FORMAT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class, FormatPluginConfig.class); static final ConstructorChecker DEFAULT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class); - + static Map<String, FormatPlugin> getFormatPlugins(DrillbitContext context, DrillFileSystem fileSystem, FileSystemConfig storageConfig){ - final DrillConfig config = context.getConfig(); + final DrillConfig config = context.getConfig(); Map<String, FormatPlugin> plugins = Maps.newHashMap(); Collection<Class<? extends FormatPlugin>> pluginClasses = PathScanner.scanForImplementations(FormatPlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); - + if(storageConfig.formats == null || storageConfig.formats.isEmpty()){ - + for(Class<? extends FormatPlugin> pluginClass: pluginClasses){ for(Constructor<?> c : pluginClass.getConstructors()){ try{ - + if(!DEFAULT_BASED.check(c)) continue; FormatPlugin plugin = (FormatPlugin) c.newInstance(null, context, fileSystem, storageConfig); plugins.put(plugin.getName(), plugin); @@ -61,9 +61,9 @@ public class FormatCreator { } } } - + }else{ - + Map<Class<?>, Constructor<?>> constructors = Maps.newHashMap(); for(Class<? extends FormatPlugin> pluginClass: pluginClasses){ for(Constructor<?> c : pluginClass.getConstructors()){ @@ -76,7 +76,7 @@ public class FormatCreator { } } } - + for(Map.Entry<String, FormatPluginConfig> e : storageConfig.formats.entrySet()){ Constructor<?> c = constructors.get(e.getValue().getClass()); if(c == null){ @@ -89,13 +89,13 @@ public class FormatCreator { logger.warn("Failure initializing storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName(), e1); } } - - + + } - + return plugins; } - - + + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index 20ea0a4..27f83f0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -39,7 +39,7 @@ public interface FormatPlugin { public boolean supportsRead(); public boolean supportsWrite(); - + public FormatMatcher getMatcher(); public AbstractWriter getWriter(PhysicalOperator child, String location) throws IOException; @@ -47,13 +47,13 @@ public interface FormatPlugin { public AbstractGroupScan getGroupScan(FileSelection selection) throws IOException; public Set<StoragePluginOptimizerRule> getOptimizerRules(); - + public AbstractGroupScan getGroupScan(FileSelection selection, List<SchemaPath> columns) throws IOException; - + public FormatPluginConfig getConfig(); public StoragePluginConfig getStorageConfig(); public DrillFileSystem getFileSystem(); public DrillbitContext getContext(); public String getName(); - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java index 5cf6ce3..1ea7da7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java @@ -28,18 +28,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class FormatSelection { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatSelection.class); - + private FormatPluginConfig format; private FileSelection selection; public FormatSelection(){} - + @JsonCreator public FormatSelection(@JsonProperty("format") FormatPluginConfig format, @JsonProperty("files") List<String> files){ this.format = format; this.selection = new FileSelection(files, true); } - + public FormatSelection(FormatPluginConfig format, FileSelection selection) { super(); this.format = format; @@ -50,15 +50,15 @@ public class FormatSelection { public FormatPluginConfig getFormat(){ return format; } - + @JsonProperty("files") public List<String> getAsFiles(){ return selection.getAsFiles(); } - + @JsonIgnore public FileSelection getSelection(){ return selection; } - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java index f514388..0d4a0f5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java @@ -19,10 +19,10 @@ package org.apache.drill.exec.store.dfs; public class MagicString { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MagicString.class); - + private long offset; private byte[] bytes; - + public MagicString(long offset, byte[] bytes) { super(); this.offset = offset; @@ -36,7 +36,7 @@ public class MagicString { public byte[] getBytes() { return bytes; } - - - + + + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java index 173dfeb..6a54bb3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.dfs; -import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.drill.common.logical.FormatPluginConfig; import com.fasterxml.jackson.annotation.JsonTypeName; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java index e421eec..ba8f99c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java @@ -22,14 +22,14 @@ public class ReadEntryWithPath { protected String path; - + public ReadEntryWithPath(String path) { super(); this.path = path; } public ReadEntryWithPath(){} - + public String getPath(){ return path; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java index 4b42660..4349fe0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -18,11 +18,8 @@ package org.apache.drill.exec.store.dfs; import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import net.hydromatic.optiq.Table; @@ -30,7 +27,6 @@ import net.hydromatic.optiq.Table; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.cache.DistributedCache.CacheConfig; import org.apache.drill.exec.dotdrill.DotDrillFile; import org.apache.drill.exec.dotdrill.DotDrillType; import org.apache.drill.exec.dotdrill.DotDrillUtil; @@ -41,18 +37,15 @@ import org.apache.drill.exec.planner.logical.DrillViewTable; import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.planner.logical.FileSystemCreateTableEntry; import org.apache.drill.exec.planner.sql.ExpandingConcurrentMap; -import org.apache.drill.exec.planner.sql.ExpandingConcurrentMap.MapValueFactory; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; -import org.apache.drill.exec.store.dfs.shim.DrillInputStream; import org.apache.drill.exec.store.dfs.shim.DrillOutputStream; import org.apache.drill.exec.store.sys.PStore; import org.apache.drill.exec.store.sys.PStoreConfig; import org.apache.drill.exec.store.sys.PStoreProvider; import org.apache.hadoop.fs.Path; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java index 3669044..0c13990 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.store.dfs.easy; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.regex.Matcher; @@ -36,7 +35,6 @@ import org.apache.drill.exec.physical.base.AbstractWriter; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.impl.ScanBatch; import org.apache.drill.exec.physical.impl.WriterRecordBatch; -import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractRecordReader; @@ -51,8 +49,8 @@ import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.CompressionCodecFactory; -import com.google.common.collect.Lists; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements FormatPlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyFormatPlugin.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java index 2bdf1a6..1341fa4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java @@ -219,6 +219,7 @@ public class EasyGroupScan extends AbstractGroupScan{ return newScan; } + @Override @JsonIgnore public boolean canPushdownProjects(List<SchemaPath> columns) { return formatPlugin.supportsPushDown(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java index 5f9226e..e78ba0b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java @@ -21,12 +21,10 @@ import java.io.IOException; import java.util.List; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractSubScan; -import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.NamedFormatPluginConfig; import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java index 170d339..80bcef2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java @@ -20,8 +20,8 @@ package org.apache.drill.exec.store.dfs.easy; public interface FileWork { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileWork.class); - - public String getPath(); + + public String getPath(); public long getStart(); - public long getLength(); + public long getLength(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java index 8af6aaf..a78871e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java @@ -33,11 +33,11 @@ public abstract class DrillFileSystem implements AutoCloseable{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFileSystem.class); public abstract FileSystem getUnderlying(); - + public abstract BlockLocation[] getBlockLocations(FileStatus status, long start, long length) throws IOException; public abstract List<FileStatus> list(boolean recursive, Path... paths) throws IOException; public abstract FileStatus getFileStatus(Path p) throws IOException; public abstract DrillOutputStream create(Path p) throws IOException; public abstract DrillInputStream open(Path p) throws IOException; - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java index 82764a3..8c3b5ae 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java @@ -25,8 +25,8 @@ public abstract class DrillInputStream implements AutoCloseable{ // public abstract AccountingByteBuf readNow(long start, long length) throws IOException; // public abstract void readNow(AccountingByteBuf b, long start, long length) throws IOException; // public abstract AccountingByteBuf readNow() throws IOException; - + public abstract FSDataInputStream getInputStream(); // public abstract CheckedFuture<Long, IOException> readFuture(AccountingByteBuf b, long start, long length) throws IOException; - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java index c2446e9..8e56232 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java @@ -25,5 +25,5 @@ public abstract class DrillOutputStream implements AutoCloseable{ public abstract OutputStream getOuputStream(); // public abstract CheckedFuture<Long, IOException> writeFuture(AccountingByteBuf b); - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java index eaab5ef..a5ad257 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.store.dfs.shim; import java.io.IOException; -import java.net.URI; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.store.dfs.shim.fallback.FallbackFileSystem; @@ -27,10 +26,10 @@ import org.apache.hadoop.fs.FileSystem; public class FileSystemCreator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemCreator.class); - + public static DrillFileSystem getFileSystem(DrillConfig config, Configuration fsConf) throws IOException{ FileSystem fs = FileSystem.get(fsConf); return new FallbackFileSystem(config, fs); } - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java index 0c18e71..3d1b9f2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java @@ -62,7 +62,7 @@ public class FallbackFileSystem extends DrillFileSystem { } } - + private void addRecursiveStatus(FileStatus parent, List<FileStatus> listToFill) throws IOException { if (parent.isDir()) { Path pattern = new Path(parent.getPath(), "*"); @@ -107,7 +107,7 @@ public class FallbackFileSystem extends DrillFileSystem { private class Out extends DrillOutputStream { private final FSDataOutputStream out; - + public Out(FSDataOutputStream out) { super(); this.out = out; @@ -128,7 +128,7 @@ public class FallbackFileSystem extends DrillFileSystem { private class In extends DrillInputStream { private final FSDataInputStream in; - + public In(FSDataInputStream in) { super(); this.in = in; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java index 122e60d..cc8386d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java @@ -28,7 +28,6 @@ import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; import org.apache.drill.exec.physical.base.SubScan; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.RecordReader; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java index 4adac20..8cd7cf2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java @@ -21,12 +21,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.Lists; - -import com.google.common.collect.Maps; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; @@ -42,10 +37,11 @@ import org.apache.drill.exec.store.dfs.easy.EasyWriter; import org.apache.drill.exec.store.dfs.easy.FileWork; import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; import org.apache.drill.exec.store.easy.json.JSONFormatPlugin.JSONFormatConfig; +import org.apache.hadoop.fs.FileSystem; import com.fasterxml.jackson.annotation.JsonTypeName; -import org.apache.drill.exec.store.text.DrillTextRecordWriter; -import org.apache.hadoop.fs.FileSystem; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java index 2bd9df5..c2218b5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java @@ -21,30 +21,25 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; -import com.fasterxml.jackson.core.JsonParseException; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.vector.BaseValueVector; -import org.apache.drill.exec.vector.complex.fn.JsonReader; import org.apache.drill.exec.vector.complex.fn.JsonReaderWithState; import org.apache.drill.exec.vector.complex.fn.JsonRecordSplitter; import org.apache.drill.exec.vector.complex.fn.UTF8JsonRecordSplitter; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter; -import org.apache.drill.exec.vector.complex.writer.FieldWriter; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import com.google.common.collect.Lists; +import com.fasterxml.jackson.core.JsonParseException; public class JSONRecordReader2 extends AbstractRecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader2.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java index da9f48b..76c4ace 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonRecordWriter.java @@ -17,8 +17,10 @@ */ package org.apache.drill.exec.store.easy.json; -import com.fasterxml.jackson.core.JsonFactory; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import java.util.Map; + import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.store.EventBasedRecordWriter; import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter; @@ -30,9 +32,8 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import java.io.IOException; -import java.util.List; -import java.util.Map; +import com.fasterxml.jackson.core.JsonFactory; +import com.google.common.collect.Lists; public class JsonRecordWriter extends JSONOutputRecordWriter implements RecordWriter { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java index 6e1aa0a..ff70ccd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java @@ -17,9 +17,10 @@ */ package org.apache.drill.exec.store.easy.text; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.collect.Maps; -import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; @@ -45,10 +46,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileSplit; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; public class TextFormatPlugin extends EasyFormatPlugin<TextFormatPlugin.TextFormatConfig> { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java index 3981ee1..ab94727 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java @@ -26,7 +26,7 @@ public class InfoSchemaDrillTable extends DrillTable{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaDrillTable.class); private final SelectedTable table; - + public InfoSchemaDrillTable(InfoSchemaStoragePlugin plugin, String storageEngineName, SelectedTable selection, StoragePluginConfig storageEngineConfig) { super(storageEngineName, plugin, selection); this.table = selection; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java index 0997152..f73bc05 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.ImmutableList; - import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.Table; @@ -35,6 +33,7 @@ import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.AbstractStoragePlugin; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java index 00ab0a0..5ef8ad2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java @@ -17,12 +17,13 @@ */ package org.apache.drill.exec.store.ischema; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import java.util.List; + import net.hydromatic.optiq.Schema.TableType; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.Table; import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl; + import org.apache.drill.exec.planner.logical.DrillViewInfoProvider; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.RecordReader; @@ -30,7 +31,8 @@ import org.apache.drill.exec.store.pojo.PojoRecordReader; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeField; -import java.util.List; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; /** Generates records for POJO RecordReader by scanning the given schema */ public abstract class RecordGenerator { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java index 131e795..5f56221 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java @@ -34,19 +34,19 @@ public enum SelectedTable{ VIEWS(new Views()), COLUMNS(new Columns()), TABLES(new Tables()); - + private final InfoSchemaTable tableDef; - + private SelectedTable(InfoSchemaTable tableDef) { this.tableDef = tableDef; } - + public RecordReader getRecordReader(SchemaPlus rootSchema) { RecordGenerator recordGenerator = tableDef.getRecordGenerator(); recordGenerator.scanSchema(rootSchema); return recordGenerator.getRecordReader(); } - + public RelDataType getRowType(RelDataTypeFactory typeFactory) { return tableDef.getRowType(typeFactory); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java index 80515fa..21923d8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java @@ -22,8 +22,6 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; -import javax.validation.constraints.Size; - import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java index 0714ab8..66851a9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java @@ -17,9 +17,9 @@ */ package org.apache.drill.exec.store.mock; -import com.google.common.collect.Lists; +import java.util.Map; + import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.exception.SchemaChangeException; @@ -32,15 +32,11 @@ import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockColumn; import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockScanEntry; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.ValueVector; -import java.util.List; -import java.util.Map; - public class MockRecordReader extends AbstractRecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockRecordReader.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java index 1adbacc..51b2208 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java @@ -27,7 +27,6 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.rpc.user.DrillUser; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java index 2567c89..a461e5a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java @@ -17,19 +17,14 @@ */ package org.apache.drill.exec.store.parquet; -import java.io.EOFException; +import io.netty.buffer.ByteBuf; + import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import parquet.bytes.BytesInput; import parquet.format.PageHeader; @@ -38,16 +33,16 @@ import parquet.hadoop.util.CompatibilityUtil; public class ColumnDataReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.class); - + private final long endPosition; public final FSDataInputStream input; - + public ColumnDataReader(FSDataInputStream input, long start, long length) throws IOException{ this.input = input; this.input.seek(start); this.endPosition = start + length; } - + public PageHeader readPageHeader() throws IOException{ return Util.readPageHeader(input); } @@ -82,11 +77,11 @@ public class ColumnDataReader { public boolean hasRemainder() throws IOException{ return input.getPos() < endPosition; } - + public class HadoopBytesInput extends BytesInput{ private final byte[] pageBytes; - + public HadoopBytesInput(byte[] pageBytes) { super(); this.pageBytes = pageBytes; @@ -106,7 +101,7 @@ public class ColumnDataReader { public void writeAllTo(OutputStream out) throws IOException { out.write(pageBytes); } - + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java index 1a49dcd..45a1dc6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java @@ -19,14 +19,16 @@ package org.apache.drill.exec.store.parquet; import io.netty.buffer.ByteBuf; -import org.apache.drill.exec.ops.OperatorContext; -import parquet.bytes.ByteBufferAllocator; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.apache.drill.exec.ops.OperatorContext; + +import parquet.bytes.ByteBufferAllocator; + public class ParquetDirectByteBufferAllocator implements ByteBufferAllocator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetDirectByteBufferAllocator.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java index d3200fe..74a90c0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.parquet; -import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.drill.common.logical.FormatPluginConfig; import com.fasterxml.jackson.annotation.JsonTypeName; @@ -34,5 +33,5 @@ public class ParquetFormatConfig implements FormatPluginConfig{ public boolean equals(Object obj) { return obj instanceof ParquetFormatConfig; } - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java index eb07d79..109033a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java @@ -29,7 +29,6 @@ import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.base.AbstractWriter; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.impl.WriterRecordBatch; @@ -77,7 +76,7 @@ public class ParquetFormatPlugin implements FormatPlugin{ public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){ this(name, context, fs, storageConfig, new ParquetFormatConfig()); } - + public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){ this.context = context; this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf()); @@ -109,7 +108,7 @@ public class ParquetFormatPlugin implements FormatPlugin{ public boolean supportsRead() { return true; } - + @Override public Set<StoragePluginOptimizerRule> getOptimizerRules() { return ImmutableSet.of(); @@ -170,7 +169,7 @@ public class ParquetFormatPlugin implements FormatPlugin{ public String getName(){ return name; } - + @Override public boolean supportsWrite() { return false; @@ -184,9 +183,9 @@ public class ParquetFormatPlugin implements FormatPlugin{ } private static class ParquetFormatMatcher extends BasicFormatMatcher{ - + private final DrillFileSystem fs; - + public ParquetFormatMatcher(ParquetFormatPlugin plugin, DrillFileSystem fs) { super(plugin, fs, // Lists.newArrayList( // @@ -195,12 +194,12 @@ public class ParquetFormatPlugin implements FormatPlugin{ // ), Lists.newArrayList(new MagicString(0, ParquetFileWriter.MAGIC)) - + ); this.fs = fs; - + } - + @Override public boolean supportDirectoryReads() { return true; @@ -208,7 +207,7 @@ public class ParquetFormatPlugin implements FormatPlugin{ @Override public FormatSelection isReadable(FileSelection selection) throws IOException { - // TODO: we only check the first file for directory reading. This is because + // TODO: we only check the first file for directory reading. This is because if(selection.containsDirectories(fs)){ if(isDirReadable(selection.getFirstPath(fs))){ return new FormatSelection(plugin.getConfig(), selection); @@ -216,7 +215,7 @@ public class ParquetFormatPlugin implements FormatPlugin{ } return super.isReadable(selection); } - + boolean isDirReadable(FileStatus dir) { Path p = new Path(dir.getPath(), ParquetFileWriter.PARQUET_METADATA_FILE); try { @@ -237,9 +236,9 @@ public class ParquetFormatPlugin implements FormatPlugin{ return false; } } - - - + + + } - + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java index bc8495c..a768fc9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java @@ -390,6 +390,7 @@ public class ParquetGroupScan extends AbstractGroupScan { return newScan; } + @Override @JsonIgnore public boolean canPushdownProjects(List<SchemaPath> columns) { return true; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java index 2c5f232..5a6ba80 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java @@ -17,7 +17,14 @@ */ package org.apache.drill.exec.store.parquet; -import com.google.common.collect.Lists; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.ExecConstants; @@ -33,6 +40,7 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; + import parquet.column.ParquetProperties.WriterVersion; import parquet.column.impl.ColumnWriteStoreImpl; import parquet.column.page.PageWriteStore; @@ -51,14 +59,7 @@ import parquet.schema.PrimitiveType.PrimitiveTypeName; import parquet.schema.Type; import parquet.schema.Type.Repetition; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.lang.Math.max; -import static java.lang.Math.min; -import static java.lang.String.format; +import com.google.common.collect.Lists; public class ParquetRecordWriter extends ParquetOutputRecordWriter { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordWriter.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index f77ac98..8aebab9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -34,10 +34,6 @@ import org.apache.drill.exec.physical.impl.ScanBatch; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.store.RecordReader; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader; import org.apache.drill.exec.store.parquet2.DrillParquetReader; import org.apache.hadoop.conf.Configuration; @@ -50,6 +46,9 @@ import parquet.hadoop.metadata.ParquetMetadata; import parquet.schema.MessageType; import parquet.schema.Type; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan>{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetScanBatchCreator.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriterBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriterBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriterBatchCreator.java index 50ab94c..10dd26d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriterBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriterBatchCreator.java @@ -17,13 +17,13 @@ */ package org.apache.drill.exec.store.parquet; +import java.util.List; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.BatchCreator; import org.apache.drill.exec.record.RecordBatch; -import java.util.List; - public class ParquetWriterBatchCreator implements BatchCreator<ParquetWriter>{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetWriterBatchCreator.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java index 43e270a..9aabc9c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java @@ -18,9 +18,11 @@ package org.apache.drill.exec.store.parquet.columnreaders; import io.netty.buffer.ByteBuf; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.BaseDataValueVector; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; @@ -30,7 +32,7 @@ final class BitReader extends ColumnReader { private byte currentByte; private byte nextByte; private ByteBuf bytebuf; - + BitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException { super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java index 272a5c3..b629dda 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java @@ -18,22 +18,24 @@ package org.apache.drill.exec.store.parquet.columnreaders; import io.netty.buffer.ByteBuf; + +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; import parquet.schema.PrimitiveType; import parquet.schema.PrimitiveType.PrimitiveTypeName; -import java.io.IOException; - public abstract class ColumnReader<V extends ValueVector> { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnReader.class); final ParquetRecordReader parentReader; - + // Value Vector for this column final V valueVec; @@ -57,10 +59,10 @@ public abstract class ColumnReader<V extends ValueVector> { // counter for the total number of values read from one or more pages // when a batch is filled all of these values should be the same for all of the columns int totalValuesRead; - + // counter for the values that have been read in this pass (a single call to the next() method) int valuesReadInCurrentPass; - + // length of single data value in bits, if the length is fixed int dataTypeLengthInBits; int bytesReadInCurrentPass;
