[
https://issues.apache.org/jira/browse/DRILL-7385?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16948661#comment-16948661
]
ASF GitHub Bot commented on DRILL-7385:
---------------------------------------
cgivre commented on pull request #1862: DRILL-7385: Convert PCAP Format Plugin
to EVF
URL: https://github.com/apache/drill/pull/1862#discussion_r333560624
##########
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapFormatPlugin.java
##########
@@ -17,112 +17,72 @@
*/
package org.apache.drill.exec.store.pcap;
-import org.apache.drill.exec.planner.common.DrillStatsTable;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import
org.apache.drill.exec.physical.impl.scan.file.FileScanFramework.FileSchemaNegotiator;
+import
org.apache.drill.exec.physical.impl.scan.file.FileScanFramework.FileReaderFactory;
+import
org.apache.drill.exec.physical.impl.scan.file.FileScanFramework.FileScanBuilder;
+import org.apache.drill.exec.physical.impl.scan.framework.ManagedReader;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.store.dfs.easy.EasySubScan;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.RecordWriter;
-import org.apache.drill.exec.store.SchemaConfig;
-import org.apache.drill.exec.store.dfs.BasicFormatMatcher;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.dfs.FileSelection;
-import org.apache.drill.exec.store.dfs.FileSystemPlugin;
-import org.apache.drill.exec.store.dfs.FormatMatcher;
-import org.apache.drill.exec.store.dfs.FormatSelection;
-import org.apache.drill.exec.store.dfs.MagicString;
import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
-import org.apache.drill.exec.store.dfs.easy.EasyWriter;
-import org.apache.drill.exec.store.dfs.easy.FileWork;
import org.apache.hadoop.conf.Configuration;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.regex.Pattern;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.drill.exec.store.pcap.PcapBatchReader.PcapReaderConfig;
public class PcapFormatPlugin extends EasyFormatPlugin<PcapFormatConfig> {
+ public static final String PLUGIN_NAME = "pcap";
- private final PcapFormatMatcher matcher;
-
- public PcapFormatPlugin(String name, DrillbitContext context, Configuration
fsConf,
- StoragePluginConfig storagePluginConfig) {
- this(name, context, fsConf, storagePluginConfig, new PcapFormatConfig());
- }
-
- public PcapFormatPlugin(String name, DrillbitContext context, Configuration
fsConf, StoragePluginConfig config, PcapFormatConfig formatPluginConfig) {
- super(name, context, fsConf, config, formatPluginConfig, true, false,
true, false, Lists.newArrayList("pcap"), "pcap");
- this.matcher = new PcapFormatMatcher(this);
- }
-
- @Override
- public boolean supportsPushDown() {
- return true;
- }
-
- @Override
- public RecordReader getRecordReader(FragmentContext context, DrillFileSystem
dfs, FileWork fileWork, List<SchemaPath> columns, String userName) throws
ExecutionSetupException {
- return new PcapRecordReader(fileWork.getPath(), dfs, columns);
- }
-
- @Override
- public RecordWriter getRecordWriter(FragmentContext context, EasyWriter
writer) throws IOException {
- throw new UnsupportedOperationException("unimplemented");
- }
+ private static class PcapReaderFactory extends FileReaderFactory {
+ private final PcapReaderConfig readerConfig;
- @Override
- public int getReaderOperatorType() {
- return UserBitShared.CoreOperatorType.PCAP_SUB_SCAN_VALUE;
- }
+ public PcapReaderFactory(PcapReaderConfig config) {
+ readerConfig = config;
+ }
- @Override
- public int getWriterOperatorType() {
- throw new UnsupportedOperationException();
+ @Override
+ public ManagedReader<? extends FileSchemaNegotiator> newReader() {
+ return new PcapBatchReader(readerConfig);
+ }
}
-
- @Override
- public FormatMatcher getMatcher() {
- return this.matcher;
+ public PcapFormatPlugin(String name, DrillbitContext context,
+ Configuration fsConf, StoragePluginConfig
storageConfig,
+ PcapFormatConfig formatConfig) {
+ super(name, easyConfig(fsConf, formatConfig), context, storageConfig,
formatConfig);
}
- @Override
- public boolean supportsStatistics() {
- return false;
+ private static EasyFormatConfig easyConfig(Configuration fsConf,
PcapFormatConfig pluginConfig) {
+ EasyFormatConfig config = new EasyFormatConfig();
+ config.readable = true;
+ config.writable = false;
+ config.blockSplittable = true;
+ config.compressible = true;
+ config.supportsProjectPushdown = true;
+ config.extensions = pluginConfig.getExtensions();
+ config.fsConf = fsConf;
+ config.defaultName = PLUGIN_NAME;
+ config.readerOperatorType =
UserBitShared.CoreOperatorType.PCAP_SUB_SCAN_VALUE;
+ config.useEnhancedScan = true;
+ return config;
}
@Override
- public DrillStatsTable.TableStatistics readStatistics(FileSystem fs, Path
statsTablePath) throws IOException {
- return null;
+ public ManagedReader<? extends FileSchemaNegotiator> newBatchReader(
+ EasySubScan scan, OptionManager options) throws ExecutionSetupException {
+ return new PcapBatchReader(new PcapReaderConfig(this));
}
@Override
- public void writeStatistics(DrillStatsTable.TableStatistics statistics,
FileSystem fs, Path statsTablePath) throws IOException {
-
+ protected FileScanBuilder frameworkBuilder(OptionManager options,
EasySubScan scan) throws ExecutionSetupException {
+ FileScanBuilder builder = new FileScanBuilder();
+ builder.setReaderFactory(new PcapReaderFactory(
+ new PcapReaderConfig(this))
Review comment:
Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Convert PCAP Format Plugin to EVF
> ---------------------------------
>
> Key: DRILL-7385
> URL: https://issues.apache.org/jira/browse/DRILL-7385
> Project: Apache Drill
> Issue Type: Improvement
> Affects Versions: 1.16.0
> Reporter: Charles Givre
> Assignee: Charles Givre
> Priority: Major
> Fix For: 1.17.0
>
>
> The PCAP format plugin has the possibility of extracting considerably more
> data out of PCAP files. To facilitate this, this PR updates the plugin to
> use the new Enhanced Vector Framework. No changes in functionality should
> occur.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
