Repository: drill Updated Branches: refs/heads/master cc97cd471 -> 168fa904b
DRILL-1833: Avoid storing view names in PStore cache ...always rely on view files in schema location for listing views. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0b6cddfa Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0b6cddfa Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0b6cddfa Branch: refs/heads/master Commit: 0b6cddfa4d8f9558f6386e7340429df4e8ec5f88 Parents: cc97cd4 Author: vkorukanti <[email protected]> Authored: Mon Mar 9 15:50:11 2015 -0700 Committer: vkorukanti <[email protected]> Committed: Thu Apr 2 00:26:39 2015 -0700 ---------------------------------------------------------------------- .../drill/exec/dotdrill/DotDrillFile.java | 9 +++ .../drill/exec/dotdrill/DotDrillType.java | 21 ++++++ .../drill/exec/dotdrill/DotDrillUtil.java | 12 ++-- .../drill/exec/store/dfs/FileSystemPlugin.java | 4 +- .../exec/store/dfs/WorkspaceSchemaFactory.java | 76 +++++--------------- 5 files changed, 54 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/0b6cddfa/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java index 6a5934b..f9a8ff5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java @@ -50,6 +50,15 @@ public class DotDrillFile { return type; } + /** + * Return base file name without the parent directory and extensions. + * @return Base file name. + */ + public String getBaseName() { + final String fileName = status.getPath().getName(); + return fileName.substring(0, fileName.lastIndexOf(type.getEnding())); + } + public View getView(DrillConfig config) throws Exception{ Preconditions.checkArgument(type == DotDrillType.VIEW); try(InputStream is = fs.open(status.getPath())){ http://git-wip-us.apache.org/repos/asf/drill/blob/0b6cddfa/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillType.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillType.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillType.java index 74b71e7..3915359 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillType.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillType.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.dotdrill; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; public enum DotDrillType { VIEW @@ -35,6 +36,26 @@ public enum DotDrillType { return status.getPath().getName().endsWith(ending); } + /** + * For a given parent directory and base file name return complete path including file type specific extensions. + * + * @param parentDir Directory where the DotDrillFile is stored. + * @param name Base file name of the DotDrillFile. + * @return Path including the extensions that can be used to read/write in filesystem. + */ + public Path getPath(String parentDir, String name) { + return new Path(parentDir, name + ending); + } + + /** + * Return extension string of file type represented by this object. + * + * @return File extension. + */ + public String getEnding() { + return ending; + } + public static final String DOT_DRILL_GLOB; static{ http://git-wip-us.apache.org/repos/asf/drill/blob/0b6cddfa/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillUtil.java index cef4359..19c9928 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillUtil.java @@ -53,11 +53,11 @@ public class DotDrillUtil { return getDrillFiles(fs, fs.globStatus(new Path(root, "*.drill")), types); } - public static List<DotDrillFile> getDotDrills(DrillFileSystem fs, Path root, String name, DotDrillType... types) throws IOException{ - if(!name.endsWith(".drill")) { - name = name + DotDrillType.DOT_DRILL_GLOB; - } - - return getDrillFiles(fs, fs.globStatus(new Path(root, name)), types); + public static List<DotDrillFile> getDotDrills(DrillFileSystem fs, Path root, String name, DotDrillType... types) throws IOException{ + if(!name.endsWith(".drill")) { + name = name + DotDrillType.DOT_DRILL_GLOB; } + + return getDrillFiles(fs, fs.globStatus(new Path(root, name)), types); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/0b6cddfa/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 7a1f61e..c5ca41b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -82,13 +82,13 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ List<WorkspaceSchemaFactory> factories = Lists.newArrayList(); if (!noWorkspace) { for (Map.Entry<String, WorkspaceConfig> space : config.workspaces.entrySet()) { - factories.add(new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, space.getKey(), name, fs, space.getValue(), matchers)); + factories.add(new WorkspaceSchemaFactory(context.getConfig(), this, space.getKey(), name, fs, space.getValue(), matchers)); } } // if the "default" workspace is not given add one. if (noWorkspace || !config.workspaces.containsKey("default")) { - factories.add(new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, "default", name, fs, WorkspaceConfig.DEFAULT, matchers)); + factories.add(new WorkspaceSchemaFactory(context.getConfig(), this, "default", name, fs, WorkspaceConfig.DEFAULT, matchers)); } this.schemaFactory = new FileSystemSchemaFactory(name, factories); http://git-wip-us.apache.org/repos/asf/drill/blob/0b6cddfa/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java index 7c8d9b3..aeff09b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -20,7 +20,6 @@ package org.apache.drill.exec.store.dfs; import java.io.IOException; import java.io.OutputStream; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -43,9 +42,6 @@ import org.apache.drill.exec.planner.logical.FileSystemCreateTableEntry; import org.apache.drill.exec.planner.sql.ExpandingConcurrentMap; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.store.AbstractSchema; -import org.apache.drill.exec.store.sys.PStore; -import org.apache.drill.exec.store.sys.PStoreConfig; -import org.apache.drill.exec.store.sys.PStoreProvider; import org.apache.hadoop.fs.Path; import com.fasterxml.jackson.databind.ObjectMapper; @@ -65,12 +61,10 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa private final String storageEngineName; private final String schemaName; private final FileSystemPlugin plugin; - - private final PStore<String> knownViews; private final ObjectMapper mapper; - public WorkspaceSchemaFactory(DrillConfig drillConfig, PStoreProvider provider, FileSystemPlugin plugin, String schemaName, String storageEngineName, - DrillFileSystem fileSystem, WorkspaceConfig config, + public WorkspaceSchemaFactory(DrillConfig drillConfig, FileSystemPlugin plugin, String schemaName, + String storageEngineName, DrillFileSystem fileSystem, WorkspaceConfig config, List<FormatMatcher> formatMatchers) throws ExecutionSetupException, IOException { this.fs = fileSystem; this.plugin = plugin; @@ -82,18 +76,6 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa this.storageEngineName = storageEngineName; this.schemaName = schemaName; - // setup cache - if (storageEngineName == null) { - this.knownViews = null; - } else { - this.knownViews = provider.getStore(PStoreConfig // - .newJacksonBuilder(drillConfig.getMapper(), String.class) // - .persist() // - .name(Joiner.on('.').join("storage.views", storageEngineName, schemaName)) // - .build()); - } - - for (FormatMatcher m : formatMatchers) { if (m.supportDirectoryReads()) { dirMatchers.add(m); @@ -117,7 +99,7 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa } private Path getViewPath(String name) { - return new Path(config.getLocation() + '/' + name + ".view.drill"); + return DotDrillType.VIEW.getPath(config.getLocation(), name); } public WorkspaceSchema createSchema(List<String> parentSchemaPath, UserSession session) { @@ -174,9 +156,6 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa try (OutputStream stream = fs.create(viewPath)) { mapper.writeValue(stream, view); } - if (knownViews != null) { - knownViews.put(view.getName(), viewPath.toString()); - } return replaced; } @@ -187,12 +166,9 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa public void dropView(String viewName) throws IOException { fs.delete(getViewPath(viewName), false); - if (knownViews != null) { - knownViews.delete(viewName); - } } - private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(WorkspaceSchemaFactory.this); + private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<>(WorkspaceSchemaFactory.this); private UserSession session; @@ -203,33 +179,20 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa private Set<String> getViews() { Set<String> viewSet = Sets.newHashSet(); - if(knownViews != null) { - String viewName; - for(Map.Entry<String, String> e : knownViews) { - viewName = e.getKey(); - if (hasView(viewName)) { - viewSet.add(viewName); - } else if (knownViews != null) { - knownViews.delete(viewName); - } - } - } - return viewSet; - } - - /** - * Checks whether underlying filesystem has the view. - * @param viewName view name - * @return true if storage has the view, false otherwise. - */ - public boolean hasView(String viewName) { - List<DotDrillFile> files = null; + // Look for files with ".view.drill" extension. + List<DotDrillFile> files; try { - files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), viewName, DotDrillType.VIEW); + files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), DotDrillType.VIEW); + for(DotDrillFile f : files) { + viewSet.add(f.getBaseName()); + } + } catch (UnsupportedOperationException e) { + logger.debug("The filesystem for this workspace does not support this operation.", e); } catch (Exception e) { - logger.warn("Failure while trying to check view[{}].", viewName, e); + logger.warn("Failure while trying to list .view.drill files in workspace [{}]", getFullSchemaName(), e); } - return files!=null && files.size()>0; + + return viewSet; } @Override @@ -249,9 +212,6 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa return tables.get(name); } - // then check known views. -// String path = knownViews.get(name); - // then look for files that start with this name and end in .drill. List<DotDrillFile> files; try { @@ -265,11 +225,10 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa } catch (UnsupportedOperationException e) { logger.debug("The filesystem for this workspace does not support this operation.", e); } catch (Exception e) { - logger.warn("Failure while trying to load .drill file.", e); + logger.warn("Failure while trying to load {}.view.drill file in workspace [{}]", name, getFullSchemaName(), e); } return tables.get(name); - } @Override @@ -285,7 +244,6 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa return config.getLocation(); } - @Override public CreateTableEntry createNewTable(String tableName) { String storage = session.getOptions().getOption(ExecConstants.OUTPUT_FORMAT_OPTION).string_val; @@ -306,7 +264,5 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa public String getTypeName() { return FileSystemConfig.NAME; } - } - }
