Repository: drill Updated Branches: refs/heads/master 7e564996f -> b2e79aac8
DRILL-4514: Add describe schema <schema_name> command close apache/drill#436. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b2e79aac Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b2e79aac Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b2e79aac Branch: refs/heads/master Commit: b2e79aac8f88bc4a1d2b9da2179d70540c6e49d4 Parents: 7e56499 Author: Arina Ielchiieva <[email protected]> Authored: Wed Mar 16 16:33:41 2016 +0000 Committer: Jinfeng Ni <[email protected]> Committed: Tue Jul 12 17:11:22 2016 -0700 ---------------------------------------------------------------------- exec/java-exec/src/main/codegen/data/Parser.tdd | 4 +- .../src/main/codegen/includes/parserImpls.ftl | 16 +++ .../sql/handlers/DescribeSchemaHandler.java | 128 +++++++++++++++++++ .../sql/parser/CompoundIdentifierConverter.java | 1 + .../planner/sql/parser/SqlDescribeSchema.java | 81 ++++++++++++ .../apache/drill/exec/sql/TestInfoSchema.java | 67 ++++++++++ 6 files changed, 296 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/main/codegen/data/Parser.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd index 3e5f336..84b3dce 100644 --- a/exec/java-exec/src/main/codegen/data/Parser.tdd +++ b/exec/java-exec/src/main/codegen/data/Parser.tdd @@ -36,13 +36,15 @@ "USE", "FILES", "REFRESH", - "METADATA" + "METADATA", + "DATABASE" ] # List of methods for parsing custom SQL statements. statementParserMethods: [ "SqlShowTables()", "SqlShowSchemas()", + "SqlDescribeSchema()" "SqlDescribeTable()", "SqlUseSchema()", "SqlCreateOrReplaceView()", http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/main/codegen/includes/parserImpls.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl index 9dce04a..3a9424b 100644 --- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl +++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl @@ -278,3 +278,19 @@ SqlNode SqlRefreshMetadata() : } } +/** +* Parses statement +* DESCRIBE { SCHEMA | DATABASE } name +*/ +SqlNode SqlDescribeSchema() : +{ + SqlParserPos pos; + SqlIdentifier schema; +} +{ + <DESCRIBE> { pos = getPos(); } + (<SCHEMA> | <DATABASE>) { schema = CompoundIdentifier(); } + { + return new SqlDescribeSchema(pos, schema); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeSchemaHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeSchemaHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeSchemaHandler.java new file mode 100644 index 0000000..869829a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeSchemaHandler.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.handlers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.SerializableString; +import com.fasterxml.jackson.core.io.CharacterEscapes; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.sql.DirectPlan; +import org.apache.drill.exec.planner.sql.SchemaUtilites; +import org.apache.drill.exec.planner.sql.parser.SqlDescribeSchema; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.dfs.FileSystemSchemaFactory; +import org.apache.drill.exec.store.dfs.WorkspaceConfig; + +import java.util.List; +import java.util.Map; + +import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT; + +public class DescribeSchemaHandler extends DefaultSqlHandler { + + public DescribeSchemaHandler(SqlHandlerConfig config) { + super(config); + } + + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DescribeSchemaHandler.class); + private static final ObjectMapper mapper = new ObjectMapper(new ObjectMapper().getFactory().setCharacterEscapes(new CharacterEscapes() { + @Override + public int[] getEscapeCodesForAscii() { + // add standard set of escaping characters + int[] esc = CharacterEscapes.standardAsciiEscapesForJSON(); + // don't escape backslash (not to corrupt windows path) + esc['\\'] = CharacterEscapes.ESCAPE_NONE; + return esc; + } + + @Override + public SerializableString getEscapeSequence(int i) { + // no further escaping (beyond ASCII chars) needed + return null; + } + })).enable(INDENT_OUTPUT); + + + @Override + public PhysicalPlan getPlan(SqlNode sqlNode) { + SqlIdentifier schema = ((SqlDescribeSchema) sqlNode).getSchema(); + SchemaPlus drillSchema = SchemaUtilites.findSchema(config.getConverter().getDefaultSchema(), schema.names); + + if (drillSchema != null) { + StoragePlugin storagePlugin; + try { + storagePlugin = context.getStorage().getPlugin(schema.names.get(0)); + } catch (ExecutionSetupException e) { + throw new DrillRuntimeException("Failure while retrieving storage plugin", e); + } + String properties; + try { + final Map configMap = mapper.convertValue(storagePlugin.getConfig(), Map.class); + if (storagePlugin instanceof FileSystemPlugin) { + transformWorkspaces(schema.names, configMap); + } + properties = mapper.writeValueAsString(configMap); + } catch (JsonProcessingException e) { + throw new DrillRuntimeException("Error while trying to convert storage config to json string", e); + } + return DirectPlan.createDirectPlan(context, new DescribeSchemaResult(Joiner.on(".").join(schema.names), properties)); + } + + throw UserException.validationError() + .message(String.format("Invalid schema name [%s]", Joiner.on(".").join(schema.names))) + .build(logger); + } + + /** + * If storage plugin has several workspaces, picks appropriate one and removes the others. + */ + private void transformWorkspaces(List<String> names, Map configMap) { + Object workspaces = configMap.remove("workspaces"); + if (workspaces != null) { + Map map = (Map) workspaces; + String key = names.size() > 1 ? names.get(1) : FileSystemSchemaFactory.DEFAULT_WS_NAME; + Object workspace = map.get(key); + if (workspace != null) { + Map workspaceMap = (Map) map.get(key); + configMap.putAll(workspaceMap); + } else if (FileSystemSchemaFactory.DEFAULT_WS_NAME.equals(key)) { + configMap.putAll(mapper.convertValue(WorkspaceConfig.DEFAULT, Map.class)); + } + } + } + + public static class DescribeSchemaResult { + public String schema; + public String properties; + + public DescribeSchemaResult(String schema, String properties) { + this.schema = schema; + this.properties = properties; + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java index 61a4c9f..a125628 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java @@ -173,6 +173,7 @@ public class CompoundIdentifierConverter extends SqlShuttle { rules.put(SqlDropTable.class, R(D)); rules.put(SqlRefreshMetadata.class, R(D)); rules.put(SqlSetOption.class, R(D, D, D)); + rules.put(SqlDescribeSchema.class, R(D)); REWRITE_RULES = ImmutableMap.copyOf(rules); } http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeSchema.java new file mode 100644 index 0000000..7ea6940 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeSchema.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; +import org.apache.drill.exec.planner.sql.handlers.DescribeSchemaHandler; +import org.apache.drill.exec.planner.sql.handlers.SqlHandlerConfig; + +import java.util.Collections; +import java.util.List; + +/** + * Sql parse tree node to represent statement: + * DESCRIBE {SCHEMA | DATABASE} schema_name + */ +public class SqlDescribeSchema extends DrillSqlCall { + + private final SqlIdentifier schema; + + public static final SqlSpecialOperator OPERATOR = + new SqlSpecialOperator("DESCRIBE_SCHEMA", SqlKind.OTHER) { + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) { + return new SqlDescribeSchema(pos, (SqlIdentifier) operands[0]); + } + }; + + public SqlDescribeSchema(SqlParserPos pos, SqlIdentifier schema) { + super(pos); + this.schema = schema; + } + + @Override + public SqlOperator getOperator() { + return OPERATOR; + } + + @Override + public List<SqlNode> getOperandList() { + return Collections.singletonList((SqlNode) schema); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword("DESCRIBE"); + writer.keyword("SCHEMA"); + schema.unparse(writer, leftPrec, rightPrec); + } + + @Override + public AbstractSqlHandler getSqlHandler(SqlHandlerConfig config) { + return new DescribeSchemaHandler(config); + } + + public SqlIdentifier getSchema() { return schema; } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/b2e79aac/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java index 15d98d8..541023a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java @@ -17,16 +17,28 @@ */ package org.apache.drill.exec.sql; +import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT; import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_CONNECT; import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_DESCRIPTION; import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.drill.BaseTestQuery; import org.apache.drill.TestBuilder; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.vector.NullableVarCharVector; import org.junit.Test; import java.util.List; +import java.util.Map; /** * Contains tests for @@ -36,6 +48,9 @@ import java.util.List; * -- SHOW FILES */ public class TestInfoSchema extends BaseTestQuery { + + private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT); + @Test public void selectFromAllTables() throws Exception{ test("select * from INFORMATION_SCHEMA.SCHEMATA"); @@ -351,4 +366,56 @@ public class TestInfoSchema extends BaseTestQuery { test("USE dfs_test.`default`"); test("SHOW FILES FROM `/tmp`"); } + + @Test + public void describeSchemaSyntax() throws Exception { + test("describe schema dfs_test"); + test("describe schema dfs_test.`default`"); + test("describe database dfs_test.`default`"); + } + + @Test + public void describeSchemaOutput() throws Exception { + final List<QueryDataBatch> result = testSqlWithResults("describe schema dfs_test.tmp"); + assertTrue(result.size() == 1); + final QueryDataBatch batch = result.get(0); + final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator()); + loader.load(batch.getHeader().getDef(), batch.getData()); + + // check schema column value + final VectorWrapper schemaValueVector = loader.getValueAccessorById( + NullableVarCharVector.class, + loader.getValueVectorId(SchemaPath.getCompoundPath("schema")).getFieldIds()); + String schema = schemaValueVector.getValueVector().getAccessor().getObject(0).toString(); + assertEquals("dfs_test.tmp", schema); + + // check properties column value + final VectorWrapper propertiesValueVector = loader.getValueAccessorById( + NullableVarCharVector.class, + loader.getValueVectorId(SchemaPath.getCompoundPath("properties")).getFieldIds()); + String properties = propertiesValueVector.getValueVector().getAccessor().getObject(0).toString(); + final Map configMap = mapper.readValue(properties, Map.class); + + // check some stable properties existence + assertTrue(configMap.containsKey("connection")); + assertTrue(configMap.containsKey("config")); + assertTrue(configMap.containsKey("formats")); + assertFalse(configMap.containsKey("workspaces")); + + // check some stable properties values + assertEquals("file", configMap.get("type")); + + final FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs_test").getConfig(); + final String tmpSchemaLocation = testConfig.workspaces.get("tmp").getLocation(); + assertEquals(tmpSchemaLocation, configMap.get("location")); + + batch.release(); + loader.clear(); + } + + @Test + public void describeSchemaInvalid() throws Exception { + errorMsgTestHelper("describe schema invalid.schema", "Invalid schema name [invalid.schema]"); + } + }
