This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5f5b3ca6a8af175e603877689c4b15a3126fe016 Author: Arina Ielchiieva <[email protected]> AuthorDate: Thu Mar 28 18:04:31 2019 +0200 DRILL-7138: Implement command to describe schema for table closes #1719 --- .../src/main/codegen/includes/parserImpls.ftl | 39 +++++-- .../drill/exec/planner/sql/DrillSqlWorker.java | 7 ++ .../exec/planner/sql/handlers/SchemaHandler.java | 95 ++++++++++++++++- .../sql/parser/CompoundIdentifierConverter.java | 1 + .../drill/exec/planner/sql/parser/SqlSchema.java | 64 +++++++++++- .../record/metadata/schema/PathSchemaProvider.java | 4 +- .../java/org/apache/drill/TestSchemaCommands.java | 112 +++++++++++++++++++++ 7 files changed, 311 insertions(+), 11 deletions(-) diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl index 015ba94..97b5052 100644 --- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl +++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl @@ -179,7 +179,7 @@ SqlNodeList ParseRequiredFieldList(String relType) : } /** -* Rarses CREATE [OR REPLACE] command for VIEW, TABLE or SCHEMA. +* Parses CREATE [OR REPLACE] command for VIEW, TABLE or SCHEMA. */ SqlNode SqlCreateOrReplace() : { @@ -374,7 +374,7 @@ void addProperty(SqlNodeList properties) : <SCH> TOKEN : { < LOAD: "LOAD" > { popState(); } | < NUM: <DIGIT> (" " | "\t" | "\n" | "\r")* > - // once schema is found, swich back to initial lexical state + // once schema is found, switch back to initial lexical state // must be enclosed in the parentheses // inside may have left parenthesis only if number precededs (covers cases with varchar(10)), // if left parenthesis is present in column name, it must be escaped with backslash @@ -494,18 +494,43 @@ SqlNode SqlRefreshMetadata() : /** * Parses statement * DESCRIBE { SCHEMA | DATABASE } name +* DESCRIBE SCHEMA FOR TABLE dfs.my_table [AS (JSON | STATEMENT)] */ SqlNode SqlDescribeSchema() : { SqlParserPos pos; - SqlIdentifier schema; + SqlIdentifier table; + String format = "JSON"; } { <DESCRIBE> { pos = getPos(); } - (<SCHEMA> | <DATABASE>) { schema = CompoundIdentifier(); } - { - return new SqlDescribeSchema(pos, schema); - } + ( + <SCHEMA> + ( + <FOR> <TABLE> { table = CompoundIdentifier(); } + [ + <AS> + ( + <JSON> { format = "JSON"; } + | + <STATEMENT> { format = "STATEMENT"; } + ) + ] + { + return new SqlSchema.Describe(pos, table, SqlLiteral.createCharString(format, getPos())); + } + + | + { + return new SqlDescribeSchema(pos, CompoundIdentifier()); + } + ) + | + <DATABASE> + { + return new SqlDescribeSchema(pos, CompoundIdentifier()); + } + ) } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 7b9050f..449dc17 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -36,11 +36,13 @@ import org.apache.drill.exec.planner.sql.handlers.DescribeSchemaHandler; import org.apache.drill.exec.planner.sql.handlers.DescribeTableHandler; import org.apache.drill.exec.planner.sql.handlers.ExplainHandler; import org.apache.drill.exec.planner.sql.handlers.RefreshMetadataHandler; +import org.apache.drill.exec.planner.sql.handlers.SchemaHandler; import org.apache.drill.exec.planner.sql.handlers.SetOptionHandler; import org.apache.drill.exec.planner.sql.handlers.SqlHandlerConfig; import org.apache.drill.exec.planner.sql.parser.DrillSqlCall; import org.apache.drill.exec.planner.sql.parser.DrillSqlDescribeTable; import org.apache.drill.exec.planner.sql.parser.SqlCreateTable; +import org.apache.drill.exec.planner.sql.parser.SqlSchema; import org.apache.drill.exec.testing.ControlsInjector; import org.apache.drill.exec.testing.ControlsInjectorFactory; import org.apache.drill.exec.util.Pointer; @@ -164,6 +166,11 @@ public class DrillSqlWorker { context.setSQLStatementType(SqlStatementType.DESCRIBE_SCHEMA); break; } + if (sqlNode instanceof SqlSchema.Describe) { + handler = new SchemaHandler.Describe(config); + context.setSQLStatementType(SqlStatementType.DESCRIBE_SCHEMA); + break; + } case CREATE_TABLE: handler = ((DrillSqlCall) sqlNode).getSqlHandler(config, textPlan); break; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java index 4882683..9df0eca 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java @@ -29,8 +29,11 @@ import org.apache.drill.exec.planner.sql.DirectPlan; import org.apache.drill.exec.planner.sql.SchemaUtilites; import org.apache.drill.exec.planner.sql.parser.SqlCreateType; import org.apache.drill.exec.planner.sql.parser.SqlSchema; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.schema.FsMetastoreSchemaProvider; import org.apache.drill.exec.record.metadata.schema.PathSchemaProvider; +import org.apache.drill.exec.record.metadata.schema.SchemaContainer; import org.apache.drill.exec.record.metadata.schema.SchemaProvider; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.StorageStrategy; @@ -44,9 +47,11 @@ import org.apache.hadoop.fs.Path; import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** - * Parent class for CREATE / DROP SCHEMA handlers. + * Parent class for CREATE / DROP / DESCRIBE SCHEMA handlers. * Contains common logic on how extract workspace, output error result. */ public abstract class SchemaHandler extends DefaultSqlHandler { @@ -211,4 +216,92 @@ public abstract class SchemaHandler extends DefaultSqlHandler { } } + /** + * DESCRIBE SCHEMA FOR TABLE command handler. + */ + public static class Describe extends SchemaHandler { + + public Describe(SqlHandlerConfig config) { + super(config); + } + + @Override + public PhysicalPlan getPlan(SqlNode sqlNode) { + SqlSchema.Describe sqlCall = ((SqlSchema.Describe) sqlNode); + + String tableName = sqlCall.getTableName(); + WorkspaceSchemaFactory.WorkspaceSchema wsSchema = getWorkspaceSchema(sqlCall.getSchemaPath(), tableName); + + try { + + SchemaProvider schemaProvider = new FsMetastoreSchemaProvider(wsSchema, tableName); + + if (schemaProvider.exists()) { + + SchemaContainer schemaContainer = schemaProvider.read(); + + String schema; + switch (sqlCall.getFormat()) { + case JSON: + schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer); + break; + case STATEMENT: + TupleMetadata metadata = schemaContainer.getSchema(); + StringBuilder builder = new StringBuilder("CREATE OR REPLACE SCHEMA \n"); + builder.append("(\n"); + + builder.append(metadata.toMetadataList().stream() + .map(ColumnMetadata::columnString) + .collect(Collectors.joining(", \n"))); + + builder.append("\n) \n"); + + builder.append("FOR TABLE ").append(schemaContainer.getTable()).append(" \n"); + + Map<String, String> properties = metadata.properties(); + if (!properties.isEmpty()) { + builder.append("PROPERTIES (\n"); + + builder.append(properties.entrySet().stream() + .map(e -> String.format("'%s' = '%s'", e.getKey(), e.getValue())) + .collect(Collectors.joining(", \n"))); + builder.append("\n)"); + } + + schema = builder.toString(); + break; + default: + throw UserException.validationError() + .message("Unsupported describe schema format: [%s]", sqlCall.getFormat()) + .build(logger); + } + + return DirectPlan.createDirectPlan(context, new SchemaResult(schema)); + } + + return DirectPlan.createDirectPlan(context, false, + String.format("Schema for table [%s] is absent", sqlCall.getTable())); + + } catch (IOException e) { + throw UserException.resourceError(e) + .message(e.getMessage()) + .addContext("Error while accessing table location for [%s]", sqlCall.getTable()) + .build(logger); + } + } + + /** + * Wrapper to output schema in a form of table with one column named `schema`. + */ + public static class SchemaResult { + + public String schema; + + public SchemaResult(String schema) { + this.schema = schema; + } + } + + } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java index 5d3d080..119b27d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java @@ -82,6 +82,7 @@ public class CompoundIdentifierConverter extends SqlShuttle { .put(SqlDropFunction.class, arrayOf(D)) .put(SqlSchema.Create.class, arrayOf(D, D, D, D, D, D)) .put(SqlSchema.Drop.class, arrayOf(D, D)) + .put(SqlSchema.Describe.class, arrayOf(D, D)) .build(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java index 7985279..bfbf06f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Map; /** - * Parent class for CREATE and DROP SCHEMA commands. + * Parent class for CREATE, DROP, DESCRIBE SCHEMA commands. * Holds logic common command property: table. */ public abstract class SqlSchema extends DrillSqlCall { @@ -277,4 +277,66 @@ public abstract class SqlSchema extends DrillSqlCall { } + /** + * DESCRIBE SCHEMA FOR TABLE sql call. + */ + public static class Describe extends SqlSchema { + + private final SqlLiteral format; + + public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator(SqlKind.DESCRIBE_SCHEMA.name(), SqlKind.DESCRIBE_SCHEMA) { + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) { + return new Describe(pos, (SqlIdentifier) operands[0], (SqlLiteral) operands[1]); + } + }; + + public Describe(SqlParserPos pos, SqlIdentifier table, SqlLiteral format) { + super(pos, table); + this.format = format; + } + + @Override + public SqlOperator getOperator() { + return OPERATOR; + } + + @Override + public List<SqlNode> getOperandList() { + return Arrays.asList(table, format); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword("DESCRIBE"); + writer.keyword("SCHEMA"); + + super.unparse(writer, leftPrec, rightPrec); + + writer.keyword("AS"); + writer.keyword(getFormat().name()); + } + + public Describe.Format getFormat() { + return Format.valueOf(format.toValue()); + } + + /** + * Enum which specifies format of DESCRIBE SCHEMA FOR table output. + */ + public enum Format { + + /** + * Schema will be output in JSON format used to store schema + * in {@link org.apache.drill.exec.record.metadata.schema.SchemaProvider#DEFAULT_SCHEMA_NAME} file. + */ + JSON, + + /** + * Schema will be output in CREATE SCHEMA command syntax. + */ + STATEMENT + } + } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java index 28754aa..d73e247 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/PathSchemaProvider.java @@ -45,12 +45,12 @@ public class PathSchemaProvider implements SchemaProvider { * Reader used to read JSON schema from file into into {@link SchemaContainer}. * Allows comment inside the JSON file. */ - private static final ObjectReader READER; + public static final ObjectReader READER; /** * Writer used to write content from {@link SchemaContainer} into JSON file. */ - private static final ObjectWriter WRITER; + public static final ObjectWriter WRITER; static { ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true); diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java index 92a7b27..7391366 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java @@ -553,4 +553,116 @@ public class TestSchemaCommands extends ClusterTest { } } + @Test + public void testDescribeForMissingTable() throws Exception { + thrown.expect(UserException.class); + thrown.expectMessage("VALIDATION ERROR: Table [t] was not found"); + + run("describe schema for table dfs.t"); + } + + @Test + public void testDescribeForMissingSchema() throws Exception { + String table = "dfs.tmp.table_describe_with_missing_schema"; + try { + run("create table %s as select 'a' as c from (values(1))", table); + + testBuilder() + .sqlQuery("describe schema for table %s", table) + .unOrdered() + .baselineColumns("ok", "summary") + .baselineValues(false, String.format("Schema for table [%s] is absent", table)) + .go(); + + } finally { + run("drop table if exists %s", table); + } + } + + @Test + public void testDescribeDefault() throws Exception { + String tableName = "table_describe_default"; + String table = String.format("dfs.tmp.%s", tableName); + try { + run("create table %s as select 'a' as c from (values(1))", table); + run("create schema (col int) for table %s", table); + + File schemaFile = Paths.get(dirTestWatcher.getDfsTestTmpDir().getPath(), + tableName, SchemaProvider.DEFAULT_SCHEMA_NAME).toFile(); + + SchemaProvider schemaProvider = new PathSchemaProvider(new Path(schemaFile.getPath())); + SchemaContainer schemaContainer = schemaProvider.read(); + String schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer); + + testBuilder() + .sqlQuery("describe schema for table %s", table) + .unOrdered() + .baselineColumns("schema") + .baselineValues(schema) + .go(); + + } finally { + run("drop table if exists %s", table); + } + } + + @Test + public void testDescribeJson() throws Exception { + String tableName = "table_describe_json"; + String table = String.format("dfs.tmp.%s", tableName); + try { + run("create table %s as select 'a' as c from (values(1))", table); + run("create schema (col int) for table %s", table); + + File schemaFile = Paths.get(dirTestWatcher.getDfsTestTmpDir().getPath(), + tableName, SchemaProvider.DEFAULT_SCHEMA_NAME).toFile(); + + SchemaProvider schemaProvider = new PathSchemaProvider(new Path(schemaFile.getPath())); + SchemaContainer schemaContainer = schemaProvider.read(); + String schema = PathSchemaProvider.WRITER.writeValueAsString(schemaContainer); + + testBuilder() + .sqlQuery("describe schema for table %s as json", table) + .unOrdered() + .baselineColumns("schema") + .baselineValues(schema) + .go(); + + } finally { + run("drop table if exists %s", table); + } + } + + @Test + public void testDescribeStatement() throws Exception { + String tableName = "table_describe_statement"; + String table = String.format("dfs.tmp.%s", tableName); + try { + run("create table %s as select 'a' as c from (values(1))", table); + + String statement = "CREATE OR REPLACE SCHEMA \n" + + "(\n" + + "`col1` DATE FORMAT 'yyyy-MM-dd' DEFAULT '-1', \n" + + "`col2` INT NOT NULL FORMAT 'yyyy-MM-dd' PROPERTIES { 'drill.strict' = 'true', 'some_column_prop' = 'some_column_val' }\n" + + ") \n" + + "FOR TABLE dfs.tmp.`table_describe_statement` \n" + + "PROPERTIES (\n" + + "'drill.strict' = 'false', \n" + + "'some_schema_prop' = 'some_schema_val'\n" + + ")"; + + run(statement); + + testBuilder() + .sqlQuery("describe schema for table %s as statement", table) + .unOrdered() + .baselineColumns("schema") + .baselineValues(statement) + .go(); + + } finally { + run("drop table if exists %s", table); + } + } + }
