Repository: calcite Updated Branches: refs/heads/master 90f49be63 -> f0dd0db0b
[CALCITE-1026] Implementation of YAML's model format. Make Json/Yaml mappers static final. Tests to check model inline: json and yaml and to check model uri to hson/yaml files Close apache/calcite#791 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/f0dd0db0 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/f0dd0db0 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/f0dd0db0 Branch: refs/heads/master Commit: f0dd0db0b5f80ce0a5c565dde72b98f5b8ff7ac3 Parents: 90f49be Author: snuyanzin <[email protected]> Authored: Tue Jul 31 20:17:01 2018 +0300 Committer: snuyanzin <[email protected]> Committed: Thu Aug 30 13:31:17 2018 +0300 ---------------------------------------------------------------------- core/pom.xml | 4 + .../org/apache/calcite/model/ModelHandler.java | 27 ++- .../java/org/apache/calcite/test/ModelTest.java | 31 +++- core/src/test/resources/empty-model.yaml | 20 ++ example/csv/src/test/resources/bug.yaml | 24 +++ .../src/test/resources/filterable-model.yaml | 29 +++ .../src/test/resources/model-stream-table.yaml | 29 +++ .../test/resources/model-with-custom-table.yaml | 27 +++ .../csv/src/test/resources/model-with-view.yaml | 30 +++ example/csv/src/test/resources/model.yaml | 26 +++ .../src/test/resources/order-stream-table.yaml | 33 ++++ example/csv/src/test/resources/smart.yaml | 34 ++++ example/function/src/test/resources/model.yaml | 30 +++ pom.xml | 5 + site/_docs/adapter.md | 2 +- site/_docs/model.md | 186 ++++++++++++++++++- 16 files changed, 525 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index c492b33..c8d6143 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -80,6 +80,10 @@ limitations under the License. <artifactId>jackson-databind</artifactId> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + </dependency> + <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/core/src/main/java/org/apache/calcite/model/ModelHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java index 1bcae90..ff76591 100644 --- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java +++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java @@ -46,6 +46,7 @@ import org.apache.calcite.util.Util; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -66,6 +67,12 @@ import javax.sql.DataSource; * Reads a model and creates schema objects accordingly. */ public class ModelHandler { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper() + .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) + .configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true) + .configure(JsonParser.Feature.ALLOW_COMMENTS, true); + private static final ObjectMapper YAML_MAPPER = new YAMLMapper(); + private final CalciteConnection connection; private final Deque<Pair<String, SchemaPlus>> schemaStack = new ArrayDeque<>(); private final String modelUri; @@ -77,15 +84,18 @@ public class ModelHandler { super(); this.connection = connection; this.modelUri = uri; - final ObjectMapper mapper = new ObjectMapper(); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + JsonRoot root; + ObjectMapper mapper; if (uri.startsWith("inline:")) { - root = mapper.readValue( - uri.substring("inline:".length()), JsonRoot.class); + // trim here is to correctly autodetect if it is json or not in case of leading spaces + String inline = uri.substring("inline:".length()).trim(); + mapper = (inline.startsWith("/*") || inline.startsWith("{")) + ? JSON_MAPPER + : YAML_MAPPER; + root = mapper.readValue(inline, JsonRoot.class); } else { + mapper = uri.endsWith(".yaml") || uri.endsWith(".yml") ? YAML_MAPPER : JSON_MAPPER; root = mapper.readValue(new File(uri), JsonRoot.class); } visit(root); @@ -547,7 +557,10 @@ public class ModelHandler { * {@link JsonCustomSchema#operand}, as extra context for the adapter. */ public enum ExtraOperand { /** URI of model, e.g. "target/test-classes/model.json", - * "http://localhost/foo/bar.json";, "inline:{...}". */ + * "http://localhost/foo/bar.json";, "inline:{...}", + * "target/test-classes/model.yaml", + * "http://localhost/foo/bar.yaml";, "inline:..." + * */ MODEL_URI("modelUri"), /** Base directory from which to read files. */ http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/core/src/test/java/org/apache/calcite/test/ModelTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/ModelTest.java b/core/src/test/java/org/apache/calcite/test/ModelTest.java index 158527d..7940a0d 100644 --- a/core/src/test/java/org/apache/calcite/test/ModelTest.java +++ b/core/src/test/java/org/apache/calcite/test/ModelTest.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.test; +import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.model.JsonColumn; import org.apache.calcite.model.JsonCustomSchema; import org.apache.calcite.model.JsonCustomTable; @@ -33,6 +34,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.io.IOException; +import java.net.URL; import java.util.List; import static org.hamcrest.CoreMatchers.equalTo; @@ -342,12 +344,39 @@ public class ModelTest { final JsonView table1 = (JsonView) schema.tables.get(0); try { String s = table1.getSql(); - fail("exprcted error, got " + s); + fail("expected error, got " + s); } catch (RuntimeException e) { assertThat(e.getMessage(), equalTo("each element of a string list must be a string; found: 2")); } } + + @Test public void testYamlInlineDetection() throws Exception { + // yaml model with different line endings + final String yamlModel = "version: 1.0\r\n" + + "schemas: \n" + + "- type: custom\r\n" + + " name: 'MyCustomSchema'\n" + + " factory: " + JdbcTest.MySchemaFactory.class.getName() + "\r\n"; + CalciteAssert.model(yamlModel).doWithConnection(calciteConnection -> null); + // with a comment + CalciteAssert.model("\n \r\n# comment\n " + yamlModel) + .doWithConnection(calciteConnection -> null); + // if starts with { => treated as json + CalciteAssert.model(" { " + yamlModel + " }") + .connectThrows("Unexpected character ('s' (code 115)): " + + "was expecting comma to separate Object entries"); + // if starts with /* => treated as json + CalciteAssert.model(" /* " + yamlModel) + .connectThrows("Unexpected end-of-input in a comment"); + } + + @Test public void testYamlFileDetection() throws Exception { + final URL inUrl = ModelTest.class.getResource("/empty-model.yaml"); + CalciteAssert.that() + .with(CalciteConnectionProperty.MODEL, inUrl.getFile()) + .doWithConnection(calciteConnection -> null); + } } // End ModelTest.java http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/core/src/test/resources/empty-model.yaml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/empty-model.yaml b/core/src/test/resources/empty-model.yaml new file mode 100644 index 0000000..1e2f58d --- /dev/null +++ b/core/src/test/resources/empty-model.yaml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a simple Calcite schema. +# +version: 1.0 +defaultSchema: EMPTY_SCHEMA http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/bug.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/bug.yaml b/example/csv/src/test/resources/bug.yaml new file mode 100644 index 0000000..fd50d5b --- /dev/null +++ b/example/csv/src/test/resources/bug.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 1.0 +defaultSchema: BUG +schemas: +- name: BUG + type: custom + factory: org.apache.calcite.adapter.csv.CsvSchemaFactory + operand: + directory: bug http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/filterable-model.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/filterable-model.yaml b/example/csv/src/test/resources/filterable-model.yaml new file mode 100644 index 0000000..17ccffb --- /dev/null +++ b/example/csv/src/test/resources/filterable-model.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a Calcite schema that is similar to model.json, +# except that it produces tables that implement FilterableTable. +# These tables can implement their own simple filtering. +# +version: 1.0, +defaultSchema: SALES +schemas: +- name: SALES + type: custom + factory: org.apache.calcite.adapter.csv.CsvSchemaFactory + operand: + directory: sales + flavor: FILTERABLE http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/model-stream-table.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/model-stream-table.yaml b/example/csv/src/test/resources/model-stream-table.yaml new file mode 100644 index 0000000..3a3f2a7 --- /dev/null +++ b/example/csv/src/test/resources/model-stream-table.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 1.0 +defaultSchema: STREAM +schemas: +- name: SS + tables: + - name: DEPTS + type: custom + factory: org.apache.calcite.adapter.csv.CsvStreamTableFactory + stream: + stream: true + operand: + file: sales/SDEPTS.csv + flavor: scannable http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/model-with-custom-table.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/model-with-custom-table.yaml b/example/csv/src/test/resources/model-with-custom-table.yaml new file mode 100644 index 0000000..5aa6084 --- /dev/null +++ b/example/csv/src/test/resources/model-with-custom-table.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 1.0 +defaultSchema: CUSTOM_TABLE +schemas: +- name: CUSTOM_TABLE + tables: + - name: EMPS + type: custom + factory: org.apache.calcite.adapter.csv.CsvTableFactory + operand: + file: sales/EMPS.csv.gz + flavor: scannable http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/model-with-view.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/model-with-view.yaml b/example/csv/src/test/resources/model-with-view.yaml new file mode 100644 index 0000000..aca379e --- /dev/null +++ b/example/csv/src/test/resources/model-with-view.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a Calcite schema that includes a view. +# +version: 1.0 +defaultSchema: SALES +schemas: +- name: SALES + type: custom + factory: org.apache.calcite.adapter.csv.CsvSchemaFactory + operand: + directory: sales + tables: + - name: FEMALE_EMPS + type: view + sql: SELECT * FROM emps WHERE gender = 'F' http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/model.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/model.yaml b/example/csv/src/test/resources/model.yaml new file mode 100644 index 0000000..9478e58 --- /dev/null +++ b/example/csv/src/test/resources/model.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a simple Calcite schema. +# +version: 1.0 +defaultSchema: SALES +schemas: +- name: SALES + type: custom + factory: org.apache.calcite.adapter.csv.CsvSchemaFactory + operand: + directory: sales http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/order-stream-table.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/order-stream-table.yaml b/example/csv/src/test/resources/order-stream-table.yaml new file mode 100644 index 0000000..24cb255 --- /dev/null +++ b/example/csv/src/test/resources/order-stream-table.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 1.0 +defaultSchema: foodmart +schemas: +- name: STREAMS + tables: + - type: custom + name: ORDERS + stream: + stream: true + factory: org.apache.calcite.test.StreamTest$OrdersStreamTableFactory +- name: INFINITE_STREAMS + tables: + - type: custom + name: ORDERS + stream: + stream: true + factory: org.apache.calcite.test.StreamTest$InfiniteOrdersStreamTableFactory http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/csv/src/test/resources/smart.yaml ---------------------------------------------------------------------- diff --git a/example/csv/src/test/resources/smart.yaml b/example/csv/src/test/resources/smart.yaml new file mode 100644 index 0000000..1fa57f3 --- /dev/null +++ b/example/csv/src/test/resources/smart.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a Calcite schema that is similar to model.json, +# except that it produces tables that implement FilterableTable. +# These tables can implement their own simple filtering. +# +# A JSON model of a Calcite schema that is similar to model.json, +# except that it produces tables that implement +# TranslatableTable. These tables are translated to a CsvTableScan +# relational expression which participates in query planning. +# +version: 1.0 +defaultSchema: SALES +schemas: +- name: SALES + type: custom + factory: org.apache.calcite.adapter.csv.CsvSchemaFactory + operand: + directory: sales + flavor: TRANSLATABLE http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/example/function/src/test/resources/model.yaml ---------------------------------------------------------------------- diff --git a/example/function/src/test/resources/model.yaml b/example/function/src/test/resources/model.yaml new file mode 100644 index 0000000..961c0c1 --- /dev/null +++ b/example/function/src/test/resources/model.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A JSON model of a Calcite that contains various user-defined functions. +# +version: 1.0 +defaultSchema: MAZE +schemas: +- name: MAZE + type: map + functions: + - name: MAZE + className: org.apache.calcite.example.maze.MazeTable + methodName: generate + - name: SOLVE + className: org.apache.calcite.example.maze.MazeTable + methodName: solve http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9ddb80d..ddd0222 100644 --- a/pom.xml +++ b/pom.xml @@ -270,6 +270,11 @@ limitations under the License. <version>${jackson.version}</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + <version>${jackson.version}</version> + </dependency> + <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> <version>${joda.version}</version> http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/site/_docs/adapter.md ---------------------------------------------------------------------- diff --git a/site/_docs/adapter.md b/site/_docs/adapter.md index 01c8c87..7e50a7a 100644 --- a/site/_docs/adapter.md +++ b/site/_docs/adapter.md @@ -92,7 +92,7 @@ as implemented by Avatica's | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#FUN">fun</a> | Collection of built-in functions and operators. Valid values are "standard" (the default), "oracle", "spatial", and may be combined using commas, for example "oracle,spatial". | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#LEX">lex</a> | Lexical policy. Values are ORACLE (default), MYSQL, MYSQL_ANSI, SQL_SERVER, JAVA. | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#MATERIALIZATIONS_ENABLED">materializationsEnabled</a> | Whether Calcite should use materializations. Default false. -| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#MODEL">model</a> | URI of the JSON model file. +| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#MODEL">model</a> | URI of the JSON/YAML model file or inline like `inline:{...}` for JSON and `inline:...` for YAML. | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#PARSER_FACTORY">parserFactory</a> | Parser factory. The name of a class that implements [<tt>interface SqlParserImplFactory</tt>]({{ site.apiRoot }}/org/apache/calcite/sql/parser/SqlParserImplFactory.html) and has a public default constructor or an `INSTANCE` constant. | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#QUOTING">quoting</a> | How identifiers are quoted. Values are DOUBLE_QUOTE, BACK_QUOTE, BRACKET. If not specified, value from `lex` is used. | <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#QUOTED_CASING">quotedCasing</a> | How identifiers are stored if they are quoted. Values are UNCHANGED, TO_UPPER, TO_LOWER. If not specified, value from `lex` is used. http://git-wip-us.apache.org/repos/asf/calcite/blob/f0dd0db0/site/_docs/model.md ---------------------------------------------------------------------- diff --git a/site/_docs/model.md b/site/_docs/model.md index 0f646d0..01fa9b2 100644 --- a/site/_docs/model.md +++ b/site/_docs/model.md @@ -1,6 +1,6 @@ --- layout: docs -title: JSON models +title: JSON/YAML models permalink: /docs/model.html --- <!-- @@ -22,7 +22,7 @@ limitations under the License. {% endcomment %} --> -Calcite models can be represented as JSON files. +Calcite models can be represented as JSON/YAML files. This page describes the structure of those files. Models can also be built programmatically using the `Schema` SPI. @@ -31,6 +31,7 @@ Models can also be built programmatically using the `Schema` SPI. ### Root +#### JSON {% highlight json %} { version: '1.0', @@ -39,6 +40,14 @@ Models can also be built programmatically using the `Schema` SPI. } {% endhighlight %} +#### YAML +{% highlight yaml %} +version: 1.0 +defaultSchema: mongo +schemas: +- [Schema...] +{% endhighlight %} + `version` (required string) must have value `1.0`. `defaultSchema` (optional string). If specified, it is @@ -51,6 +60,7 @@ become the default schema for connections to Calcite that use this model. Occurs within `root.schemas`. +#### JSON {% highlight json %} { name: 'foodmart', @@ -60,6 +70,16 @@ Occurs within `root.schemas`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: foodmart +path: + lib +cache: true +materializations: +- [ Materialization... ] +{% endhighlight %} + `name` (required string) is the name of the schema. `type` (optional string, default `map`) indicates sub-type. Values are: @@ -73,10 +93,18 @@ resolve functions used in this schema. If specified it must be a list, and each element of the list must be either a string or a list of strings. For example, +#### JSON {% highlight json %} path: [ ['usr', 'lib'], 'lib' ] {% endhighlight %} +#### YAML +{% highlight yaml %} +path: +- [usr, lib] +- lib +{% endhighlight %} + declares a path with two elements: the schema '/usr/lib' and the schema '/lib'. Most schemas are at the top level, and for these you can use a string. @@ -110,6 +138,7 @@ immediately, and are never flushed. Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. +#### JSON {% highlight json %} { name: 'foodmart', @@ -120,6 +149,18 @@ Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: foodmart +type: map +tables: +- [ Table... ] +functions: +- [ Function... ] +types: +- [ Type... ] +{% endhighlight %} + `name`, `type`, `path`, `cache`, `materializations` inherited from <a href="#schema">Schema</a>. @@ -135,6 +176,7 @@ defines the functions in this schema. Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. +#### JSON {% highlight json %} { name: 'mongo', @@ -147,6 +189,16 @@ Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: mongo +type: custom +factory: org.apache.calcite.adapter.mongodb.MongoSchemaFactory +operand: + host: localhost + database: test +{% endhighlight %} + `name`, `type`, `path`, `cache`, `materializations` inherited from <a href="#schema">Schema</a>. @@ -161,7 +213,7 @@ factory. ### JDBC Schema Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. - +#### JSON {% highlight json %} { name: 'foodmart', @@ -175,6 +227,18 @@ Like base class <a href="#schema">Schema</a>, occurs within `root.schemas`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: foodmart +type: jdbc +jdbcDriver: TODO +jdbcUrl: TODO +jdbcUser: TODO +jdbcPassword: TODO +jdbcCatalog: TODO +jdbcSchema: TODO +{% endhighlight %} + `name`, `type`, `path`, `cache`, `materializations` inherited from <a href="#schema">Schema</a>. @@ -198,6 +262,7 @@ data source. Occurs within `root.schemas.materializations`. +#### JSON {% highlight json %} { view: 'V', @@ -206,6 +271,13 @@ Occurs within `root.schemas.materializations`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +view: V +table: T +sql: select deptno, count(*) as c, sum(sal) as s from emp group by deptno +{% endhighlight %} + `view` (optional string) is the name of the view; null means that the table already exists and is populated with the correct data. @@ -220,6 +292,7 @@ Calcite will create and populate an in-memory table. Occurs within `root.schemas.tables`. +#### JSON {% highlight json %} { name: 'sales_fact', @@ -227,6 +300,13 @@ Occurs within `root.schemas.tables`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: sales_fact +columns: + [ Column... ] +{% endhighlight %} + `name` (required string) is the name of this table. Must be unique within the schema. `type` (optional string, default `custom`) indicates sub-type. Values are: @@ -241,6 +321,7 @@ some kinds of table, optional for others such as View) Like base class <a href="#table">Table</a>, occurs within `root.schemas.tables`. +#### JSON {% highlight json %} { name: 'female_emps', @@ -250,6 +331,14 @@ Like base class <a href="#table">Table</a>, occurs within `root.schemas.tables`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: female_emps +type: view +sql: select * from emps where gender = 'F' +modifiable: true +{% endhighlight %} + `name`, `type`, `columns` inherited from <a href="#table">Table</a>. `sql` (required string, or list of strings that will be concatenated as a @@ -287,6 +376,7 @@ Errors regarding modifiable views: Like base class <a href="#table">Table</a>, occurs within `root.schemas.tables`. +#### JSON {% highlight json %} { name: 'female_emps', @@ -298,6 +388,15 @@ Like base class <a href="#table">Table</a>, occurs within `root.schemas.tables`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: female_emps +type: custom +factory: TODO +operand: + todo: TODO +{% endhighlight %} + `name`, `type`, `columns` inherited from <a href="#table">Table</a>. `factory` (required string) is the name of the factory class for this @@ -314,6 +413,7 @@ Information about whether a table allows streaming. Occurs within `root.schemas.tables.stream`. +#### JSON {% highlight json %} { stream: true, @@ -321,6 +421,12 @@ Occurs within `root.schemas.tables.stream`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +stream: true +history: false +{% endhighlight %} + `stream` (optional; default true) is whether the table allows streaming. `history` (optional; default false) is whether the history of the stream is @@ -330,18 +436,25 @@ available. Occurs within `root.schemas.tables.columns`. +#### JSON {% highlight json %} { name: 'empno' } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: empno +{% endhighlight %} + `name` (required string) is the name of this column. ### Function Occurs within `root.schemas.functions`. +#### JSON {% highlight json %} { name: 'MY_PLUS', @@ -351,6 +464,14 @@ Occurs within `root.schemas.functions`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: MY_PLUS +className: com.example.functions.MyPlusFunction +methodName: apply +path: {} +{% endhighlight %} + `name` (required string) is the name of this function. `className` (required string) is the name of the class that implements this @@ -377,6 +498,7 @@ if found, creates an aggregate function. Occurs within `root.schemas.types`. +#### JSON {% highlight json %} { name: 'mytype1', @@ -390,6 +512,15 @@ Occurs within `root.schemas.types`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: mytype1 +type: BIGINT +attributes: +- name: f1 + type: BIGINT +{% endhighlight %} + `name` (required string) is the name of this type. `type` (optional) is the SQL type. @@ -402,6 +533,7 @@ If `attributes` and `type` both exist at the same level, Occurs within `root.schemas.lattices`. +#### JSON {% highlight json %} { name: 'star', @@ -433,6 +565,30 @@ Occurs within `root.schemas.lattices`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +name: star +sql: > + select 1 from "foodmart"."sales_fact_1997" as "s"', + join "foodmart"."product" as "p" using ("product_id")', + join "foodmart"."time_by_day" as "t" using ("time_id")', + join "foodmart"."product_class" as "pc" on "p"."product_class_id" = "pc"."product_class_id" +auto: false +algorithm: true +algorithmMaxMillis: 10000 +rowCountEstimate: 86837 +defaultMeasures: +- agg: count +tiles: +- dimensions: [ 'the_year', ['t', 'quarter'] ] + measures: + - agg: sum + args: unit_sales + - agg: sum + args: store_sales + - agg: 'count' +{% endhighlight %} + `name` (required string) is the name of this lattice. `sql` (required string, or list of strings that will be concatenated as a @@ -461,10 +617,16 @@ Any tile defined in `tiles` can still define its own measures, including measures not on this list. If not specified, the default list of measures is just 'count(*)': +#### JSON {% highlight json %} [ { name: 'count' } ] {% endhighlight %} +#### YAML +{% highlight yaml %} +name: count +{% endhighlight %} + `statisticProvider` (optional name of a class that implements [org.apache.calcite.materialize.LatticeStatisticProvider]({{ site.apiRoot }}/org/apache/calcite/materialize/LatticeStatisticProvider.html)) provides estimates of the number of distinct values in each column. @@ -499,6 +661,17 @@ Occurs within `root.schemas.lattices.tiles`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +dimensions: [ 'the_year', ['t', 'quarter'] ] +measures: +- agg: sum + args: unit_sales +- agg: sum + args: store_sales +- agg: count +{% endhighlight %} + `dimensions` (list of strings or string lists, required, but may be empty) defines the dimensionality of this tile. Each dimension is a column from the lattice, like a `GROUP BY` clause. @@ -515,6 +688,7 @@ lattice's default measure list. Occurs within `root.schemas.lattices.defaultMeasures` and `root.schemas.lattices.tiles.measures`. +#### JSON {% highlight json %} { agg: 'sum', @@ -522,6 +696,12 @@ and `root.schemas.lattices.tiles.measures`. } {% endhighlight %} +#### YAML +{% highlight yaml %} +agg: sum +args: unit_sales +{% endhighlight %} + `agg` is the name of an aggregate function (usually 'count', 'sum', 'min', 'max').
