Updated Branches: refs/heads/execwork 0d2428fde -> 5ede21ffc
Exploration modifications to physical plan representation, moving further away from similarities to LogicalPlan. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5ede21ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5ede21ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5ede21ff Branch: refs/heads/execwork Commit: 5ede21ffc77eb5effe5f2e614f93ee0f86860716 Parents: 0d2428f Author: Jacques Nadeau <[email protected]> Authored: Mon Apr 22 23:04:27 2013 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon Apr 22 23:10:06 2013 -0700 ---------------------------------------------------------------------- .../apache/drill/common/physical/PhysicalPlan.java | 13 +-- .../apache/drill/common/physical/ReadEntry.java | 2 +- .../apache/drill/common/physical/RecordField.java | 8 +-- .../apache/drill/common/physical/WriteEntry.java | 22 +++++ .../drill/common/physical/pop/ExchangePOP.java | 2 - .../drill/common/physical/pop/FieldCombinePOP.java | 28 ++++++ .../common/physical/pop/FieldSubdividePOP.java | 22 +++++ .../drill/common/physical/pop/ProjectPOP.java | 53 +++++++++++ .../apache/drill/common/physical/pop/ScanPOP.java | 28 +----- .../apache/drill/common/physical/pop/SortPOP.java | 54 +++++++++++ .../apache/drill/common/physical/pop/StorePOP.java | 19 +++- .../apache/drill/common/physical/MockScanPOP.java | 47 ++++++++++ .../apache/drill/common/physical/MockStorePOP.java | 56 +++++++++++ .../common/src/test/resources/drill-module.conf | 3 +- .../common/src/test/resources/dsort-logical.json | 40 ++++++++ .../common/src/test/resources/dsort-physical.json | 66 +++++++++++++ .../common/src/test/resources/dsort_logical.json | 40 -------- .../common/src/test/resources/dsort_physical.json | 72 --------------- 18 files changed, 412 insertions(+), 163 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/PhysicalPlan.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/PhysicalPlan.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/PhysicalPlan.java index 0ef5164..b81ca42 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/PhysicalPlan.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/PhysicalPlan.java @@ -37,17 +37,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -@JsonPropertyOrder({ "head", "storage", "graph" }) +@JsonPropertyOrder({ "head", "graph" }) public class PhysicalPlan { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalPlan.class); - Map<String, StorageEngineConfig> storageEngines; PlanProperties properties; Graph<PhysicalOperator, SinkPOP, SourcePOP> graph; @JsonCreator - public PhysicalPlan(@JsonProperty("head") PlanProperties properties, @JsonProperty("storage") Map<String, StorageEngineConfig> storageEngines, @JsonProperty("graph") List<PhysicalOperator> operators){ - this.storageEngines = storageEngines; + public PhysicalPlan(@JsonProperty("head") PlanProperties properties, @JsonProperty("graph") List<PhysicalOperator> operators){ this.properties = properties; this.graph = Graph.newGraph(operators, SinkPOP.class, SourcePOP.class); } @@ -58,12 +56,7 @@ public class PhysicalPlan { // reverse the list so that nested references are flattened rather than nested. return Lists.reverse(list); } - - - @JsonProperty("storage") - public Map<String, StorageEngineConfig> getStorageEngines() { - return storageEngines; - } + @JsonProperty("head") public PlanProperties getProperties() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/ReadEntry.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/ReadEntry.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/ReadEntry.java index 47cfb5c..7c23cf5 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/ReadEntry.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/ReadEntry.java @@ -21,5 +21,5 @@ package org.apache.drill.common.physical; * Describes a chunk of read work that will be done. */ public interface ReadEntry { - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/RecordField.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/RecordField.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/RecordField.java index 2867084..8d0072a 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/RecordField.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/RecordField.java @@ -26,22 +26,16 @@ public class RecordField { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordField.class); - private String name; private DataType type; private ValueMode mode; @JsonCreator - public RecordField(@JsonProperty("name") String name, @JsonProperty("type") DataType type, @JsonProperty("mode") ValueMode mode) { + public RecordField(@JsonProperty("type") DataType type, @JsonProperty("mode") ValueMode mode) { super(); - this.name = name; this.type = type; this.mode = mode; } - public String getName() { - return name; - } - public DataType getType() { return type; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/WriteEntry.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/WriteEntry.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/WriteEntry.java new file mode 100644 index 0000000..7440ce2 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/WriteEntry.java @@ -0,0 +1,22 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical; + +public interface WriteEntry { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WriteEntry.class); +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ExchangePOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ExchangePOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ExchangePOP.java index 4c1f08a..757f03b 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ExchangePOP.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ExchangePOP.java @@ -40,8 +40,6 @@ public class ExchangePOP extends SingleChildPOP{ this.partition = partition; this.stitch = stitch; } - - public PartitionDef getPartition() { return partition; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldCombinePOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldCombinePOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldCombinePOP.java new file mode 100644 index 0000000..ac7e036 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldCombinePOP.java @@ -0,0 +1,28 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical.pop; + +/** + * Creates a complex field out of two or more component fields + */ +public class FieldCombinePOP { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldCombinePOP.class); + + // fieldsInSortOrder + private int[] fieldIds; +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldSubdividePOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldSubdividePOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldSubdividePOP.java new file mode 100644 index 0000000..c5bd1f9 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/FieldSubdividePOP.java @@ -0,0 +1,22 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical.pop; + +public class FieldSubdividePOP { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldSubdividePOP.class); +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ProjectPOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ProjectPOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ProjectPOP.java new file mode 100644 index 0000000..bd481d4 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ProjectPOP.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical.pop; + +import java.util.List; + +import org.apache.drill.common.defs.PartitionDef; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.physical.FieldSet; +import org.apache.drill.common.physical.StitchDef; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("project") +public class ProjectPOP extends SingleChildPOP{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectPOP.class); + + private List<Integer> fieldIds; + private List<LogicalExpression> exprs; + + @JsonCreator + public ProjectPOP(@JsonProperty("output") FieldSet fields, @JsonProperty("fields") List<Integer> fieldIds, @JsonProperty("exprs") List<LogicalExpression> exprs) { + super(fields); + this.fieldIds = fieldIds; + this.exprs = exprs; + } + + public List<Integer> getFields() { + return fieldIds; + } + + public List<LogicalExpression> getExprs() { + return exprs; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ScanPOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ScanPOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ScanPOP.java index 30cb2b0..2aaf8fa 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ScanPOP.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/ScanPOP.java @@ -31,45 +31,25 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -@JsonTypeName("scan") -public class ScanPOP extends POPBase implements SourcePOP{ +public abstract class ScanPOP<T extends ReadEntry> extends POPBase implements SourcePOP{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanPOP.class); - private List<JSONOptions> readEntries; - private String storageEngine; + private List<T> readEntries; - @JsonCreator - public ScanPOP(@JsonProperty("storageengine") String storageEngine, @JsonProperty("entries") List<JSONOptions> readEntries, @JsonProperty("fields") FieldSet fieldSet) { + public ScanPOP(List<T> readEntries, FieldSet fieldSet) { super(fieldSet); - this.storageEngine = storageEngine; this.readEntries = readEntries; } @JsonProperty("entries") - public List<JSONOptions> getReadEntries() { + public List<T> getReadEntries() { return readEntries; } - public <T extends ReadEntry> List<T> getReadEntries(DrillConfig config, Class<T> clazz){ - List<T> e = Lists.newArrayList(); - for(JSONOptions o : readEntries){ - e.add(o.getWith(config, clazz)); - } - return e; - } - @Override public Iterator<PhysicalOperator> iterator() { return Iterators.emptyIterator(); } - public static org.slf4j.Logger getLogger() { - return logger; - } - - @JsonProperty("storageengine") - public String getStorageEngine() { - return storageEngine; - } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/SortPOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/SortPOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/SortPOP.java new file mode 100644 index 0000000..4d0263b --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/SortPOP.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical.pop; + +import java.util.List; + +import org.apache.drill.common.defs.PartitionDef; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.physical.FieldSet; +import org.apache.drill.common.physical.StitchDef; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("sort") +public class SortPOP extends SingleChildPOP{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SortPOP.class); + + private int field; + private boolean reverse = false; + + @JsonCreator + public SortPOP(@JsonProperty("output") FieldSet fields, @JsonProperty("field") int field, @JsonProperty("reverse") boolean reverse) { + super(fields); + this.field = field; + this.reverse = reverse; + } + + public int getField() { + return field; + } + + public boolean getReverse() { + return reverse; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/StorePOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/StorePOP.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/StorePOP.java index 2fbaa99..2b8e075 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/StorePOP.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/physical/pop/StorePOP.java @@ -22,33 +22,40 @@ import java.util.List; import org.apache.drill.common.JSONOptions; import org.apache.drill.common.defs.PartitionDef; import org.apache.drill.common.physical.FieldSet; +import org.apache.drill.common.physical.WriteEntry; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -@JsonTypeName("store") -public class StorePOP extends SingleChildPOP implements SinkPOP{ +public abstract class StorePOP<T extends WriteEntry> extends SingleChildPOP implements SinkPOP{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorePOP.class); public static enum StoreMode {SYSTEM_CHOICE, PREDEFINED_PARTITIONS}; private StoreMode mode; - private PartitionDef partitioning; + private PartitionDef partition; + private List<T> entries; @JsonCreator - public StorePOP(@JsonProperty("storageengine") String storageEngineName, @JsonProperty("fields") FieldSet fieldSet, @JsonProperty("mode") StoreMode mode, @JsonProperty("entries") List<JSONOptions> entries) { + public StorePOP(FieldSet fieldSet, StoreMode mode, PartitionDef partition, List<T> entries) { super(fieldSet); + this.mode = mode; + this.partition = partition; + this.entries = entries; } public StoreMode getMode() { return mode; } - public PartitionDef getPartitioning() { - return partitioning; + public PartitionDef getPartition() { + return partition; } + public List<T> getEntries(){ + return entries; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockScanPOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockScanPOP.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockScanPOP.java new file mode 100644 index 0000000..f77ac88 --- /dev/null +++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockScanPOP.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical; + +import java.util.List; + +import org.apache.drill.common.physical.pop.ScanPOP; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("mock-scan") +public class MockScanPOP extends ScanPOP<MockScanPOP.MockScanEntry>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockScanPOP.class); + + private final String url; + + @JsonCreator + public MockScanPOP(@JsonProperty("url") String url, @JsonProperty("entries") List<MockScanEntry> readEntries, @JsonProperty("output") FieldSet fields) { + super(readEntries, fields); + this.url = url; + } + + public String getUrl() { + return url; + } + + public static class MockScanEntry implements ReadEntry{ + public int id; + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockStorePOP.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockStorePOP.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockStorePOP.java new file mode 100644 index 0000000..cf2df59 --- /dev/null +++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/physical/MockStorePOP.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.physical; + +import java.util.List; + +import org.apache.drill.common.defs.PartitionDef; +import org.apache.drill.common.physical.MockStorePOP.MockWriteEntry; +import org.apache.drill.common.physical.pop.StorePOP; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("mock-store") +public class MockStorePOP extends StorePOP<MockWriteEntry>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorePOP.class); + + private List<String> fieldNames; + + + @JsonCreator + public MockStorePOP(@JsonProperty("output") FieldSet fields, @JsonProperty("mode") StoreMode mode, @JsonProperty("entries") List<MockWriteEntry> entries, @JsonProperty("partition") PartitionDef partition, @JsonProperty("fieldNames") List<String> fieldNames) { + super(fields, mode, partition, entries); + this.fieldNames = fieldNames; + } + + + public List<String> getFieldNames() { + return fieldNames; + } + + + public static class MockWriteEntry implements WriteEntry{ + public String path; + public String key; + public String type; + } + +} + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/resources/drill-module.conf ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/resources/drill-module.conf b/sandbox/prototype/common/src/test/resources/drill-module.conf index 936ed79..d304d7b 100644 --- a/sandbox/prototype/common/src/test/resources/drill-module.conf +++ b/sandbox/prototype/common/src/test/resources/drill-module.conf @@ -1 +1,2 @@ -drill.logical.storage.packages += "org.apache.drill.storage" \ No newline at end of file +drill.logical.storage.packages += "org.apache.drill.storage" +drill.physical.operator.packages += "org.apache.drill.common.physical" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/resources/dsort-logical.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/resources/dsort-logical.json b/sandbox/prototype/common/src/test/resources/dsort-logical.json new file mode 100644 index 0000000..83d30e8 --- /dev/null +++ b/sandbox/prototype/common/src/test/resources/dsort-logical.json @@ -0,0 +1,40 @@ +{ + head:{ type:"apache_drill_logical_plan", version:"1", generator:{ type:"manual", info:"na"}}}, + storage:[ { type:"fs", name:"fs1", root:"file:///" }], + query:[ { op: "sequence", sequence: [ + { + op:"scan", + storageengine:"fs1", + ref: "data", + selection: { + path: "/sort/unsorted/*.seq", + type: "JAVA_SEQUENCE" + } + }, + { + op: "order", + orderings: [ + {order: "desc", expr: "data.key" } + ] + }, + { + op: "project", + projections: [ + { ref: "output.key", expr: "data.key" }, + { ref: "output.value", expr: "data.value" } + ] + }, + { + op: "store", + storageengine: "fs1", + target: { + path: "/sort/sorted/${partition}.seq", + type: "JAVA_SEQUENCE", + partition: { + type: "ORDERED", + exprs: ["key"] + } + } + } + ]}] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/resources/dsort-physical.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/resources/dsort-physical.json b/sandbox/prototype/common/src/test/resources/dsort-physical.json new file mode 100644 index 0000000..131265a --- /dev/null +++ b/sandbox/prototype/common/src/test/resources/dsort-physical.json @@ -0,0 +1,66 @@ +{ + head:{ + type:"APACHE_DRILL_PHYSICAL", + version:"1", + generator:{ + type:"manual" + } + }, + graph:[ + { + @id:1, + pop:"mock-scan", + url: "http://apache.org";, + entries:[ + {id:1} + ], + output: [ + {mode: "VECTOR", type: "INT32"}, //field 0 + {mode: "VECTOR", type: "BYTES"}, //field 1 + {mode: "VECTOR", type: "BYTES"} //field 2 + ] + }, + { + @id:2, + child: 1, + pop:"project", + fields: [0, 1], + exprs: [], + output: [ + {mode: "VECTOR", type: "INT32"}, //field 0 + {mode: "VECTOR", type: "BYTES"} //field 1 + ] + }, + { + @id:3, + child: 2, + pop:"sort", + field: 0, + output: [ + {mode: "VECTOR", type: "INT32"}, //field 0 + {mode: "VECTOR", type: "BYTES"} //field 1 + ] + }, + { + @id:4, + child:3, + pop: "mock-store", + mode: "SYSTEM_CHOICE", + fieldNames: ["key", "value"], // maps to incoming fieldids + entries:[ + { + path:"/sort/sorted/${partition_number}.seq", + key:"Text", + type:"JAVA_SEQUENCE" + } + ], + output:[ + // the output here is data about the result of the query. + {mode: "VECTOR", type:"INT32"}, + {mode: "VECTOR", type:"INT64"}, + {mode: "VECTOR", type:"INT64"}, + {mode: "VECTOR", type:"INT64"} + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/resources/dsort_logical.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/resources/dsort_logical.json b/sandbox/prototype/common/src/test/resources/dsort_logical.json deleted file mode 100644 index 83d30e8..0000000 --- a/sandbox/prototype/common/src/test/resources/dsort_logical.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - head:{ type:"apache_drill_logical_plan", version:"1", generator:{ type:"manual", info:"na"}}}, - storage:[ { type:"fs", name:"fs1", root:"file:///" }], - query:[ { op: "sequence", sequence: [ - { - op:"scan", - storageengine:"fs1", - ref: "data", - selection: { - path: "/sort/unsorted/*.seq", - type: "JAVA_SEQUENCE" - } - }, - { - op: "order", - orderings: [ - {order: "desc", expr: "data.key" } - ] - }, - { - op: "project", - projections: [ - { ref: "output.key", expr: "data.key" }, - { ref: "output.value", expr: "data.value" } - ] - }, - { - op: "store", - storageengine: "fs1", - target: { - path: "/sort/sorted/${partition}.seq", - type: "JAVA_SEQUENCE", - partition: { - type: "ORDERED", - exprs: ["key"] - } - } - } - ]}] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ede21ff/sandbox/prototype/common/src/test/resources/dsort_physical.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/resources/dsort_physical.json b/sandbox/prototype/common/src/test/resources/dsort_physical.json deleted file mode 100644 index 7c31df2..0000000 --- a/sandbox/prototype/common/src/test/resources/dsort_physical.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - head:{ - type:"APACHE_DRILL_PHYSICAL", - version:"1", - generator:{ - type:"manual" - } - }, - storage:{ - fs1:{ - type:"mock" - } - }, - graph:[ - { - @id:1, - pop:"scan", - storageengine:"fs1", - entries:[{}], - output:[ - { "name":"key", mode: "VECTOR", type:"LATE"}, - { "name":"value", mode: "VECTOR", type:"LATE"} - ] - }, - { - @id:2, - child: 1, - pop:"quicknwaysort", - orderings:[ - { - order: "DESC", - expr: "data.key" - } - ], - output:[ - { "name":"key", mode: "VECTOR", type:"LATE"}, - { "name":"value", mode: "VECTOR", type:"LATE"} - ] - - }, - { - @id:3, - child: 2, - pop:"exchange", - partition:{ - mode:"RANGE", - exprs:["key"] - }, - stitch:{ - mode:"RANDOM" - }, - fields:[ - { "name":"key" mode: "VECTOR", type:"LATE"}, - { "name":"value" mode: "VECTOR", type:"LATE"} - ] - }, - { - @id:4, - child:3, - pop: "store", - mode: "SYSTEM_CHOICE", - storageengine: "fs1", - entries:[ - { - path:"/sort/sorted/${partition_number}.seq", - key:"Text", - type:"JAVA_SEQUENCE" - } - ] - } - ] -} \ No newline at end of file
