Repository: carbondata Updated Branches: refs/heads/carbonstore e4bfb5701 -> 044a995aa
http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/SelectDescriptor.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/SelectDescriptor.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/SelectDescriptor.java deleted file mode 100644 index e10dc84..0000000 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/SelectDescriptor.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.horizon.rest.model.descriptor; - -public class SelectDescriptor { - - private String id; - private String databaseName; - private String tableName; - private String[] projection; - private String filter; - private int limit; - - public SelectDescriptor() { - } - - public SelectDescriptor(String databaseName, String tableName, String[] projection, String filter, - int limit) { - this.databaseName = databaseName; - this.tableName = tableName; - this.projection = projection; - this.filter = filter; - this.limit = limit; - } - - public String getDatabaseName() { - return databaseName; - } - - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String[] getProjection() { - return projection; - } - - public void setProjection(String[] projection) { - this.projection = projection; - } - - public String getFilter() { - return filter; - } - - public void setFilter(String filter) { - this.filter = filter; - } - - public int getLimit() { - return limit; - } - - public void setLimit(int limit) { - this.limit = limit; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/TableDescriptor.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/TableDescriptor.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/TableDescriptor.java deleted file mode 100644 index db1ce98..0000000 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/descriptor/TableDescriptor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.horizon.rest.model.descriptor; - -import java.util.Map; - -import org.apache.carbondata.sdk.file.Schema; - -public class TableDescriptor { - - private boolean ifNotExists; - private String database; - private String name; - private Schema schema; - private Map<String, String> properties; - private String comment; - - public TableDescriptor(boolean ifNotExists, String database, String name, Schema schema, - Map<String, String> properties, String comment) { - this.ifNotExists = ifNotExists; - this.database = database; - this.name = name; - this.schema = schema; - this.properties = properties; - this.comment = comment; - } - - public boolean isIfNotExists() { - return ifNotExists; - } - - public void setIfNotExists(boolean ifNotExists) { - this.ifNotExists = ifNotExists; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Schema getSchema() { - return schema; - } - - public void setSchema(Schema schema) { - this.schema = schema; - } - - public Map<String, String> getProperties() { - return properties; - } - - public void setProperties(Map<String, String> properties) { - this.properties = properties; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java index de82860..fbba57b 100644 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java @@ -18,9 +18,10 @@ package org.apache.carbondata.horizon.rest.model.validate; import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest; +import org.apache.carbondata.horizon.rest.model.view.DropTableRequest; import org.apache.carbondata.horizon.rest.model.view.LoadRequest; import org.apache.carbondata.horizon.rest.model.view.SelectRequest; -import org.apache.carbondata.store.exception.StoreException; +import org.apache.carbondata.store.api.exception.StoreException; import org.apache.commons.lang.StringUtils; @@ -67,4 +68,16 @@ public class RequestValidator { throw new StoreException("input path is invalid"); } } + + public static void validateDrop(DropTableRequest request) throws StoreException { + if (request == null) { + throw new StoreException("DropTableRequest should not be null"); + } + if (StringUtils.isEmpty(request.getDatabaseName())) { + throw new StoreException("database name is invalid"); + } + if (StringUtils.isEmpty(request.getTableName())) { + throw new StoreException("table name is invalid"); + } + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java index 9c81bf0..cf59f7f 100644 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java @@ -22,31 +22,33 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.carbondata.horizon.rest.model.descriptor.TableDescriptor; import org.apache.carbondata.sdk.file.Field; import org.apache.carbondata.sdk.file.Schema; +import org.apache.carbondata.store.api.descriptor.TableDescriptor; +import org.apache.carbondata.store.api.descriptor.TableIdentifier; -public class CreateTableRequest { +public class CreateTableRequest extends Request { private boolean ifNotExists; private String databaseName; private String tableName; + private String tablePath; private FieldRequest[] fields; private Map<String, String> properties; private String comment; public CreateTableRequest() { - } public CreateTableRequest(boolean ifNotExists, String databaseName, String tableName, - FieldRequest[] fields, Map<String, String> properties, String comment) { + FieldRequest[] fields, Map<String, String> properties, String tablePath, String comment) { this.databaseName = databaseName; this.tableName = tableName; this.ifNotExists = ifNotExists; this.fields = fields; this.properties = properties; this.comment = comment; + this.tablePath = tablePath; } public boolean isIfNotExists() { @@ -104,7 +106,8 @@ public class CreateTableRequest { schemaFields[i] = fields[i].convertToDto(); schemaFields[i].setSchemaOrdinal(i); } - return new TableDescriptor(ifNotExists, databaseName, tableName, schema, properties, comment); + return new TableDescriptor(new TableIdentifier(tableName, databaseName), + schema, properties, tablePath, comment, ifNotExists); } public static class Builder { http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/DropTableRequest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/DropTableRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/DropTableRequest.java new file mode 100644 index 0000000..26b084c --- /dev/null +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/DropTableRequest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.horizon.rest.model.view; + +public class DropTableRequest extends Request { + private String databaseName; + private String tableName; + private boolean ifExists; + + public DropTableRequest() { + } + + public DropTableRequest(String databaseName, String tableName, boolean ifExists) { + this.databaseName = databaseName; + this.tableName = tableName; + this.ifExists = ifExists; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public boolean isIfExists() { + return ifExists; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java index a3e9f1c..c91f5f5 100644 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java @@ -20,9 +20,10 @@ package org.apache.carbondata.horizon.rest.model.view; import java.util.HashMap; import java.util.Map; -import org.apache.carbondata.horizon.rest.model.descriptor.LoadDescriptor; +import org.apache.carbondata.store.api.descriptor.LoadDescriptor; +import org.apache.carbondata.store.api.descriptor.TableIdentifier; -public class LoadRequest { +public class LoadRequest extends Request { private String databaseName; private String tableName; @@ -83,7 +84,8 @@ public class LoadRequest { } public LoadDescriptor convertToDto() { - return new LoadDescriptor(databaseName, tableName, inputPath, options, isOverwrite); + return new LoadDescriptor(new TableIdentifier(tableName, databaseName), + inputPath, options, isOverwrite); } public static class Builder { http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Request.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Request.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Request.java new file mode 100644 index 0000000..d922e22 --- /dev/null +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Request.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.horizon.rest.model.view; + +class Request { + private long requestId; + + Request() { + this.requestId = System.nanoTime(); + } + + public long getRequestId() { + return requestId; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Response.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Response.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Response.java new file mode 100644 index 0000000..c114af3 --- /dev/null +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/Response.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.horizon.rest.model.view; + +public class Response { + private long responseId; + + public Response() { + } + + Response(Request request) { + this.responseId = request.getRequestId(); + } + + public long getResponseId() { + return responseId; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectRequest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectRequest.java index 3d5b3df..bb8f382 100644 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectRequest.java +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectRequest.java @@ -17,25 +17,22 @@ package org.apache.carbondata.horizon.rest.model.view; -import org.apache.carbondata.horizon.rest.model.descriptor.SelectDescriptor; - -public class SelectRequest { +public class SelectRequest extends Request { private String databaseName; private String tableName; - private String[] select; + private String[] selectColumns; private String filter; private int limit; public SelectRequest() { - } public SelectRequest(String databaseName, String tableName, String[] select, String filter, int limit) { this.databaseName = databaseName; this.tableName = tableName; - this.select = select; + this.selectColumns = select; this.filter = filter; this.limit = limit; } @@ -57,11 +54,11 @@ public class SelectRequest { } public String[] getSelect() { - return select; + return selectColumns; } - public void setSelect(String[] select) { - this.select = select; + public void setSelect(String[] selectColumns) { + this.selectColumns = selectColumns; } public String getFilter() { @@ -80,11 +77,6 @@ public class SelectRequest { this.limit = limit; } - public SelectDescriptor convertToDto() { - return new SelectDescriptor( - databaseName, tableName, select, filter, limit); - } - public static class Builder { private SelectRequest select; http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java index edf584a..a0f1b55 100644 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java +++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java @@ -17,26 +17,20 @@ package org.apache.carbondata.horizon.rest.model.view; -public class SelectResponse { +public class SelectResponse extends Response { - private String selectId; private Object[][] rows; public SelectResponse() { - } - public SelectResponse(String selectId, Object[][] rows) { - this.selectId = selectId; - this.rows = rows; + public SelectResponse(SelectRequest request) { + super(request); } - public String getSelectId() { - return selectId; - } - - public void setSelectId(String selectId) { - this.selectId = selectId; + public SelectResponse(SelectRequest request, Object[][] rows) { + super(request); + this.rows = rows; } public Object[][] getRows() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/service/HorizonService.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/service/HorizonService.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/service/HorizonService.java deleted file mode 100644 index 7495ca3..0000000 --- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/service/HorizonService.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.horizon.rest.service; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; -import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.datastore.row.CarbonRow; -import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry; -import org.apache.carbondata.core.metadata.schema.table.CarbonTable; -import org.apache.carbondata.core.metadata.schema.table.TableInfo; -import org.apache.carbondata.core.metadata.schema.table.TableSchema; -import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder; -import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; -import org.apache.carbondata.core.scan.expression.Expression; -import org.apache.carbondata.horizon.antlr.ANTLRNoCaseStringStream; -import org.apache.carbondata.horizon.antlr.FilterVisitor; -import org.apache.carbondata.horizon.antlr.gen.ExpressionLexer; -import org.apache.carbondata.horizon.antlr.gen.ExpressionParser; -import org.apache.carbondata.horizon.rest.model.descriptor.LoadDescriptor; -import org.apache.carbondata.horizon.rest.model.descriptor.SelectDescriptor; -import org.apache.carbondata.horizon.rest.model.descriptor.TableDescriptor; -import org.apache.carbondata.processing.loading.model.CarbonLoadModel; -import org.apache.carbondata.processing.loading.model.CarbonLoadModelBuilder; -import org.apache.carbondata.sdk.file.CarbonWriterBuilder; -import org.apache.carbondata.sdk.file.Field; -import org.apache.carbondata.store.exception.StoreException; -import org.apache.carbondata.store.master.Master; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenStream; - -public class HorizonService { - - private static LogService LOGGER = - LogServiceFactory.getLogService(HorizonService.class.getName()); - - private static HorizonService instance; - - private Master master; - - private HorizonService() { - master = Master.getInstance(null); - } - - public boolean createTable(TableDescriptor tableDescriptor) throws StoreException { - TableSchemaBuilder builder = TableSchema.builder(); - builder.tableName(tableDescriptor.getName()).properties(tableDescriptor.getProperties()); - - Field[] fields = tableDescriptor.getSchema().getFields(); - // sort_columns - List<String> sortColumnsList = null; - try { - sortColumnsList = - tableDescriptor.getSchema().prepareSortColumns(tableDescriptor.getProperties()); - } catch (MalformedCarbonCommandException e) { - throw new StoreException(e.getMessage()); - } - ColumnSchema[] sortColumnsSchemaList = new ColumnSchema[sortColumnsList.size()]; - // tableDescriptor schema - CarbonWriterBuilder.buildTableSchema(fields, builder, sortColumnsList, sortColumnsSchemaList); - builder.setSortColumns(Arrays.asList(sortColumnsSchemaList)); - - TableSchema schema = builder.build(); - SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry(); - schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis()); - schema.getSchemaEvolution().getSchemaEvolutionEntryList().add(schemaEvolutionEntry); - schema.setTableName(tableDescriptor.getName()); - - TableInfo tableInfo = CarbonTable - .builder() - .databaseName(tableDescriptor.getDatabase()) - .tableName(tableDescriptor.getName()) - .tablePath( - master.getTableFolder(tableDescriptor.getDatabase(), tableDescriptor.getName())) - .tableSchema(schema) - .isTransactionalTable(true) - .buildTableInfo(); - - try { - return master.createTable(tableInfo, tableDescriptor.isIfNotExists()); - } catch (IOException e) { - LOGGER.error(e, "create tableDescriptor failed"); - throw new StoreException(e.getMessage()); - } - } - - public boolean loadData(LoadDescriptor loadDescriptor) throws StoreException { - CarbonTable table = master.getTable( - loadDescriptor.getDatabaseName(), loadDescriptor.getTableName()); - CarbonLoadModelBuilder modelBuilder = new CarbonLoadModelBuilder(table); - modelBuilder.setInputPath(loadDescriptor.getInputPath()); - try { - CarbonLoadModel model = - modelBuilder.build(loadDescriptor.getOptions(), System.currentTimeMillis(), "0"); - - return master.loadData(model, loadDescriptor.isOverwrite()); - } catch (InvalidLoadOptionException e) { - LOGGER.error(e, "Invalid loadDescriptor options"); - throw new StoreException(e.getMessage()); - } catch (IOException e) { - LOGGER.error(e, "Failed to loadDescriptor data"); - throw new StoreException(e.getMessage()); - } - } - - public CarbonRow[] select(SelectDescriptor selectDescriptor) throws StoreException { - try { - CarbonTable carbonTable = master.getTable( - selectDescriptor.getDatabaseName(), selectDescriptor.getTableName()); - return master.search( - carbonTable, - selectDescriptor.getProjection(), - parseFilter(selectDescriptor.getFilter(), carbonTable), - selectDescriptor.getLimit(), - selectDescriptor.getLimit()); - } catch (IOException e) { - LOGGER.error(e, "[" + selectDescriptor.getId() + "] select failed"); - throw new StoreException(e.getMessage()); - } - } - - public static Expression parseFilter(String filter, CarbonTable carbonTable) { - if (filter == null) { - return null; - } - CharStream input = new ANTLRNoCaseStringStream(filter); - ExpressionLexer lexer = new ExpressionLexer(input); - CommonTokenStream tokens = new CommonTokenStream(lexer); - ExpressionParser parser = new ExpressionParser(tokens); - ExpressionParser.ParseFilterContext tree = parser.parseFilter(); - FilterVisitor visitor = new FilterVisitor(carbonTable); - return visitor.visitParseFilter(tree); - } - - public static synchronized HorizonService getInstance() { - if (instance == null) { - instance = new HorizonService(); - } - return instance; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/test/java/org/apache/carbondata/horizon/FilterParseTest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/test/java/org/apache/carbondata/horizon/FilterParseTest.java b/store/horizon/src/test/java/org/apache/carbondata/horizon/FilterParseTest.java index 72ea0a7..c40d62c 100644 --- a/store/horizon/src/test/java/org/apache/carbondata/horizon/FilterParseTest.java +++ b/store/horizon/src/test/java/org/apache/carbondata/horizon/FilterParseTest.java @@ -28,11 +28,11 @@ import org.apache.carbondata.core.metadata.schema.table.TableSchema; import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.core.scan.expression.Expression; -import org.apache.carbondata.horizon.rest.model.descriptor.TableDescriptor; -import org.apache.carbondata.horizon.rest.service.HorizonService; import org.apache.carbondata.sdk.file.CarbonWriterBuilder; import org.apache.carbondata.sdk.file.Field; import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest; +import org.apache.carbondata.store.api.descriptor.TableDescriptor; +import org.apache.carbondata.horizon.antlr.Parser; import org.junit.Assert; import org.junit.BeforeClass; @@ -66,8 +66,10 @@ public class FilterParseTest { TableDescriptor tableDescriptor = createTableRequest.convertToDto(); - TableSchemaBuilder builder = TableSchema.builder(); - builder.tableName(tableDescriptor.getName()).properties(tableDescriptor.getProperties()); + TableSchemaBuilder builder = TableSchema + .builder() + .tableName(tableDescriptor.getTable().getTableName()) + .properties(tableDescriptor.getProperties()); Field[] fields = tableDescriptor.getSchema().getFields(); // sort_columns @@ -82,12 +84,12 @@ public class FilterParseTest { SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry(); schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis()); schema.getSchemaEvolution().getSchemaEvolutionEntryList().add(schemaEvolutionEntry); - schema.setTableName(tableDescriptor.getName()); + schema.setTableName(tableDescriptor.getTable().getTableName()); carbonTable = CarbonTable .builder() - .databaseName(tableDescriptor.getDatabase()) - .tableName(tableDescriptor.getName()) + .databaseName(tableDescriptor.getTable().getDatabaseName()) + .tableName(tableDescriptor.getTable().getTableName()) .tablePath("") .tableSchema(schema) .isTransactionalTable(true) @@ -96,7 +98,7 @@ public class FilterParseTest { private void checkExpression(String sql1, String sql2) { - Expression expression = HorizonService.parseFilter(sql1, carbonTable); + Expression expression = Parser.parseFilter(sql1, carbonTable); Assert.assertEquals(sql2, expression.getStatement()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java ---------------------------------------------------------------------- diff --git a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java b/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java index e57e813..8adfd38 100644 --- a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java +++ b/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java @@ -19,17 +19,22 @@ package org.apache.carbondata.horizon; import java.io.File; import java.io.IOException; +import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.store.conf.StoreConf; +import org.apache.carbondata.core.datastore.row.CarbonRow; +import org.apache.carbondata.horizon.rest.client.HorizonClient; +import org.apache.carbondata.horizon.rest.client.impl.SimpleHorizonClient; import org.apache.carbondata.horizon.rest.controller.Horizon; -import org.apache.carbondata.store.master.Master; +import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest; +import org.apache.carbondata.horizon.rest.model.view.DropTableRequest; import org.apache.carbondata.horizon.rest.model.view.LoadRequest; import org.apache.carbondata.horizon.rest.model.view.SelectRequest; import org.apache.carbondata.horizon.rest.model.view.SelectResponse; -import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest; +import org.apache.carbondata.store.api.conf.StoreConf; +import org.apache.carbondata.store.api.exception.StoreException; +import org.apache.carbondata.store.impl.worker.Worker; import org.apache.carbondata.store.util.StoreUtil; -import org.apache.carbondata.store.worker.Worker; import org.junit.AfterClass; import org.junit.Assert; @@ -39,7 +44,6 @@ import org.springframework.web.client.RestTemplate; public class HorizonTest { - private static Master master; private static Worker worker; private static String serviceUri = "http://localhost:8080";; private static String projectFolder; @@ -54,6 +58,7 @@ public class HorizonTest { String confFile = projectFolder + "/store/conf/store.conf"; System.setProperty("log.path", projectFolder + "/store/core/target/master_worker.log"); + System.setProperty("carbonstore.conf.file", confFile); StoreUtil.initLog4j(log4jFile); StoreConf storeConf = new StoreConf(confFile); @@ -61,13 +66,9 @@ public class HorizonTest { StoreConf.STORE_LOCATION, storeConf.storeLocation() + System.currentTimeMillis()); - // start master - master = Master.getInstance(storeConf); - master.startService(); - new Thread() { public void run() { - Horizon.main(new String[0]); + Horizon.start(new String[0]); } }.start(); Thread.sleep(10000); @@ -79,75 +80,123 @@ public class HorizonTest { restTemplate = new RestTemplate(); } + @AfterClass + public static void shutdown() { + worker.stop(); + Horizon.stop(); + } + @Test public void testHorizon() { + DropTableRequest request = createDropTableRequest(); + String response = + restTemplate.postForObject(serviceUri + "/table/drop", request, String.class); + Assert.assertEquals(true, Boolean.valueOf(response)); + // create table if not exists - CreateTableRequest table = CreateTableRequest - .builder() - .ifNotExists() - .databaseName("default") - .tableName("table_1") - .comment("first table") - .column("shortField", "SHORT", "short field") - .column("intField", "INT", "int field") - .column("bigintField", "LONG", "long field") - .column("doubleField", "DOUBLE", "double field") - .column("stringField", "STRING", "string field") - .column("timestampField", "TIMESTAMP", "timestamp field") - .column("decimalField", "DECIMAL", 18, 2, "decimal field") - .column("dateField", "DATE", "date field") - .column("charField", "CHAR", "char field") - .column("floatField", "FLOAT", "float field") - .tblProperties(CarbonCommonConstants.SORT_COLUMNS, "intField") - .create(); + CreateTableRequest table = createCreateTableRequest(); String createTable = restTemplate.postForObject(serviceUri + "/table/create", table, String.class); Assert.assertEquals(true, Boolean.valueOf(createTable)); // load one segment - LoadRequest load = LoadRequest - .builder() - .databaseName("default") - .tableName("table_1") - .overwrite(false) - .inputPath(projectFolder + "/store/horizon/src/test/resources/data1.csv") - .options("header", "true") - .create(); + LoadRequest load = createLoadRequest(); String loadData = restTemplate.postForObject(serviceUri + "/table/load", load, String.class); Assert.assertEquals(true, Boolean.valueOf(loadData)); // select row - SelectRequest select = SelectRequest - .builder() - .databaseName("default") - .tableName("table_1") - .select("intField", "stringField") - .limit(5) - .create(); + SelectRequest select = createSelectRequest(5, null, "intField", "stringField"); SelectResponse result = restTemplate.postForObject(serviceUri + "/table/select", select, SelectResponse.class); Assert.assertEquals(5, result.getRows().length); // select row with filter - SelectRequest filter = SelectRequest - .builder() - .databaseName("default") - .tableName("table_1") - .select("intField", "stringField") - .filter("intField = 11") - .limit(5) - .create(); - SelectResponse fitlerResult = + SelectRequest filter = createSelectRequest(5, "intField = 11", "intField", "stringField"); + SelectResponse filterResult = restTemplate.postForObject(serviceUri + "/table/select", filter, SelectResponse.class); - Assert.assertEquals(1, fitlerResult.getRows().length); + Assert.assertEquals(1, filterResult.getRows().length); + + request = createDropTableRequest(); + response = restTemplate.postForObject(serviceUri + "/table/drop", request, String.class); + Assert.assertEquals(true, Boolean.valueOf(response)); + } - @AfterClass - public static void release() throws InterruptedException { - worker.stop(); - Horizon.close(); - master.stopService(); + private DropTableRequest createDropTableRequest() { + return new DropTableRequest("default", "table_1", false); } + private SelectRequest createSelectRequest(int limit, String filter, String... select) { + SelectRequest.Builder builder = SelectRequest + .builder() + .databaseName("default") + .tableName("table_1") + .select(select) + .limit(limit); + if (filter != null) { + builder = builder.filter(filter); + } + return builder.create(); + } + + private LoadRequest createLoadRequest() { + return LoadRequest + .builder() + .databaseName("default") + .tableName("table_1") + .overwrite(false) + .inputPath(projectFolder + "/store/core/src/test/resources/data1.csv") + .options("header", "true") + .create(); + } + + private CreateTableRequest createCreateTableRequest() { + return CreateTableRequest + .builder() + .ifNotExists() + .databaseName("default") + .tableName("table_1") + .comment("first table") + .column("shortField", "SHORT", "short field") + .column("intField", "INT", "int field") + .column("bigintField", "LONG", "long field") + .column("doubleField", "DOUBLE", "double field") + .column("stringField", "STRING", "string field") + .column("timestampField", "TIMESTAMP", "timestamp field") + .column("decimalField", "DECIMAL", 18, 2, "decimal field") + .column("dateField", "DATE", "date field") + .column("charField", "CHAR", "char field") + .column("floatField", "FLOAT", "float field") + .tblProperties(CarbonCommonConstants.SORT_COLUMNS, "intField") + .create(); + } + + @Test + public void testHorizonClient() throws IOException, StoreException { + HorizonClient client = new SimpleHorizonClient(serviceUri); + DropTableRequest drop = createDropTableRequest(); + client.dropTable(drop); + + // create table if not exists + CreateTableRequest create = createCreateTableRequest(); + client.createTable(create); + + // load one segment + LoadRequest load = createLoadRequest(); + client.loadData(load); + + // select row + SelectRequest select = createSelectRequest(5, null, "intField", "stringField"); + List<CarbonRow> result = client.select(select); + Assert.assertEquals(5, result.size()); + + // select row with filter + SelectRequest filter = createSelectRequest(5, "intField = 11", "intField", "stringField"); + List<CarbonRow> filterResult = client.select(filter); + Assert.assertEquals(5, result.size()); Assert.assertEquals(1, filterResult.size()); + + drop = createDropTableRequest(); + client.dropTable(drop); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/horizon/src/test/resources/data1.csv ---------------------------------------------------------------------- diff --git a/store/horizon/src/test/resources/data1.csv b/store/horizon/src/test/resources/data1.csv deleted file mode 100644 index cf732eb..0000000 --- a/store/horizon/src/test/resources/data1.csv +++ /dev/null @@ -1,11 +0,0 @@ -shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField -1,10,1100,48.4,spark,2015-4-23 12:01:01,1.23,2015-4-23,aaa,2.5 -5,17,1140,43.4,spark,2015-7-27 12:01:02,3.45,2015-7-27,bbb,2.5 -1,11,1100,44.4,flink,2015-5-23 12:01:03,23.23,2015-5-23,ccc,2.5 -1,10,1150,43.4,spark,2015-7-24 12:01:04,254.12,2015-7-24,ddd,2.5 -1,10,1100,47.4,spark,2015-7-23 12:01:05,876.14,2015-7-23,eeee,3.5 -3,14,1160,43.4,hive,2015-7-26 12:01:06,3454.32,2015-7-26,ff,2.5 -2,10,1100,43.4,impala,2015-7-23 12:01:07,456.98,2015-7-23,ggg,2.5 -1,10,1100,43.4,spark,2015-5-23 12:01:08,32.53,2015-5-23,hhh,2.5 -4,16,1130,42.4,impala,2015-7-23 12:01:09,67.23,2015-7-23,iii,2.5 -1,10,1100,43.4,spark,2015-7-23 12:01:10,832.23,2015-7-23,jjj,2.5 http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index a4ca510..f94730a 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -415,10 +415,13 @@ public class CarbonWriterBuilder { } TableSchema schema = tableSchemaBuilder.build(); schema.setTableName(tableName); - CarbonTable table = - CarbonTable.builder().tableName(schema.getTableName()).databaseName(dbName).tablePath(path) - .tableSchema(schema).isTransactionalTable(isTransactionalTable).build(); - return table; + return CarbonTable.builder() + .tableName(schema.getTableName()) + .databaseName(dbName) + .tablePath(path) + .tableSchema(schema) + .isTransactionalTable(isTransactionalTable) + .build(); } public static void buildTableSchema(Field[] fields, TableSchemaBuilder tableSchemaBuilder, http://git-wip-us.apache.org/repos/asf/carbondata/blob/044a995a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java index 6d4cfd9..add10c1 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java @@ -23,9 +23,9 @@ import java.util.List; import org.apache.carbondata.common.annotations.InterfaceAudience; import org.apache.carbondata.common.annotations.InterfaceStability; import org.apache.carbondata.core.metadata.datatype.DataType; -import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.datatype.StructField; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.util.DataTypeUtil; /** * A field represent one column @@ -51,72 +51,13 @@ public class Field { * @param type datatype of field, specified in strings. */ public Field(String name, String type) { - this.name = name; - if (type.equalsIgnoreCase("string")) { - this.type = DataTypes.STRING; - } else if (type.equalsIgnoreCase("date")) { - this.type = DataTypes.DATE; - } else if (type.equalsIgnoreCase("timestamp")) { - this.type = DataTypes.TIMESTAMP; - } else if (type.equalsIgnoreCase("boolean")) { - this.type = DataTypes.BOOLEAN; - } else if (type.equalsIgnoreCase("byte")) { - this.type = DataTypes.BYTE; - } else if (type.equalsIgnoreCase("short")) { - this.type = DataTypes.SHORT; - } else if (type.equalsIgnoreCase("int")) { - this.type = DataTypes.INT; - } else if (type.equalsIgnoreCase("long")) { - this.type = DataTypes.LONG; - } else if (type.equalsIgnoreCase("float")) { - this.type = DataTypes.FLOAT; - } else if (type.equalsIgnoreCase("double")) { - this.type = DataTypes.DOUBLE; - } else if (type.equalsIgnoreCase("array")) { - this.type = DataTypes.createDefaultArrayType(); - } else if (type.equalsIgnoreCase("struct")) { - this.type = DataTypes.createDefaultStructType(); - } - else { - throw new IllegalArgumentException("unsupported data type: " + type); - } + this(name, DataTypeUtil.valueOf(type)); } public Field(String name, String type, List<StructField> fields) { - this.name = name; - this.children = fields; - if (type.equalsIgnoreCase("string")) { - this.type = DataTypes.STRING; - } else if (type.equalsIgnoreCase("date")) { - this.type = DataTypes.DATE; - } else if (type.equalsIgnoreCase("timestamp")) { - this.type = DataTypes.TIMESTAMP; - } else if (type.equalsIgnoreCase("boolean")) { - this.type = DataTypes.BOOLEAN; - } else if (type.equalsIgnoreCase("byte")) { - this.type = DataTypes.BYTE; - } else if (type.equalsIgnoreCase("short")) { - this.type = DataTypes.SHORT; - } else if (type.equalsIgnoreCase("int")) { - this.type = DataTypes.INT; - } else if (type.equalsIgnoreCase("long")) { - this.type = DataTypes.LONG; - } else if (type.equalsIgnoreCase("float")) { - this.type = DataTypes.FLOAT; - } else if (type.equalsIgnoreCase("double")) { - this.type = DataTypes.DOUBLE; - } else if (type.equalsIgnoreCase("array")) { - this.type = DataTypes.createArrayType(fields.get(0).getDataType()); - } else if (type.equalsIgnoreCase("struct")) { - this.type = DataTypes.createStructType(fields); - } - else { - throw new IllegalArgumentException("unsupported data type: " + type); - } + this(name, DataTypeUtil.valueOf(type), fields); } - - public Field(String name, DataType type, List<StructField> fields) { this.name = name; this.type = type;
