Repository: metamodel Updated Branches: refs/heads/master c57d50805 -> bda8d764f
http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchDataContextTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchDataContextTest.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchDataContextTest.java deleted file mode 100644 index 53dbdf6..0000000 --- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchDataContextTest.java +++ /dev/null @@ -1,615 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.elasticsearch.rest; - -import io.searchbox.client.JestClient; -import io.searchbox.client.JestClientFactory; -import io.searchbox.client.config.HttpClientConfig; -import org.apache.metamodel.MetaModelHelper; -import org.apache.metamodel.UpdateCallback; -import org.apache.metamodel.UpdateScript; -import org.apache.metamodel.UpdateableDataContext; -import org.apache.metamodel.create.CreateTable; -import org.apache.metamodel.data.DataSet; -import org.apache.metamodel.data.DataSetTableModel; -import org.apache.metamodel.data.InMemoryDataSet; -import org.apache.metamodel.data.Row; -import org.apache.metamodel.delete.DeleteFrom; -import org.apache.metamodel.drop.DropTable; -import org.apache.metamodel.elasticsearch.rest.utils.EmbeddedElasticsearchServer; -import org.apache.metamodel.query.FunctionType; -import org.apache.metamodel.query.Query; -import org.apache.metamodel.query.SelectItem; -import org.apache.metamodel.query.parser.QueryParserException; -import org.apache.metamodel.schema.Column; -import org.apache.metamodel.schema.ColumnType; -import org.apache.metamodel.schema.Schema; -import org.apache.metamodel.schema.Table; -import org.apache.metamodel.update.Update; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import javax.swing.table.TableModel; -import java.io.IOException; -import java.util.*; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.*; - -public class JestElasticSearchDataContextTest { - - private static final String indexName = "twitter"; - private static final String indexType1 = "tweet1"; - private static final String indexType2 = "tweet2"; - private static final String indexName2 = "twitter2"; - private static final String indexType3 = "tweet3"; - private static final String bulkIndexType = "bulktype"; - private static final String peopleIndexType = "peopletype"; - private static final String mapping = - "{\"date_detection\":\"false\",\"properties\":{\"message\":{\"type\":\"string\",\"index\":\"not_analyzed\",\"doc_values\":\"true\"}}}"; - private static EmbeddedElasticsearchServer embeddedElasticsearchServer; - private static JestClient client; - private static UpdateableDataContext dataContext; - - @BeforeClass - public static void beforeTests() throws Exception { - embeddedElasticsearchServer = new EmbeddedElasticsearchServer(); - final int port = Integer.parseInt(embeddedElasticsearchServer.getClient().settings().get("http.port")); - JestClientFactory factory = new JestClientFactory(); - factory.setHttpClientConfig(new HttpClientConfig - .Builder("http://localhost:"; + port) - .multiThreaded(true) - .build()); - client = factory.getObject(); - - indexTweeterDocument(indexType1, 1); - indexTweeterDocument(indexType2, 1); - indexTweeterDocument(indexType2, 2, null); - insertPeopleDocuments(); - indexTweeterDocument(indexType2, 1); - indexBulkDocuments(indexName, bulkIndexType, 10); - - // The refresh API allows to explicitly refresh one or more index, - // making all operations performed since the last refresh available for - // search - dataContext = new ElasticSearchRestDataContext(client, indexName); - Thread.sleep(1000); - System.out.println("Embedded ElasticSearch server created!"); - } - - private static void insertPeopleDocuments() throws IOException { - indexOnePeopleDocument("female", 20, 5); - indexOnePeopleDocument("female", 17, 8); - indexOnePeopleDocument("female", 18, 9); - indexOnePeopleDocument("female", 19, 10); - indexOnePeopleDocument("female", 20, 11); - indexOnePeopleDocument("male", 19, 1); - indexOnePeopleDocument("male", 17, 2); - indexOnePeopleDocument("male", 18, 3); - indexOnePeopleDocument("male", 18, 4); - } - - @AfterClass - public static void afterTests() { - embeddedElasticsearchServer.shutdown(); - System.out.println("Embedded ElasticSearch server shut down!"); - } - - @Test - public void testSimpleQuery() throws Exception { - assertEquals("[bulktype, peopletype, tweet1, tweet2]", - Arrays.toString(dataContext.getDefaultSchema().getTableNames().toArray())); - - Table table = dataContext.getDefaultSchema().getTableByName("tweet1"); - - assertEquals("[_id, message, postDate, user]", Arrays.toString(table.getColumnNames().toArray())); - - assertEquals(ColumnType.STRING, table.getColumnByName("user").getType()); - assertEquals(ColumnType.DATE, table.getColumnByName("postDate").getType()); - assertEquals(ColumnType.BIGINT, table.getColumnByName("message").getType()); - - try (DataSet ds = dataContext.query().from(indexType1).select("user").and("message").execute()) { - assertEquals(JestElasticSearchDataSet.class, ds.getClass()); - - assertTrue(ds.next()); - assertEquals("Row[values=[user1, 1]]", ds.getRow().toString()); - } - } - - @Test - public void testDocumentIdAsPrimaryKey() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName("tweet2"); - Column[] pks = table.getPrimaryKeys().toArray(new Column[0]); - assertEquals(1, pks.length); - assertEquals("_id", pks[0].getName()); - - try (DataSet ds = dataContext.query().from(table).select("user", "_id").orderBy("_id").asc().execute()) { - assertTrue(ds.next()); - assertEquals("Row[values=[user1, tweet_tweet2_1]]", ds.getRow().toString()); - } - } - - @Test - public void testExecutePrimaryKeyLookupQuery() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName("tweet2"); - Column[] pks = table.getPrimaryKeys().toArray(new Column[0]); - - try (DataSet ds = dataContext.query().from(table).selectAll().where(pks[0]).eq("tweet_tweet2_1").execute()) { - assertTrue(ds.next()); - Object dateValue = ds.getRow().getValue(2); - assertEquals("Row[values=[tweet_tweet2_1, 1, " + dateValue + ", user1]]", ds.getRow().toString()); - - assertFalse(ds.next()); - - assertEquals(InMemoryDataSet.class, ds.getClass()); - } - } - - @Test - public void testDateIsHandledAsDate() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName("tweet1"); - Column column = table.getColumnByName("postDate"); - ColumnType type = column.getType(); - assertEquals(ColumnType.DATE, type); - - DataSet dataSet = dataContext.query().from(table).select(column).execute(); - while (dataSet.next()) { - Object value = dataSet.getRow().getValue(column); - assertTrue("Got class: " + value.getClass() + ", expected Date (or subclass)", value instanceof Date); - } - } - - @Test - public void testNumberIsHandledAsNumber() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName(peopleIndexType); - Column column = table.getColumnByName("age"); - ColumnType type = column.getType(); - assertEquals(ColumnType.BIGINT, type); - - DataSet dataSet = dataContext.query().from(table).select(column).execute(); - while (dataSet.next()) { - Object value = dataSet.getRow().getValue(column); - assertTrue("Got class: " + value.getClass() + ", expected Number (or subclass)", value instanceof Number); - } - } - - @Test - public void testCreateTableInsertQueryAndDrop() throws Exception { - final Schema schema = dataContext.getDefaultSchema(); - final CreateTable createTable = new CreateTable(schema, "testCreateTable"); - createTable.withColumn("foo").ofType(ColumnType.STRING); - createTable.withColumn("bar").ofType(ColumnType.NUMBER); - dataContext.executeUpdate(createTable); - - final Table table = schema.getTableByName("testCreateTable"); - assertNotNull(table); - assertEquals("[" + ElasticSearchRestDataContext.FIELD_ID + ", foo, bar]", Arrays.toString(table.getColumnNames().toArray())); - - final Column fooColumn = table.getColumnByName("foo"); - final Column idColumn = table.getPrimaryKeys().get(0); - assertEquals("Column[name=_id,columnNumber=0,type=STRING,nullable=null,nativeType=null,columnSize=null]", - idColumn.toString()); - - dataContext.executeUpdate(new UpdateScript() { - @Override - public void run(UpdateCallback callback) { - callback.insertInto(table).value("foo", "hello").value("bar", 42).execute(); - callback.insertInto(table).value("foo", "world").value("bar", 43).execute(); - } - }); - - dataContext.refreshSchemas(); - - - try (DataSet ds = dataContext.query().from(table).selectAll().orderBy("bar").execute()) { - assertTrue(ds.next()); - assertEquals("hello", ds.getRow().getValue(fooColumn).toString()); - assertNotNull(ds.getRow().getValue(idColumn)); - assertTrue(ds.next()); - assertEquals("world", ds.getRow().getValue(fooColumn).toString()); - assertNotNull(ds.getRow().getValue(idColumn)); - assertFalse(ds.next()); - } - - dataContext.executeUpdate(new DropTable(table)); - - dataContext.refreshSchemas(); - - assertNull(dataContext.getTableByQualifiedLabel(table.getName())); - } - - @Test - public void testDetectOutsideChanges() throws Exception { - // Create the type in ES - final IndicesAdminClient indicesAdmin = embeddedElasticsearchServer.getClient().admin().indices(); - final String tableType = "outsideTable"; - - Object[] sourceProperties = { "testA", "type=string, store=true", "testB", "type=string, store=true" }; - - new PutMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).setSource(sourceProperties) - .execute().actionGet(); - - dataContext.refreshSchemas(); - - assertNotNull(dataContext.getDefaultSchema().getTableByName(tableType)); - - new DeleteMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).execute().actionGet(); - dataContext.refreshSchemas(); - assertNull(dataContext.getTableByQualifiedLabel(tableType)); - } - - @Test - public void testDeleteAll() throws Exception { - final Schema schema = dataContext.getDefaultSchema(); - final CreateTable createTable = new CreateTable(schema, "testCreateTable"); - createTable.withColumn("foo").ofType(ColumnType.STRING); - createTable.withColumn("bar").ofType(ColumnType.NUMBER); - dataContext.executeUpdate(createTable); - - final Table table = schema.getTableByName("testCreateTable"); - - dataContext.executeUpdate(new UpdateScript() { - @Override - public void run(UpdateCallback callback) { - callback.insertInto(table).value("foo", "hello").value("bar", 42).execute(); - callback.insertInto(table).value("foo", "world").value("bar", 43).execute(); - } - }); - - dataContext.executeUpdate(new DeleteFrom(table)); - - Row row = MetaModelHelper.executeSingleRowQuery(dataContext, dataContext.query().from(table).selectCount() - .toQuery()); - assertEquals("Count is wrong", 0, ((Number) row.getValue(0)).intValue()); - - dataContext.executeUpdate(new DropTable(table)); - } - - @Test - public void testDeleteByQuery() throws Exception { - final Schema schema = dataContext.getDefaultSchema(); - final CreateTable createTable = new CreateTable(schema, "testCreateTable"); - createTable.withColumn("foo").ofType(ColumnType.STRING); - createTable.withColumn("bar").ofType(ColumnType.NUMBER); - dataContext.executeUpdate(createTable); - - final Table table = schema.getTableByName("testCreateTable"); - - dataContext.executeUpdate(new UpdateScript() { - @Override - public void run(UpdateCallback callback) { - callback.insertInto(table).value("foo", "hello").value("bar", 42).execute(); - callback.insertInto(table).value("foo", "world").value("bar", 43).execute(); - } - }); - - dataContext.executeUpdate(new DeleteFrom(table).where("foo").eq("hello").where("bar").eq(42)); - - Row row = MetaModelHelper.executeSingleRowQuery(dataContext, - dataContext.query().from(table).select("foo", "bar").toQuery()); - assertEquals("Row[values=[world, 43]]", row.toString()); - - dataContext.executeUpdate(new DropTable(table)); - } - - @Test - public void testDeleteUnsupportedQueryType() throws Exception { - final Schema schema = dataContext.getDefaultSchema(); - final CreateTable createTable = new CreateTable(schema, "testCreateTable"); - createTable.withColumn("foo").ofType(ColumnType.STRING); - createTable.withColumn("bar").ofType(ColumnType.NUMBER); - dataContext.executeUpdate(createTable); - - final Table table = schema.getTableByName("testCreateTable"); - try { - - dataContext.executeUpdate(new UpdateScript() { - @Override - public void run(UpdateCallback callback) { - callback.insertInto(table).value("foo", "hello").value("bar", 42).execute(); - callback.insertInto(table).value("foo", "world").value("bar", 43).execute(); - } - }); - - // greater than is not yet supported - try { - dataContext.executeUpdate(new DeleteFrom(table).where("bar").gt(40)); - fail("Exception expected"); - } catch (UnsupportedOperationException e) { - assertEquals("Could not push down WHERE items to delete by query request: [testCreateTable.bar > 40]", - e.getMessage()); - } - - } finally { - dataContext.executeUpdate(new DropTable(table)); - } - } - - @Test - public void testUpdateRow() throws Exception { - final Schema schema = dataContext.getDefaultSchema(); - final CreateTable createTable = new CreateTable(schema, "testCreateTable"); - createTable.withColumn("foo").ofType(ColumnType.STRING); - createTable.withColumn("bar").ofType(ColumnType.NUMBER); - dataContext.executeUpdate(createTable); - - final Table table = schema.getTableByName("testCreateTable"); - try { - - dataContext.executeUpdate(new UpdateScript() { - @Override - public void run(UpdateCallback callback) { - callback.insertInto(table).value("foo", "hello").value("bar", 42).execute(); - callback.insertInto(table).value("foo", "world").value("bar", 43).execute(); - } - }); - - dataContext.executeUpdate(new Update(table).value("foo", "howdy").where("bar").eq(42)); - - DataSet dataSet = dataContext.query().from(table).select("foo", "bar").orderBy("bar").execute(); - assertTrue(dataSet.next()); - assertEquals("Row[values=[howdy, 42]]", dataSet.getRow().toString()); - assertTrue(dataSet.next()); - assertEquals("Row[values=[world, 43]]", dataSet.getRow().toString()); - assertFalse(dataSet.next()); - dataSet.close(); - } finally { - dataContext.executeUpdate(new DropTable(table)); - } - } - - @Test - public void testDropTable() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName(peopleIndexType); - - // assert that the table was there to begin with - { - DataSet ds = dataContext.query().from(table).selectCount().execute(); - ds.next(); - assertEquals("Count is wrong", 9, ((Number) ds.getRow().getValue(0)).intValue()); - ds.close(); - } - - dataContext.executeUpdate(new DropTable(table)); - try { - DataSet ds = dataContext.query().from(table).selectCount().execute(); - ds.next(); - assertEquals("Count is wrong", 0, ((Number) ds.getRow().getValue(0)).intValue()); - ds.close(); - } finally { - // restore the people documents for the next tests - insertPeopleDocuments(); - embeddedElasticsearchServer.getClient().admin().indices().prepareRefresh().execute().actionGet(); - dataContext = new ElasticSearchRestDataContext(client, indexName); - } - } - - @Test - public void testWhereColumnEqualsValues() throws Exception { - try (DataSet ds = dataContext.query().from(bulkIndexType).select("user").and("message").where("user") - .isEquals("user4").execute()) { - assertEquals(JestElasticSearchDataSet.class, ds.getClass()); - - assertTrue(ds.next()); - assertEquals("Row[values=[user4, 4]]", ds.getRow().toString()); - assertFalse(ds.next()); - } - } - - @Test - public void testWhereColumnIsNullValues() throws Exception { - try (DataSet ds = dataContext.query().from(indexType2).select("message").where("postDate") - .isNull().execute()) { - assertEquals(JestElasticSearchDataSet.class, ds.getClass()); - - assertTrue(ds.next()); - assertEquals("Row[values=[2]]", ds.getRow().toString()); - assertFalse(ds.next()); - } - } - - @Test - public void testWhereColumnIsNotNullValues() throws Exception { - try (DataSet ds = dataContext.query().from(indexType2).select("message").where("postDate") - .isNotNull().execute()) { - assertEquals(JestElasticSearchDataSet.class, ds.getClass()); - - assertTrue(ds.next()); - assertEquals("Row[values=[1]]", ds.getRow().toString()); - assertFalse(ds.next()); - } - } - - @Test - public void testWhereMultiColumnsEqualValues() throws Exception { - try (DataSet ds = dataContext.query().from(bulkIndexType).select("user").and("message").where("user") - .isEquals("user4").and("message").ne(5).execute()) { - assertEquals(JestElasticSearchDataSet.class, ds.getClass()); - - assertTrue(ds.next()); - assertEquals("Row[values=[user4, 4]]", ds.getRow().toString()); - assertFalse(ds.next()); - } - } - - @Test - public void testWhereColumnInValues() throws Exception { - try (DataSet ds = dataContext.query().from(bulkIndexType).select("user").and("message").where("user") - .in("user4", "user5").orderBy("message").execute()) { - assertTrue(ds.next()); - - String row1 = ds.getRow().toString(); - assertEquals("Row[values=[user4, 4]]", row1); - assertTrue(ds.next()); - - String row2 = ds.getRow().toString(); - assertEquals("Row[values=[user5, 5]]", row2); - - assertFalse(ds.next()); - } - } - - @Test - public void testGroupByQuery() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName(peopleIndexType); - - Query q = new Query(); - q.from(table); - q.groupBy(table.getColumnByName("gender")); - q.select(new SelectItem(table.getColumnByName("gender")), - new SelectItem(FunctionType.MAX, table.getColumnByName("age")), - new SelectItem(FunctionType.MIN, table.getColumnByName("age")), new SelectItem(FunctionType.COUNT, "*", - "total"), new SelectItem(FunctionType.MIN, table.getColumnByName("id")).setAlias("firstId")); - q.orderBy("gender"); - DataSet data = dataContext.executeQuery(q); - assertEquals( - "[peopletype.gender, MAX(peopletype.age), MIN(peopletype.age), COUNT(*) AS total, MIN(peopletype.id) AS firstId]", - Arrays.toString(data.getSelectItems().toArray())); - - assertTrue(data.next()); - assertEquals("Row[values=[female, 20, 17, 5, 5]]", data.getRow().toString()); - assertTrue(data.next()); - assertEquals("Row[values=[male, 19, 17, 4, 1]]", data.getRow().toString()); - assertFalse(data.next()); - } - - @Test - public void testFilterOnNumberColumn() { - Table table = dataContext.getDefaultSchema().getTableByName(bulkIndexType); - Query q = dataContext.query().from(table).select("user").where("message").greaterThan(7).toQuery(); - DataSet data = dataContext.executeQuery(q); - String[] expectations = new String[] { "Row[values=[user8]]", "Row[values=[user9]]" }; - - assertTrue(data.next()); - assertTrue(Arrays.asList(expectations).contains(data.getRow().toString())); - assertTrue(data.next()); - assertTrue(Arrays.asList(expectations).contains(data.getRow().toString())); - assertFalse(data.next()); - } - - @Test - public void testMaxRows() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName(peopleIndexType); - Query query = new Query().from(table).select(table.getColumns()).setMaxRows(5); - DataSet dataSet = dataContext.executeQuery(query); - - TableModel tableModel = new DataSetTableModel(dataSet); - assertEquals(5, tableModel.getRowCount()); - } - - @Test - public void testCountQuery() throws Exception { - Table table = dataContext.getDefaultSchema().getTableByName(bulkIndexType); - Query q = new Query().selectCount().from(table); - - List<Object[]> data = dataContext.executeQuery(q).toObjectArrays(); - assertEquals(1, data.size()); - Object[] row = data.get(0); - assertEquals(1, row.length); - assertEquals(10, ((Number) row[0]).intValue()); - } - - @Test(expected = IllegalArgumentException.class) - public void testQueryForANonExistingTable() throws Exception { - dataContext.query().from("nonExistingTable").select("user").and("message").execute(); - } - - @Test(expected = QueryParserException.class) - public void testQueryForAnExistingTableAndNonExistingField() throws Exception { - indexTweeterDocument(indexType1, 1); - dataContext.query().from(indexType1).select("nonExistingField").execute(); - } - - @Test - public void testNonDynamicMapingTableNames() throws Exception { - createIndex(); - - ElasticSearchRestDataContext dataContext2 = new ElasticSearchRestDataContext(client, indexName2); - - assertEquals("[tweet3]", Arrays.toString(dataContext2.getDefaultSchema().getTableNames().toArray())); - } - - private static void createIndex() { - CreateIndexRequest cir = new CreateIndexRequest(indexName2); - CreateIndexResponse response = - embeddedElasticsearchServer.getClient().admin().indices().create(cir).actionGet(); - - System.out.println("create index: " + response.isAcknowledged()); - - PutMappingRequest pmr = new PutMappingRequest(indexName2).type(indexType3).source(mapping); - - PutMappingResponse response2 = - embeddedElasticsearchServer.getClient().admin().indices().putMapping(pmr).actionGet(); - System.out.println("put mapping: " + response2.isAcknowledged()); - } - - private static void indexBulkDocuments(String indexName, String indexType, int numberOfDocuments) { - BulkRequestBuilder bulkRequest = embeddedElasticsearchServer.getClient().prepareBulk(); - - for (int i = 0; i < numberOfDocuments; i++) { - bulkRequest.add(embeddedElasticsearchServer.getClient().prepareIndex(indexName, indexType, - Integer.toString(i)).setSource( - buildTweeterJson(i))); - } - bulkRequest.execute().actionGet(); - } - - private static void indexTweeterDocument(String indexType, int id, Date date) { - embeddedElasticsearchServer.getClient().prepareIndex(indexName, indexType).setSource(buildTweeterJson(id, date)) - .setId("tweet_" + indexType + "_" + id).execute().actionGet(); - } - - private static void indexTweeterDocument(String indexType, int id) { - embeddedElasticsearchServer.getClient().prepareIndex(indexName, indexType).setSource(buildTweeterJson(id)) - .setId("tweet_" + indexType + "_" + id).execute().actionGet(); - } - - private static void indexOnePeopleDocument(String gender, int age, int id) throws IOException { - embeddedElasticsearchServer.getClient().prepareIndex(indexName, peopleIndexType) - .setSource(buildPeopleJson(gender, age, id)).execute() - .actionGet(); - } - - private static Map<String, Object> buildTweeterJson(int elementId) { - return buildTweeterJson(elementId, new Date()); - } - - private static Map<String, Object> buildTweeterJson(int elementId, Date date) { - Map<String, Object> map = new LinkedHashMap<>(); - map.put("user", "user" + elementId); - map.put("postDate", date); - map.put("message", elementId); - return map; - } - - private static XContentBuilder buildPeopleJson(String gender, int age, int elementId) throws IOException { - return jsonBuilder().startObject().field("gender", gender).field("age", age).field("id", elementId).endObject(); - } - -} http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchMetaDataParserTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchMetaDataParserTest.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchMetaDataParserTest.java deleted file mode 100644 index 6eeac6a..0000000 --- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchMetaDataParserTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.elasticsearch.rest; - -import java.util.LinkedHashMap; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.metamodel.elasticsearch.common.ElasticSearchMetaData; -import org.apache.metamodel.schema.ColumnType; -import org.elasticsearch.common.collect.MapBuilder; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; - -public class JestElasticSearchMetaDataParserTest extends TestCase { - - public void testParseMetadataInfo() throws Exception { - Map<String, Object> metadata = new LinkedHashMap<>(); - metadata.put("message", MapBuilder.newMapBuilder().put("type", "long").immutableMap()); - metadata.put("postDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap()); - metadata.put("anotherDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap()); - metadata.put("user", MapBuilder.newMapBuilder().put("type", "string").immutableMap()); - metadata.put("critical", MapBuilder.newMapBuilder().put("type", "boolean").immutableMap()); - metadata.put("income", MapBuilder.newMapBuilder().put("type", "double").immutableMap()); - metadata.put("untypedthingie", MapBuilder.newMapBuilder().put("foo", "bar").immutableMap()); - final Gson gson = new Gson(); - ElasticSearchMetaData metaData = JestElasticSearchMetaDataParser - .parse((JsonObject) gson.toJsonTree(metadata)); - String[] columnNames = metaData.getColumnNames(); - ColumnType[] columnTypes = metaData.getColumnTypes(); - - assertTrue(columnNames.length == 8); - assertEquals(columnNames[0], "_id"); - assertEquals(columnNames[1], "message"); - assertEquals(columnNames[2], "postDate"); - assertEquals(columnNames[3], "anotherDate"); - assertEquals(columnNames[4], "user"); - assertEquals(columnNames[5], "critical"); - assertEquals(columnNames[6], "income"); - assertEquals(columnNames[7], "untypedthingie"); - - assertTrue(columnTypes.length == 8); - assertEquals(columnTypes[0], ColumnType.STRING); - assertEquals(columnTypes[1], ColumnType.BIGINT); - assertEquals(columnTypes[2], ColumnType.DATE); - assertEquals(columnTypes[3], ColumnType.DATE); - assertEquals(columnTypes[4], ColumnType.STRING); - assertEquals(columnTypes[5], ColumnType.BOOLEAN); - assertEquals(columnTypes[6], ColumnType.DOUBLE); - assertEquals(columnTypes[7], ColumnType.STRING); - } -} http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchUtilsTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchUtilsTest.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchUtilsTest.java deleted file mode 100644 index 4c8cca1..0000000 --- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/JestElasticSearchUtilsTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.elasticsearch.rest; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import org.apache.metamodel.data.DataSetHeader; -import org.apache.metamodel.data.Row; -import org.apache.metamodel.data.SimpleDataSetHeader; -import org.apache.metamodel.query.SelectItem; -import org.apache.metamodel.schema.Column; -import org.apache.metamodel.schema.ColumnType; -import org.apache.metamodel.schema.MutableColumn; -import org.junit.Test; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; - -public class JestElasticSearchUtilsTest { - - @Test - public void testAssignDocumentIdForPrimaryKeys() throws Exception { - MutableColumn primaryKeyColumn = new MutableColumn("value1", ColumnType.STRING).setPrimaryKey(true); - SelectItem primaryKeyItem = new SelectItem(primaryKeyColumn); - List<SelectItem> selectItems1 = Collections.singletonList(primaryKeyItem); - String documentId = "doc1"; - DataSetHeader header = new SimpleDataSetHeader(selectItems1); - JsonObject values = new JsonObject(); - - values.addProperty("value1", "theValue"); - Row row = JestElasticSearchUtils.createRow(values, documentId, header); - String primaryKeyValue = (String) row.getValue(primaryKeyItem); - - assertEquals(primaryKeyValue, documentId); - } - - @Test - public void testCreateRowWithNullValues() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.STRING); - final Column col2 = new MutableColumn("col2", ColumnType.STRING); - final DataSetHeader header = new SimpleDataSetHeader(Lists.newArrayList(col1, col2).stream().map(SelectItem::new).collect(Collectors.toList())); - final JsonObject source = new JsonObject(); - source.addProperty("col1", "foo"); - source.addProperty("col2", (String) null); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - assertEquals("Row[values=[foo, null]]", row.toString()); - } - - @Test - public void testCreateRowWithNumberValueAndStringType() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.STRING); - final DataSetHeader header = SimpleDataSetHeader.fromColumns(Lists.newArrayList(col1)); - final JsonObject source = new JsonObject(); - source.addProperty("col1", 42); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - assertEquals("Row[values=[42]]", row.toString()); - } - - @Test - public void testCreateRowWithStringValueAndNumberType() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.NUMBER); - final DataSetHeader header = SimpleDataSetHeader.fromColumns(Lists.newArrayList(col1)); - final JsonObject source = new JsonObject(); - source.addProperty("col1", "hello world"); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - - // whether or not 'null' should be returned (bad value, but preserves - // type) or 'hello world' should be returned (correct value, breaks - // type) can be debated. For now it is added here as an assertion to - // keep track of any regressions. - assertEquals("Row[values=[null]]", row.toString()); - } - - @Test - public void testCreateRowWithJsonObject() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.MAP); - final DataSetHeader header = SimpleDataSetHeader.fromColumns(Lists.newArrayList(col1)); - final JsonObject source = new JsonObject(); - final JsonObject value = new JsonObject(); - value.addProperty("foo1", "bar"); - value.addProperty("foo2", 42); - source.add("col1", value); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - assertEquals("Row[values=[{foo1=bar, foo2=42.0}]]", row.toString()); - - final Map<?, ?> rowValue = (Map<?, ?>) row.getValue(col1); - assertEquals("bar", rowValue.get("foo1")); - } - - @Test - public void testCreateRowWithJsonArray() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.LIST); - final DataSetHeader header = SimpleDataSetHeader.fromColumns(Lists.newArrayList(col1)); - final JsonObject source = new JsonObject(); - final JsonArray value = new JsonArray(); - value.add(new JsonPrimitive("foo")); - value.add(new JsonPrimitive("bar")); - source.add("col1", value); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - assertEquals("Row[values=[[foo, bar]]]", row.toString()); - - final List<?> rowValue = (List<?>) row.getValue(col1); - assertEquals("foo", rowValue.get(0)); - } - - @Test - public void testCreateRowWithDeepNesting() throws Exception { - final Column col1 = new MutableColumn("col1", ColumnType.LIST); - final DataSetHeader header = SimpleDataSetHeader.fromColumns(Lists.newArrayList(col1)); - final JsonObject source = new JsonObject(); - - final JsonObject obj2 = new JsonObject(); - obj2.addProperty("foo", 43); - - final JsonArray arr1 = new JsonArray(); - arr1.add(new JsonPrimitive("foo")); - arr1.add(new JsonPrimitive("bar")); - arr1.add(obj2); - - final JsonObject obj1 = new JsonObject(); - obj1.addProperty("mybool", true); - obj1.add("arr1", arr1); - source.add("col1", obj1); - final String documentId = "row1"; - - final Row row = JestElasticSearchUtils.createRow(source, documentId, header); - assertEquals("Row[values=[{mybool=true, arr1=[foo, bar, {foo=43.0}]}]]", row.toString()); - - final Map<?, ?> rowObj1 = (Map<?, ?>) row.getValue(col1); - final List<?> rowList = (List<?>) rowObj1.get("arr1"); - final Map<?, ?> rowObj2 = (Map<?, ?>) rowList.get(2); - assertEquals(43.0, rowObj2.get("foo")); - } - - @Test - public void testCreateRowWithParseableDates() throws Exception { - SelectItem item1 = new SelectItem(new MutableColumn("value1", ColumnType.STRING)); - SelectItem item2 = new SelectItem(new MutableColumn("value2", ColumnType.DATE)); - List<SelectItem> selectItems1 = Arrays.asList(item1, item2); - String documentId = "doc1"; - DataSetHeader header = new SimpleDataSetHeader(selectItems1); - JsonObject values = new JsonObject(); - values.addProperty("value1", "theValue"); - values.addProperty("value2", "2013-01-04T15:55:51.217+01:00"); - Row row = JestElasticSearchUtils.createRow(values, documentId, header); - Object stringValue = row.getValue(item1); - Object dateValue = row.getValue(item2); - - assertTrue(stringValue instanceof String); - assertTrue(dateValue instanceof Date); - } -} http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/utils/EmbeddedElasticsearchServer.java ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/utils/EmbeddedElasticsearchServer.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/utils/EmbeddedElasticsearchServer.java deleted file mode 100644 index 11e7eb5..0000000 --- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/utils/EmbeddedElasticsearchServer.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.elasticsearch.rest.utils; - -import org.apache.commons.io.FileUtils; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.node.Node; - -import java.io.File; -import java.io.IOException; - -import static org.elasticsearch.node.NodeBuilder.nodeBuilder; - -public class EmbeddedElasticsearchServer { - - private static final String DEFAULT_DATA_DIRECTORY = "target/elasticsearch-data"; - - private final Node node; - private final String dataDirectory; - - public EmbeddedElasticsearchServer() { - this(DEFAULT_DATA_DIRECTORY); - } - - public EmbeddedElasticsearchServer(String dataDirectory) { - this.dataDirectory = dataDirectory; - - ImmutableSettings.Builder elasticsearchSettings = ImmutableSettings.settingsBuilder() - .put("http.enabled", "true") - .put("path.data", dataDirectory) - .put("http.port", 9292); - - node = nodeBuilder() - .local(true) - .settings(elasticsearchSettings.build()) - .node(); - } - - public Client getClient() { - return node.client(); - } - - public void shutdown() { - node.close(); - deleteDataDirectory(); - } - - private void deleteDataDirectory() { - try { - FileUtils.deleteDirectory(new File(dataDirectory)); - } catch (IOException e) { - throw new RuntimeException("Could not delete data directory of embedded elasticsearch server", e); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/resources/Dockerfile ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/resources/Dockerfile b/elasticsearch/rest/src/test/resources/Dockerfile new file mode 100644 index 0000000..6c10f8e --- /dev/null +++ b/elasticsearch/rest/src/test/resources/Dockerfile @@ -0,0 +1,5 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.3 +ADD elasticsearch.yml /usr/share/elasticsearch/config/ +USER root +RUN chown elasticsearch:elasticsearch config/elasticsearch.yml +USER elasticsearch \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/elasticsearch/rest/src/test/resources/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/test/resources/elasticsearch.yml b/elasticsearch/rest/src/test/resources/elasticsearch.yml new file mode 100644 index 0000000..ba7c07f --- /dev/null +++ b/elasticsearch/rest/src/test/resources/elasticsearch.yml @@ -0,0 +1,13 @@ +bootstrap.memory_lock: true +cluster.name: docker-cluster +http.port: 9200 +node.data: true +node.ingest: true +node.master: true +node.max_local_storage_nodes: 1 +node.name: estest +path.data: /usr/share/elasticsearch/data +path.logs: /usr/share/elasticsearch/logs +transport.tcp.port: 9300 +discovery.type: single-node +network.host: 0.0.0.0 http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/full/src/main/java/org/apache/metamodel/DataContextFactory.java ---------------------------------------------------------------------- diff --git a/full/src/main/java/org/apache/metamodel/DataContextFactory.java b/full/src/main/java/org/apache/metamodel/DataContextFactory.java index 427101e..eeb7df5 100644 --- a/full/src/main/java/org/apache/metamodel/DataContextFactory.java +++ b/full/src/main/java/org/apache/metamodel/DataContextFactory.java @@ -33,6 +33,7 @@ import org.apache.metamodel.couchdb.CouchDbDataContext; import org.apache.metamodel.csv.CsvConfiguration; import org.apache.metamodel.csv.CsvDataContext; import org.apache.metamodel.elasticsearch.nativeclient.ElasticSearchDataContext; +import org.apache.metamodel.elasticsearch.rest.ElasticSearchRestClient; import org.apache.metamodel.elasticsearch.rest.ElasticSearchRestDataContext; import org.apache.metamodel.excel.ExcelConfiguration; import org.apache.metamodel.excel.ExcelDataContext; @@ -64,8 +65,6 @@ import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.client.MongoDatabase; -import io.searchbox.client.JestClient; - /** * A factory for DataContext objects. This class substantially easens the task * of creating and initializing DataContext objects and/or their strategies for @@ -681,7 +680,8 @@ public class DataContextFactory { * The ElasticSearch index name * @return a DataContext object that matches the request */ - public static UpdateableDataContext createElasticSearchDataContext(JestClient client, String indexName) { + public static UpdateableDataContext createElasticSearchDataContext(final ElasticSearchRestClient client, + final String indexName) { return new ElasticSearchRestDataContext(client, indexName); } http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/hbase/pom.xml b/hbase/pom.xml index 172e2f6..6659dad 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -111,6 +111,10 @@ <artifactId>netty</artifactId> </exclusion> <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-all</artifactId> + </exclusion> + <exclusion> <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> http://git-wip-us.apache.org/repos/asf/metamodel/blob/bda8d764/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4caae5b..3df13d5 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ under the License. <spring.version>4.2.6.RELEASE</spring.version> <httpcomponents.version>4.4.1</httpcomponents.version> <checksum-maven-plugin.version>1.2</checksum-maven-plugin.version> + <docker-maven-plugin.version>0.23.0</docker-maven-plugin.version> <skipTests>false</skipTests> </properties> <parent>
