Repository: metamodel Updated Branches: refs/heads/master 60989cc25 -> 9553357cd
METAMODEL-197: Fixed Fixes #61 Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/9553357c Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/9553357c Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/9553357c Branch: refs/heads/master Commit: 9553357cd5b1b8e38b774c283d859f6b6f8c62ed Parents: 60989cc Author: Dennis Du Krøger <[email protected]> Authored: Fri Oct 23 12:45:49 2015 +0200 Committer: Dennis Du Krøger <[email protected]> Committed: Fri Oct 23 12:45:49 2015 +0200 ---------------------------------------------------------------------- .../apache/metamodel/util/SimpleTableDef.java | 3 + .../ElasticSearchCreateTableBuilder.java | 3 +- .../elasticsearch/ElasticSearchDataContext.java | 64 +++++++++++++------- .../ElasticSearchInsertBuilder.java | 2 +- .../ElasticSearchDataContextTest.java | 31 ++++++++++ 5 files changed, 78 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java b/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java index 0dd98a9..f9027fb 100644 --- a/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java +++ b/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java @@ -81,6 +81,9 @@ public class SimpleTableDef implements Serializable, HasName { * the column types of the columns specified. */ public SimpleTableDef(String name, String[] columnNames, ColumnType[] columnTypes) { + if(name == null){ + throw new NullPointerException("Table name cannot be null"); + } _name = name; _columnNames = columnNames; if (columnTypes == null) { http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java index e550506..0a1750d 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java @@ -59,7 +59,7 @@ final class ElasticSearchCreateTableBuilder extends AbstractTableCreationBuilder final IndicesAdminClient indicesAdmin = dataContext.getElasticSearchClient().admin().indices(); final String indexName = dataContext.getIndexName(); - final List<Object> sourceProperties = new ArrayList<Object>(); + final List<Object> sourceProperties = new ArrayList<>(); for (Column column : table.getColumns()) { // each column is defined as a property pair of the form: ("field1", // "type=string,store=true") @@ -87,7 +87,6 @@ final class ElasticSearchCreateTableBuilder extends AbstractTableCreationBuilder final MutableSchema schema = (MutableSchema) getSchema(); schema.addTable(table); - return table; } http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java index 6d8d5b4..34bb983 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java @@ -93,8 +93,12 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem public static final TimeValue TIMEOUT_SCROLL = TimeValue.timeValueSeconds(60); private final Client elasticSearchClient; - private final SimpleTableDef[] tableDefs; private final String indexName; + // Table definitions that are set from the beginning, not supposed to be changed. + private final List<SimpleTableDef> staticTableDefinitions; + + // Table definitions that are discovered, these can change + private final List<SimpleTableDef> dynamicTableDefinitions = new ArrayList<>(); /** * Constructs a {@link ElasticSearchDataContext}. This constructor accepts a @@ -105,11 +109,11 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem * the ElasticSearch client * @param indexName * the name of the ElasticSearch index to represent - * @param tableDefs + * @param tableDefinitions * an array of {@link SimpleTableDef}s, which define the table * and column model of the ElasticSearch index. */ - public ElasticSearchDataContext(Client client, String indexName, SimpleTableDef... tableDefs) { + public ElasticSearchDataContext(Client client, String indexName, SimpleTableDef... tableDefinitions) { if (client == null) { throw new IllegalArgumentException("ElasticSearch Client cannot be null"); } @@ -118,13 +122,14 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem } this.elasticSearchClient = client; this.indexName = indexName; - this.tableDefs = tableDefs; + this.staticTableDefinitions = Arrays.asList(tableDefinitions); + this.dynamicTableDefinitions.addAll(Arrays.asList(detectSchema())); } /** * Constructs a {@link ElasticSearchDataContext} and automatically detects * the schema structure/view on all indexes (see - * {@link #detectSchema(Client, String)}). + * {@link this.detectSchema(Client, String)}). * * @param client * the ElasticSearch client @@ -132,7 +137,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem * the name of the ElasticSearch index to represent */ public ElasticSearchDataContext(Client client, String indexName) { - this(client, indexName, detectSchema(client, indexName)); + this(client, indexName, new SimpleTableDef[0]); } /** @@ -141,18 +146,15 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem * based on the metadata provided by the ElasticSearch java client. * * @see #detectTable(ClusterState, String, String) - * - * @param client - * the client to inspect - * @param indexName * @return a mutable schema instance, useful for further fine tuning by the * user. */ - public static SimpleTableDef[] detectSchema(Client client, String indexName) { + private SimpleTableDef[] detectSchema() { logger.info("Detecting schema for index '{}'", indexName); final ClusterState cs; - final ClusterStateRequestBuilder clusterStateRequestBuilder = client.admin().cluster().prepareState(); + final ClusterStateRequestBuilder clusterStateRequestBuilder = + getElasticSearchClient().admin().cluster().prepareState(); // different methods here to set the index name, so we have to use // reflection :-/ @@ -172,7 +174,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem } cs = clusterStateRequestBuilder.execute().actionGet().getState(); - final List<SimpleTableDef> result = new ArrayList<SimpleTableDef>(); + final List<SimpleTableDef> result = new ArrayList<>(); final IndexMetaData imd = cs.getMetaData().index(indexName); if (imd == null) { @@ -192,7 +194,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem } } } - final SimpleTableDef[] tableDefArray = (SimpleTableDef[]) result.toArray(new SimpleTableDef[result.size()]); + final SimpleTableDef[] tableDefArray = result.toArray(new SimpleTableDef[result.size()]); Arrays.sort(tableDefArray, new Comparator<SimpleTableDef>() { @Override public int compare(SimpleTableDef o1, SimpleTableDef o2) { @@ -244,18 +246,36 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem @Override protected Schema getMainSchema() throws MetaModelException { final MutableSchema theSchema = new MutableSchema(getMainSchemaName()); - for (final SimpleTableDef tableDef : tableDefs) { - final MutableTable table = tableDef.toTable().setSchema(theSchema); - final Column idColumn = table.getColumnByName(FIELD_ID); - if (idColumn != null && idColumn instanceof MutableColumn) { - final MutableColumn mutableColumn = (MutableColumn) idColumn; - mutableColumn.setPrimaryKey(true); + for (final SimpleTableDef tableDef : staticTableDefinitions) { + addTable(theSchema, tableDef); + } + + final SimpleTableDef[] tables = detectSchema(); + synchronized (this) { + dynamicTableDefinitions.clear(); + dynamicTableDefinitions.addAll(Arrays.asList(tables)); + for (final SimpleTableDef tableDef : dynamicTableDefinitions) { + final List<String> tableNames = Arrays.asList(theSchema.getTableNames()); + + if (!tableNames.contains(tableDef.getName())) { + addTable(theSchema, tableDef); + } } - theSchema.addTable(table); } + return theSchema; } + private void addTable(final MutableSchema theSchema, final SimpleTableDef tableDef) { + final MutableTable table = tableDef.toTable().setSchema(theSchema); + final Column idColumn = table.getColumnByName(FIELD_ID); + if (idColumn != null && idColumn instanceof MutableColumn) { + final MutableColumn mutableColumn = (MutableColumn) idColumn; + mutableColumn.setPrimaryKey(true); + } + theSchema.addTable(table); + } + @Override protected String getMainSchemaName() throws MetaModelException { return indexName; @@ -317,7 +337,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem return QueryBuilders.matchAllQuery(); } - List<QueryBuilder> children = new ArrayList<QueryBuilder>(whereItems.size()); + List<QueryBuilder> children = new ArrayList<>(whereItems.size()); for (FilterItem item : whereItems) { final QueryBuilder itemQueryBuilder; http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java index 321af1d..f2d8746 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java @@ -47,7 +47,7 @@ final class ElasticSearchInsertBuilder extends AbstractRowInsertionBuilder<Elast final String documentType = getTable().getName(); final IndexRequestBuilder requestBuilder = new IndexRequestBuilder(client, indexName).setType(documentType); - final Map<String, Object> valueMap = new HashMap<String, Object>(); + final Map<String, Object> valueMap = new HashMap<>(); final Column[] columns = getColumns(); final Object[] values = getValues(); for (int i = 0; i < columns.length; i++) { http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java index efd0c97..5c17587 100644 --- a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java +++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java @@ -49,10 +49,13 @@ import org.apache.metamodel.schema.Table; import org.apache.metamodel.update.Update; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.xcontent.XContentBuilder; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -214,6 +217,8 @@ public class ElasticSearchDataContextTest { } }); + dataContext.refreshSchemas(); + try (DataSet ds = dataContext.query().from(table).selectAll().orderBy("bar").execute()) { assertTrue(ds.next()); assertEquals("hello", ds.getRow().getValue(fooColumn).toString()); @@ -225,6 +230,32 @@ public class ElasticSearchDataContextTest { } dataContext.executeUpdate(new DropTable(table)); + + dataContext.refreshSchemas(); + + assertNull(dataContext.getTableByQualifiedLabel(table.getName())); + } + + @Test + public void testDetectOutsideChanges() throws Exception { + ElasticSearchDataContext elasticSearchDataContext = (ElasticSearchDataContext) dataContext; + + // Create the type in ES + final IndicesAdminClient indicesAdmin = elasticSearchDataContext.getElasticSearchClient().admin().indices(); + final String tableType = "outsideTable"; + + Object[] sourceProperties = { "testA", "type=string, store=true", "testB", "type=string, store=true" }; + + new PutMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).setSource(sourceProperties) + .execute().actionGet(); + + dataContext.refreshSchemas(); + + assertNotNull(dataContext.getDefaultSchema().getTableByName(tableType)); + + new DeleteMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).execute().actionGet(); + dataContext.refreshSchemas(); + assertNull(dataContext.getTableByQualifiedLabel(tableType)); } @Test
