Fixing empty schema issue that seems to happen when new field definitions are added to a table or when new fields are added via MR.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/043e52e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/043e52e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/043e52e5 Branch: refs/heads/master Commit: 043e52e5c55e6d0989ecd3e6a26042d3d699efab Parents: bc688a4 Author: Aaron McCurry <[email protected]> Authored: Fri Dec 6 10:53:55 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Fri Dec 6 10:55:05 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 42 ---------------- .../org/apache/blur/server/TableContext.java | 2 +- .../blur/thrift/BlurControllerServer.java | 31 ------------ .../org/apache/blur/thrift/BlurShardServer.java | 8 +-- .../java/org/apache/blur/thrift/TableAdmin.java | 52 ++++++++++++++++++++ .../apache/blur/manager/IndexManagerTest.java | 15 ------ .../org/apache/blur/thrift/BlurClusterTest.java | 25 ++++++++++ .../apache/blur/analysis/BaseFieldManager.java | 7 ++- .../org/apache/blur/analysis/FieldManager.java | 2 + 9 files changed, 86 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 8804697..6cd9f8e 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -45,7 +45,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.analysis.FieldManager; -import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.blur.concurrent.Executors; import org.apache.blur.index.ExitableReader; import org.apache.blur.index.ExitableReader.ExitingReaderException; @@ -69,7 +68,6 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.ErrorType; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.FetchRowResult; @@ -81,7 +79,6 @@ import org.apache.blur.thrift.generated.RecordMutationType; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.RowMutationType; -import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.ScoreType; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.trace.Trace; @@ -818,45 +815,6 @@ public class IndexManager { return new Term(columnFamily + "." + columnName, value); } - public Schema schema(String table) throws IOException { - TableContext tableContext = getTableContext(table); - FieldManager fieldManager = tableContext.getFieldManager(); - Schema schema = new Schema().setTable(table); - schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>()); - Set<String> fieldNames = fieldManager.getFieldNames(); - INNER: for (String fieldName : fieldNames) { - FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName); - if (fieldTypeDefinition == null) { - continue INNER; - } - String columnName = fieldTypeDefinition.getColumnName(); - String columnFamily = fieldTypeDefinition.getFamily(); - String subColumnName = fieldTypeDefinition.getSubColumnName(); - Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily); - if (map == null) { - map = new HashMap<String, ColumnDefinition>(); - schema.putToFamilies(columnFamily, map); - } - if (subColumnName == null) { - map.put(columnName, getColumnDefinition(fieldTypeDefinition)); - } else { - map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition)); - } - } - return schema; - } - - private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition) { - ColumnDefinition columnDefinition = new ColumnDefinition(); - columnDefinition.setFamily(fieldTypeDefinition.getFamily()); - columnDefinition.setColumnName(fieldTypeDefinition.getColumnName()); - columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName()); - columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed()); - columnDefinition.setFieldType(fieldTypeDefinition.getFieldType()); - columnDefinition.setProperties(fieldTypeDefinition.getProperties()); - return columnDefinition; - } - public void mutate(final RowMutation mutation) throws BlurException, IOException { long s = System.nanoTime(); doMutate(mutation); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index d361625..d0d54e6 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -134,7 +134,7 @@ public class TableContext { HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps); loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager); - hdfsFieldManager.load(); + hdfsFieldManager.loadFromStorage(); tableContext.fieldManager = hdfsFieldManager; } catch (IOException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 03a3134..71780f4 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -802,37 +802,6 @@ public class BlurControllerServer extends TableAdmin implements Iface { } @Override - public Schema schema(final String table) throws BlurException, TException { - checkTable(table); - try { - return scatterGather(getCluster(table), new BlurCommand<Schema>() { - @Override - public Schema call(Client client) throws BlurException, TException { - return client.schema(table); - } - }, new Merger<Schema>() { - @Override - public Schema merge(BlurExecutorCompletionService<Schema> service) throws BlurException { - Schema result = null; - while (service.getRemainingCount() > 0) { - Future<Schema> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, true, table); - Schema schema = service.getResultThrowException(future, table); - if (result == null) { - result = schema; - } else { - result = BlurControllerServer.merge(result, schema); - } - } - return result; - } - }); - } catch (Exception e) { - LOG.error("Unknown error while trying to schema table [{0}]", e, table); - throw new BException("Unknown error while trying to schema table [{0}]", e, table); - } - } - - @Override public List<String> terms(final String table, final String columnFamily, final String columnName, final String startWith, final short size) throws BlurException, TException { checkTable(table); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 2c24ad9..1397ba7 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -286,14 +286,8 @@ public class BlurShardServer extends TableAdmin implements Iface { @Override public Schema schema(String table) throws BlurException, TException { - checkTable(_cluster, table); resetSearchers(); - try { - return _indexManager.schema(table); - } catch (Exception e) { - LOG.error("Unknown error while trying to get schema for table [{0}={1}]", e, "table", table); - throw new BException(e.getMessage(), e); - } + return super.schema(table); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index d487916..95f9275 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.clusterstatus.ClusterStatus; @@ -34,6 +35,7 @@ import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Metric; +import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.ShardState; import org.apache.blur.thrift.generated.TableDescriptor; @@ -517,4 +519,54 @@ public abstract class TableAdmin implements Iface { public void setMaxRecordsPerRowFetchRequest(int _maxRecordsPerRowFetchRequest) { this._maxRecordsPerRowFetchRequest = _maxRecordsPerRowFetchRequest; } + + @Override + public Schema schema(String table) throws BlurException, TException { + checkTable(table); + try { + TableContext tableContext = getTableContext(table); + FieldManager fieldManager = tableContext.getFieldManager(); + fieldManager.loadFromStorage(); + Schema schema = new Schema().setTable(table); + schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>()); + Set<String> fieldNames = fieldManager.getFieldNames(); + INNER: for (String fieldName : fieldNames) { + FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName); + if (fieldTypeDefinition == null) { + continue INNER; + } + String columnName = fieldTypeDefinition.getColumnName(); + String columnFamily = fieldTypeDefinition.getFamily(); + String subColumnName = fieldTypeDefinition.getSubColumnName(); + Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily); + if (map == null) { + map = new HashMap<String, ColumnDefinition>(); + schema.putToFamilies(columnFamily, map); + } + if (subColumnName == null) { + map.put(columnName, getColumnDefinition(fieldTypeDefinition)); + } else { + map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition)); + } + } + return schema; + } catch (Exception e) { + throw new BException("Unknown error while trying to get schema for table [{0}]", table); + } + } + + private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition) { + ColumnDefinition columnDefinition = new ColumnDefinition(); + columnDefinition.setFamily(fieldTypeDefinition.getFamily()); + columnDefinition.setColumnName(fieldTypeDefinition.getColumnName()); + columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName()); + columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed()); + columnDefinition.setFieldType(fieldTypeDefinition.getFieldType()); + columnDefinition.setProperties(fieldTypeDefinition.getProperties()); + return columnDefinition; + } + + private TableContext getTableContext(final String table) { + return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true, table), table)); + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index 55cf481..851df4f 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -41,9 +41,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Random; -import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicLongArray; @@ -56,7 +54,6 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Facet; import org.apache.blur.thrift.generated.FetchRecordResult; import org.apache.blur.thrift.generated.FetchResult; @@ -67,7 +64,6 @@ import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.RowMutationType; -import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.ScoreType; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableDescriptor; @@ -715,17 +711,6 @@ public class IndexManagerTest { } @Test - public void testSchema() throws Exception { - Schema schema = indexManager.schema(TABLE); - assertEquals(TABLE, schema.table); - Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); - assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(families.keySet())); - assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3", "testcol12", "testcol13")), - new TreeSet<String>(families.get(FAMILY).keySet())); - assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(families.get(FAMILY2).keySet())); - } - - @Test public void testQuerySuperQueryTrue() throws Exception { BlurQuery blurQuery = new BlurQuery(); blurQuery.query = new Query(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index 6b07d04..59201d0 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -40,7 +40,9 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.blur.MiniCluster; import org.apache.blur.TestType; +import org.apache.blur.analysis.FieldManager; import org.apache.blur.manager.IndexManager; +import org.apache.blur.server.TableContext; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.Blur.Iface; @@ -48,6 +50,7 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.ErrorType; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Query; @@ -112,6 +115,7 @@ public class BlurClusterTest { KeeperException { testCreateTable(); testLoadTable(); + testForEmptySchema(); testQueryWithSelector(); testBatchFetch(); testQueryCancel(); @@ -122,6 +126,27 @@ public class BlurClusterTest { testCreateTableWithCustomType(); } + private void testForEmptySchema() throws BlurException, TException, IOException { + Blur.Iface client = getClient(); + Schema schema = client.schema("test"); + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); + assertTrue(!families.isEmpty()); + int size = families.size(); + System.out.println(size); + + TableContext tableContext = TableContext.create(client.describe("test")); + FieldManager fieldManager = tableContext.getFieldManager(); + + assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false, "string", null)); + + TableContext.clear(); + Schema newschema = client.schema("test"); + Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies(); + assertTrue(!newfamilies.isEmpty()); + int newsize = newfamilies.size(); + assertEquals(size + 1, newsize); + } + private void testCreateTableWithCustomType() throws IOException, BlurException, TException { Blur.Iface client = getClient(); TableDescriptor tableDescriptor = new TableDescriptor(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index 0121d72..9c974c5 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -181,10 +181,13 @@ public abstract class BaseFieldManager extends FieldManager { return new TreeSet<String>(_fieldNameToDefMap.keySet()); } - public void load() throws IOException { + @Override + public void loadFromStorage() throws IOException { List<String> fieldNamesToLoad = getFieldNamesToLoad(); for (String fieldName : fieldNamesToLoad) { - tryToLoad(fieldName); + if (!_fieldNameToDefMap.containsKey(fieldName)) { + tryToLoad(fieldName); + } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index 050e6cc..b6b36fd 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -271,4 +271,6 @@ public abstract class FieldManager { public abstract String resolveField(String field); + public abstract void loadFromStorage() throws IOException; + }
