Changing the create table call to be idempotent as well as faster. It longer creates all the shard dirs in hdfs, the shard servers now do that as they bring the shards online for the first time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ce2bc4d5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ce2bc4d5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ce2bc4d5 Branch: refs/heads/master Commit: ce2bc4d5d50b9e65f7fcc2c0446dce8a14d46b92 Parents: 2dacede Author: Aaron McCurry <[email protected]> Authored: Thu Jan 29 09:42:48 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Jan 29 09:42:48 2015 -0500 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexServer.java | 9 ----- .../AbstractDistributedIndexServer.java | 42 -------------------- .../indexserver/DistributedIndexServer.java | 13 +++++- .../manager/indexserver/LocalIndexServer.java | 21 ---------- .../java/org/apache/blur/thrift/TableAdmin.java | 18 +++++++-- .../blur/thrift/ThriftBlurShardServer.java | 1 - .../blur/command/ShardCommandManagerTest.java | 5 --- .../apache/blur/thrift/BlurClusterTestBase.java | 1 + .../blur/thrift/BlurClusterTestNoSecurity.java | 13 ------ .../java/org/apache/blur/thrift/FacetTests.java | 11 +---- .../org/apache/blur/thrift/SuiteCluster.java | 1 - 11 files changed, 30 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java index ab5aad7..9275df4 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java @@ -57,15 +57,6 @@ public interface IndexServer extends Closeable { // Table Meta Data /** - * The shard list for a given table. - * - * @param table - * the table name. - * @return the list of shards. - */ - List<String> getShardList(String table); - - /** * Gets the current nodes name. * * @return http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java index f50b58e..d1e7500 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java @@ -22,24 +22,15 @@ import static org.apache.blur.metrics.MetricsConstants.INDEX_MEMORY_USAGE; import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR; import static org.apache.blur.metrics.MetricsConstants.SEGMENT_COUNT; import static org.apache.blur.metrics.MetricsConstants.TABLE_COUNT; -import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicLong; -import org.apache.blur.log.Log; -import org.apache.blur.log.LogFactory; import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.metrics.AtomicLongGauge; import org.apache.blur.server.TableContext; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.trace.Trace; -import org.apache.blur.trace.Tracer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -48,8 +39,6 @@ import com.yammer.metrics.core.MetricName; public abstract class AbstractDistributedIndexServer extends AbstractIndexServer { - private static final Log LOG = LogFactory.getLog(AbstractDistributedIndexServer.class); - protected final ClusterStatus _clusterStatus; protected final Configuration _configuration; protected final String _nodeName; @@ -92,37 +81,6 @@ public abstract class AbstractDistributedIndexServer extends AbstractIndexServer return contentSummary.getLength(); } - @Override - public final List<String> getShardList(String table) { - checkTable(table); - Tracer trace = Trace.trace("filesystem - getShardList", Trace.param("table", table)); - List<String> result = new ArrayList<String>(); - try { - TableContext tableContext = getTableContext(table); - TableDescriptor descriptor = tableContext.getDescriptor(); - Path tablePath = new Path(descriptor.tableUri); - FileSystem fileSystem = tablePath.getFileSystem(_configuration); - if (!fileSystem.exists(tablePath)) { - LOG.error("Table [{0}] is missing, defined location [{1}]", table, tablePath.toUri()); - throw new RuntimeException("Table [" + table + "] is missing, defined location [" + tablePath.toUri() + "]"); - } - FileStatus[] listStatus = fileSystem.listStatus(tablePath); - for (FileStatus status : listStatus) { - if (status.isDir()) { - String name = status.getPath().getName(); - if (name.startsWith(SHARD_PREFIX)) { - result.add(name); - } - } - } - return result; - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - trace.done(); - } - } - protected final TableContext getTableContext(final String table) { return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true, table), table)); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index c087ed8..79a5b3c 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -57,6 +57,7 @@ import org.apache.blur.store.hdfs_v2.JoinDirectory; import org.apache.blur.thrift.generated.ShardState; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ShardUtil; import org.apache.blur.zookeeper.WatchChildren; import org.apache.blur.zookeeper.WatchChildren.OnChange; import org.apache.blur.zookeeper.ZookeeperPathConstants; @@ -602,7 +603,8 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { throw new RuntimeException("Table [" + table + "] is not found."); } List<String> onlineShardServerList = _clusterStatus.getOnlineShardServers(false, cluster); - List<String> shardList = getShardList(table); + TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(false, cluster, table); + List<String> shardList = generateShardList(tableDescriptor); String shutdownPath = ZookeeperPathConstants.getShutdownPath(cluster); if (isShuttingDown(shutdownPath)) { @@ -625,6 +627,15 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { return shardsToServeCache; } + private List<String> generateShardList(TableDescriptor tableDescriptor) { + int shardCount = tableDescriptor.getShardCount(); + List<String> list = new ArrayList<String>(); + for (int i = 0; i < shardCount; i++) { + list.add(ShardUtil.getShardName(i)); + } + return list; + } + private boolean isShuttingDown(String shutdownPath) { try { Stat stat = _zookeeper.exists(shutdownPath, false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java index 8cce939..54a06f6 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java @@ -23,8 +23,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -173,25 +171,6 @@ public class LocalIndexServer extends AbstractIndexServer { } @Override - public List<String> getShardList(String table) { - try { - List<String> result = new ArrayList<String>(); - Path tablePath = _tableContext.getTablePath(); - File tableFile = new File(new File(tablePath.toUri()), table); - if (tableFile.isDirectory()) { - for (File f : tableFile.listFiles()) { - if (f.isDirectory()) { - result.add(f.getName()); - } - } - } - return result; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override public String getNodeName() { return "localhost"; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index 588c87c..69f74a2 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -48,7 +48,6 @@ import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.CommandDescriptor; import org.apache.blur.thrift.generated.Level; import org.apache.blur.thrift.generated.Metric; -import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.ShardState; @@ -117,10 +116,23 @@ public abstract class TableAdmin implements Iface { @Override public final void createTable(TableDescriptor tableDescriptor) throws BlurException, TException { try { - TableContext.clear(tableDescriptor.getName()); ShardUtil.validateTableName(tableDescriptor.getName()); assignClusterIfNull(tableDescriptor); - _clusterStatus.createTable(tableDescriptor); + List<String> tableList = _clusterStatus.getTableList(false, tableDescriptor.getCluster()); + if (!tableList.contains(tableDescriptor.getName())) { + TableContext.clear(tableDescriptor.getName()); + _clusterStatus.createTable(tableDescriptor); + } else { + TableDescriptor existing = _clusterStatus.getTableDescriptor(false, tableDescriptor.getCluster(), + tableDescriptor.getName()); + if (existing.equals(tableDescriptor)) { + LOG.warn("Table [{0}] has already exists, but tried to create same table a second time.", + tableDescriptor.getName()); + } else { + LOG.warn("Table [{0}] has already exists.", tableDescriptor.getName()); + throw new BException("Table [{0}] has already exists.", tableDescriptor.getName()); + } + } } catch (Exception e) { LOG.error("Unknown error during create of [table={0}, tableDescriptor={1}]", e, tableDescriptor.name, tableDescriptor); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 6f7516c..cbdd2ce 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -58,7 +58,6 @@ import static org.apache.blur.utils.BlurUtil.quietClose; import java.io.Closeable; import java.io.File; -import java.io.IOException; import java.lang.reflect.Constructor; import java.util.Map.Entry; import java.util.Set; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index e9edf32..817718a 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -342,11 +342,6 @@ public class ShardCommandManagerTest { } @Override - public List<String> getShardList(String table) { - throw new RuntimeException("Not implemented."); - } - - @Override public long getRowCount(String table) throws IOException { throw new RuntimeException("Not implemented."); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java index a52e81d..61b79ee 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java @@ -323,6 +323,7 @@ public abstract class BlurClusterTestBase { columnDefinition.setFieldLessIndexed(true); columnDefinition.setFieldType("string"); columnDefinition.setSortable(true); + columnDefinition.setMultiValueField(false); columnDefinition.setProperties(new HashMap<String, String>()); client.addColumnDefinition(tableName, columnDefinition); long s = System.nanoTime(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java index a7dc5dd..1504f01 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java @@ -16,26 +16,13 @@ */ package org.apache.blur.thrift; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Random; -import java.util.UUID; -import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur.Iface; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.thrift.generated.TableStats; -import org.apache.blur.thrift.util.BlurThriftHelper; import org.apache.blur.user.User; -import org.junit.Test; public class BlurClusterTestNoSecurity extends BlurClusterTestBase { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java b/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java index 433fa16..d6f858e 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java @@ -15,22 +15,15 @@ package org.apache.blur.thrift; * See the License for the specific language governing permissions and * limitations under the License. */ -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.List; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResults; -import org.apache.blur.thrift.generated.Facet; -import org.apache.blur.thrift.generated.Query; -import org.apache.blur.thrift.generated.Selector; -import org.apache.blur.thrift.generated.Blur.Iface; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; public class FacetTests extends BaseClusterTest { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java index 7768c6c..46b9173 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java @@ -42,7 +42,6 @@ public class SuiteCluster { } } - @SuppressWarnings({ "unchecked", "rawtypes" }) public static void setupMiniCluster() throws IOException { File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs();
