Author: ecn
Date: Thu Aug 9 20:20:28 2012
New Revision: 1371431
URL: http://svn.apache.org/viewvc?rev=1371431&view=rev
Log:
ACCUMULO-722 prototype changes to run Accumulo against Accumulo-as-Namenode
Modified:
accumulo/branches/ACCUMULO-722/1.4/docs/config.html
accumulo/branches/ACCUMULO-722/1.4/pom.xml
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
accumulo/branches/ACCUMULO-722/1.4/src/server/pom.xml
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/tables/TableManager.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/Auditor.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Modified: accumulo/branches/ACCUMULO-722/1.4/docs/config.html
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/docs/config.html?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-722/1.4/docs/config.html (original)
+++ accumulo/branches/ACCUMULO-722/1.4/docs/config.html Thu Aug 9 20:20:28 2012
@@ -517,6 +517,13 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><pre>1G</pre></td>
<td>The maximum size for each write-ahead log. See comment for property
tserver.memory.maps.max</td>
</tr>
+ <tr >
+ <td>tserver.workq.threads</td>
+ <td><b><a href='#COUNT'>count</a></b></td>
+ <td>yes</td>
+ <td><pre>2</pre></td>
+ <td>The number of threads for the distributed workq. These threads are
used for copying failed bulk files.</td>
+ </tr>
<tr><td colspan='5'><a name='LOGGER_PREFIX'
class='large'>logger.*</a></td></tr>
<tr><td colspan='5'><i>Properties in this category affect the behavior of
the write-ahead logger servers</i></td></tr>
<tr><th>Property</th><th>Type</th><th>Zookeeper Mutable</th><th>Default
Value</th><th>Description</th></tr>
Modified: accumulo/branches/ACCUMULO-722/1.4/pom.xml
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/pom.xml?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-722/1.4/pom.xml (original)
+++ accumulo/branches/ACCUMULO-722/1.4/pom.xml Thu Aug 9 20:20:28 2012
@@ -106,7 +106,7 @@
<configuration>
<outputDirectory>../../lib</outputDirectory>
<!-- just grab the non-provided runtime dependencies -->
-
<includeArtifactIds>commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api</includeArtifactIds>
+
<includeArtifactIds>commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api,curator-framework,curator-client,guava</includeArtifactIds>
<excludeTransitive>true</excludeTransitive>
</configuration>
</execution>
@@ -636,7 +636,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.203.0</version>
+ <version>1.0.3</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -694,17 +694,17 @@
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
- <version>1.5</version>
+ <version>1.6</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
+ <version>2.1</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
- <version>3.2</version>
+ <version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
@@ -729,7 +729,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.3</version>
+ <version>1.4</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -751,6 +751,21 @@
<artifactId>commons-jci-fam</artifactId>
<version>1.0</version>
</dependency>
+ <dependency>
+ <groupId>com.netflix.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>1.1.15</version>
+ </dependency>
+ <dependency>
+ <groupId>com.netflix.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>1.1.15</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>12.0.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/Constants.java
Thu Aug 9 20:20:28 2012
@@ -37,7 +37,7 @@ public class Constants {
public static final String ZINSTANCES = "/instances";
public static final String ZTABLES = "/tables";
- public static final byte[] ZTABLES_INITIAL_ID = new byte[] {'0'};
+ public static final byte[] ZTABLES_INITIAL_ID = new byte[] {'3'};
public static final String ZTABLE_NAME = "/name";
public static final String ZTABLE_CONF = "/conf";
public static final String ZTABLE_STATE = "/state";
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
Thu Aug 9 20:20:28 2012
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -257,7 +258,10 @@ public class ZooKeeperInstance implement
try {
FileSystem fs =
FileUtil.getFileSystem(CachedConfiguration.getInstance(),
AccumuloConfiguration.getSiteConfiguration());
FileStatus[] files = fs.listStatus(instanceDirectory);
- log.debug("Trying to read instance id from " + instanceDirectory);
+ for (FileStatus file : files) {
+ log.info("Looking at files " + file.getPath() + " in instanceDirectory
" + instanceDirectory);
+ }
+ log.info("Trying to read instance id from " + instanceDirectory);
if (files == null || files.length == 0) {
log.error("unable obtain instance id at " + instanceDirectory);
throw new RuntimeException("Accumulo not initialized, there is no
instance id at " + instanceDirectory);
@@ -266,6 +270,7 @@ public class ZooKeeperInstance implement
throw new RuntimeException("Accumulo found multiple possible instance
ids in " + instanceDirectory);
} else {
String result = files[0].getPath().getName();
+ log.info("got instance id of " + result);
return result;
}
} catch (IOException e) {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
Thu Aug 9 20:20:28 2012
@@ -40,8 +40,11 @@ import org.apache.accumulo.core.master.s
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.log4j.Logger;
public class ConnectorImpl extends Connector {
+ static private final Logger log = Logger.getLogger(ConnectorImpl.class);
+
private Instance instance;
private AuthInfo credentials;
private SecurityOperations secops = null;
@@ -71,7 +74,7 @@ public class ConnectorImpl extends Conne
// hardcoded string for SYSTEM user since the definition is
// in server code
- if (!user.equals("!SYSTEM")) {
+ if (false && !user.equals("!SYSTEM")) {
ServerClient.execute(instance, new ClientExec<ClientService.Iface>() {
@Override
public void execute(ClientService.Iface iface) throws Exception {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
Thu Aug 9 20:20:28 2012
@@ -151,7 +151,7 @@ public class ServerClient {
if (!opened) {
if (!warnedAboutTServersBeingDown) {
if (servers.isEmpty()) {
- log.warn("There are no tablet servers: check that zookeeper and
accumulo are running.");
+ log.warn("There are no tablet servers: check that zookeeper and
accumulo are running.", new Throwable());
} else {
log.warn("Failed to find an available server in the list of
servers: " + servers);
}
Modified: accumulo/branches/ACCUMULO-722/1.4/src/server/pom.xml
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/pom.xml?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-722/1.4/src/server/pom.xml (original)
+++ accumulo/branches/ACCUMULO-722/1.4/src/server/pom.xml Thu Aug 9 20:20:28
2012
@@ -15,8 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
<parent>
<groupId>org.apache.accumulo</groupId>
@@ -100,6 +99,18 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.netflix.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.netflix.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c%2B%2B/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h
Thu Aug 9 20:20:28 2012
@@ -1,19 +1,3 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class org_apache_accumulo_server_tabletserver_NativeMap */
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c%2B%2B/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h
Thu Aug 9 20:20:28 2012
@@ -1,19 +1,3 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class
org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator */
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c%2B%2B/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h
Thu Aug 9 20:20:28 2012
@@ -1,19 +1,3 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class org_apache_accumulo_server_tabletserver_NativeMap_NMEntry
*/
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c%2B%2B/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h
Thu Aug 9 20:20:28 2012
@@ -1,19 +1,3 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class
org_apache_accumulo_server_tabletserver_NativeMap_NMIterator */
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c%2B%2B/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h
Thu Aug 9 20:20:28 2012
@@ -1,19 +1,3 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class
org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter */
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
Thu Aug 9 20:20:28 2012
@@ -106,13 +106,14 @@ public class LogReader {
LogFileKey key = new LogFileKey();
LogFileValue value = new LogFileValue();
- if (fs.isFile(path)) {
- // read log entries from a simple hdfs file
- org.apache.hadoop.io.SequenceFile.Reader reader = new
SequenceFile.Reader(fs, new Path(file), conf);
- while (reader.next(key, value)) {
- printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
- }
- } else if (local.isFile(path)) {
+// if (fs.isFile(path)) {
+// // read log entries from a simple hdfs file
+// org.apache.hadoop.io.SequenceFile.Reader reader = new
SequenceFile.Reader(fs, new Path(file), conf);
+// while (reader.next(key, value)) {
+// printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
+// }
+// } else
+ if (local.isFile(path)) {
// read log entries from a simple file
org.apache.hadoop.io.SequenceFile.Reader reader = new
SequenceFile.Reader(local, new Path(file), conf);
while (reader.next(key, value)) {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
Thu Aug 9 20:20:28 2012
@@ -326,17 +326,19 @@ public class CoordinateRecoveryTask impl
Configuration conf = CachedConfiguration.getInstance();
@SuppressWarnings("deprecation")
JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf));
- for (JobStatus status : jc.getAllJobs()) {
- if (!status.isJobComplete()) {
- RunningJob job = jc.getJob(status.getJobID());
- if (job.getJobName().equals(LogSort.getJobName())) {
- log.info("found a running " + job.getJobName());
- Configuration jobConfig = new Configuration(false);
- log.info("fetching configuration from " + job.getJobFile());
-
jobConfig.addResource(TraceFileSystem.wrap(FileUtil.getFileSystem(conf,
ServerConfiguration.getSiteConfiguration())).open(
- new Path(job.getJobFile())));
- if
(HdfsZooInstance.getInstance().getInstanceID().equals(jobConfig.get(LogSort.INSTANCE_ID_PROPERTY)))
{
- log.info("Killing job " + job.getID().toString());
+ if (jc != null) {
+ for (JobStatus status : jc.getAllJobs()) {
+ if (!status.isJobComplete()) {
+ RunningJob job = jc.getJob(status.getJobID());
+ if (job.getJobName().equals(LogSort.getJobName())) {
+ log.info("found a running " + job.getJobName());
+ Configuration jobConfig = new Configuration(false);
+ log.info("fetching configuration from " + job.getJobFile());
+
jobConfig.addResource(TraceFileSystem.wrap(FileUtil.getFileSystem(conf,
ServerConfiguration.getSiteConfiguration())).open(
+ new Path(job.getJobFile())));
+ if
(HdfsZooInstance.getInstance().getInstanceID().equals(jobConfig.get(LogSort.INSTANCE_ID_PROPERTY)))
{
+ log.info("Killing job " + job.getID().toString());
+ }
}
}
}
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/tables/TableManager.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/tables/TableManager.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/tables/TableManager.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/tables/TableManager.java
Thu Aug 9 20:20:28 2012
@@ -26,6 +26,8 @@ import java.util.Set;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeExistsPolicy;
@@ -60,7 +62,13 @@ public class TableManager {
String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES
+ "/" + tableId;
IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0],
existsPolicy);
+ String zTableConf = zTablePath + Constants.ZTABLE_CONF;
+ zoo.putPersistentData(zTableConf, new byte[0], existsPolicy);
+ String iter = Property.TABLE_ITERATOR_PREFIX.getKey();
+ for (String when : "scan,minc,majc".split(",")) {
+ zoo.putPersistentData(zTableConf + "/" + iter + when + ".vers", ("10," +
VersioningIterator.class.getName()).getBytes(), existsPolicy);
+ zoo.putPersistentData(zTableConf + "/" + iter + when +
".vers.opt.maxVersions", "1".getBytes(), existsPolicy);
+ }
zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME,
tableName.getBytes(), existsPolicy);
zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE,
state.name().getBytes(), existsPolicy);
zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID,
"0".getBytes(), existsPolicy);
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
Thu Aug 9 20:20:28 2012
@@ -255,6 +255,8 @@ public class DefaultServlet extends Basi
private void doAccumuloTable(StringBuilder sb) throws IOException {
// Accumulo
+ if (true)
+ return ;
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf,
ServerConfiguration.getSiteConfiguration()));
MasterMonitorInfo info = Monitor.getMmi();
@@ -294,37 +296,42 @@ public class DefaultServlet extends Basi
private void doHdfsTable(StringBuilder sb) throws IOException {
// HDFS
Configuration conf = CachedConfiguration.getInstance();
- DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(conf);
- String httpAddress = conf.get("dfs.http.address");
- String port = httpAddress.split(":")[1];
- String href = "http://"; + fs.getUri().getHost() + ":" + port;
- String liveUrl = href + "/dfsnodelist.jsp?whatNodes=LIVE";
- String deadUrl = href + "/dfsnodelist.jsp?whatNodes=DEAD";
- sb.append("<table>\n");
- sb.append("<tr><th colspan='2'><a href='" + href +
"'>NameNode</a></th></tr>\n");
- try {
- boolean highlight = false;
- tableRow(sb, (highlight = !highlight), "Unreplicated Capacity",
bytes(fs.getRawCapacity()));
- tableRow(sb, (highlight = !highlight), "% Used",
NumberType.commas(fs.getRawUsed() * 100. / fs.getRawCapacity(), 0, 90, 0, 100)
+ "%");
- tableRow(sb, (highlight = !highlight), "Corrupt Blocks",
NumberType.commas(fs.getCorruptBlocksCount(), 0, 0));
- DatanodeInfo[] liveNodes =
fs.getClient().datanodeReport(DatanodeReportType.LIVE);
- DatanodeInfo[] deadNodes =
fs.getClient().datanodeReport(DatanodeReportType.DEAD);
- tableRow(sb, (highlight = !highlight), "<a href='" + liveUrl +
"'>Live Data Nodes</a>", NumberType.commas(liveNodes.length));
- tableRow(sb, (highlight = !highlight), "<a href='" + deadUrl +
"'>Dead Data Nodes</a>", NumberType.commas(deadNodes.length));
- long count = 0;
- for (DatanodeInfo stat : liveNodes)
- count += stat.getXceiverCount();
- tableRow(sb, (highlight = !highlight), "Xceivers",
NumberType.commas(count));
- } catch (RemoteException ex) {
- sb.append("<tr><td colspan='2'>Permission Denied</td></tr>\n");
- } catch (Exception ex) {
- sb.append("<tr><td colspan='2'><span
class='error'>Down</span></td></tr>\n");
+ FileSystem fs = FileSystem.get(conf);
+ if (false && fs instanceof DistributedFileSystem) {
+ DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(conf);
+ String httpAddress = conf.get("dfs.http.address");
+ String port = httpAddress.split(":")[1];
+ String href = "http://"; + fs.getUri().getHost() + ":" + port;
+ String liveUrl = href + "/dfsnodelist.jsp?whatNodes=LIVE";
+ String deadUrl = href + "/dfsnodelist.jsp?whatNodes=DEAD";
+ sb.append("<table>\n");
+ sb.append("<tr><th colspan='2'><a href='" + href +
"'>NameNode</a></th></tr>\n");
+ try {
+ boolean highlight = false;
+ tableRow(sb, (highlight = !highlight), "Unreplicated Capacity",
bytes(dfs.getRawCapacity()));
+ tableRow(sb, (highlight = !highlight), "% Used",
NumberType.commas(dfs.getRawUsed() * 100. / dfs.getRawCapacity(), 0, 90, 0,
100) + "%");
+ tableRow(sb, (highlight = !highlight), "Corrupt Blocks",
NumberType.commas(dfs.getCorruptBlocksCount(), 0, 0));
+ DatanodeInfo[] liveNodes =
dfs.getClient().datanodeReport(DatanodeReportType.LIVE);
+ DatanodeInfo[] deadNodes =
dfs.getClient().datanodeReport(DatanodeReportType.DEAD);
+ tableRow(sb, (highlight = !highlight), "<a href='" + liveUrl +
"'>Live Data Nodes</a>", NumberType.commas(liveNodes.length));
+ tableRow(sb, (highlight = !highlight), "<a href='" + deadUrl +
"'>Dead Data Nodes</a>", NumberType.commas(deadNodes.length));
+ long count = 0;
+ for (DatanodeInfo stat : liveNodes)
+ count += stat.getXceiverCount();
+ tableRow(sb, (highlight = !highlight), "Xceivers",
NumberType.commas(count));
+ } catch (RemoteException ex) {
+ sb.append("<tr><td colspan='2'>Permission Denied</td></tr>\n");
+ } catch (Exception ex) {
+ sb.append("<tr><td colspan='2'><span
class='error'>Down</span></td></tr>\n");
+ }
+ sb.append("</table>\n");
}
- sb.append("</table>\n");
}
private void doJobTrackerTable(StringBuilder sb) {
// Job Tracker
+ if (true)
+ return ;
Configuration conf = CachedConfiguration.getInstance();
sb.append("<table>\n");
try {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/Auditor.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/Auditor.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/Auditor.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/Auditor.java
Thu Aug 9 20:20:28 2012
@@ -157,6 +157,7 @@ public class Auditor implements Authenti
@Override
public boolean hasTablePermission(AuthInfo credentials, String user, String
table, TablePermission permission) throws AccumuloSecurityException {
+ log.info("hasTablePermission " + credentials.user);
try {
boolean result = impl.hasTablePermission(credentials, user, table,
permission);
audit(credentials, "checked permission %s on table %s for %s",
permission, table, user);
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
Thu Aug 9 20:20:28 2012
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -131,6 +132,9 @@ public final class ZKAuthenticator imple
Map<String,Set<TablePermission>> tablePerms = new
HashMap<String,Set<TablePermission>>();
// Allow the root user to flush the !METADATA table
tablePerms.put(Constants.METADATA_TABLE_ID,
Collections.singleton(TablePermission.ALTER_TABLE));
+ for (String other : "0,1,2".split(",")) {
+ tablePerms.put(other, new
HashSet<TablePermission>(Arrays.asList(TablePermission.ALTER_TABLE,TablePermission.READ,
TablePermission.WRITE)));
+ }
constructUser(rootuser, Tool.createPass(rootpass), rootPerms,
tablePerms, Constants.NO_AUTHS);
}
log.info("Initialized root user with username: " + rootuser + " at the
request of user " + credentials.user);
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/MinorCompactor.java
Thu Aug 9 20:20:28 2012
@@ -88,7 +88,7 @@ public class MinorCompactor extends Comp
return ret;
} catch (IOException e) {
- log.warn("MinC failed (" + e.getMessage() + ") to create " +
getOutputFile() + " retrying ...");
+ log.warn("MinC failed (" + e.getMessage() + ") to create " +
getOutputFile() + " retrying ...", e);
ProblemReports.getInstance().report(new
ProblemReport(getExtent().getTableId().toString(), ProblemType.FILE_WRITE,
getOutputFile(), e));
reportedProblem = true;
} catch (RuntimeException e) {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Thu Aug 9 20:20:28 2012
@@ -205,7 +205,6 @@ import org.apache.thrift.TServiceClient;
import org.apache.thrift.server.TServer;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.mortbay.log.Log;
enum ScanRunState {
QUEUED, RUNNING, FINISHED
@@ -213,7 +212,9 @@ enum ScanRunState {
public class TabletServer extends AbstractMetricsImpl implements
org.apache.accumulo.server.tabletserver.metrics.TabletServerMBean {
private static final Logger log = Logger.getLogger(TabletServer.class);
-
+
+ static private final Set<String> METASET = new
HashSet<String>(Arrays.asList("!0,0,1,2".split(",")));
+
private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
private static long lastMemorySize = 0;
private static long gcTimeIncreasedCount;
@@ -1071,7 +1072,6 @@ public class TabletServer extends Abstra
throws NotServingTabletException, ThriftSecurityException,
org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
Authorizations userauths = null;
-
try {
if (!authenticator.hasTablePermission(credentials, credentials.user,
new String(textent.getTable()), TablePermission.READ))
throw new ThriftSecurityException(credentials.user,
SecurityErrorCode.PERMISSION_DENIED);
@@ -1462,7 +1462,7 @@ public class TabletServer extends Abstra
boolean containsMetadataTablet = false;
for (Tablet tablet : us.queuedMutations.keySet())
- if
(tablet.getExtent().getTableId().toString().equals(Constants.METADATA_TABLE_ID))
+ if (METASET.contains(tablet.getExtent().getTableId().toString()))
containsMetadataTablet = true;
if (!containsMetadataTablet && us.queuedMutations.size() > 0)
@@ -1649,7 +1649,7 @@ public class TabletServer extends Abstra
throw new NotServingTabletException(tkeyExtent);
}
- if
(!keyExtent.getTableId().toString().equals(Constants.METADATA_TABLE_ID))
+ if (!METASET.contains(keyExtent.getTableId().toString()))
TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
long opid = writeTracker.startWrite(TabletType.type(keyExtent));
@@ -1865,7 +1865,7 @@ public class TabletServer extends Abstra
}
}.start();
} else {
- if (extent.getTableId().compareTo(new
Text(Constants.METADATA_TABLE_ID)) == 0) {
+ if (METASET.contains(extent.getTableId().toString())) {
resourceManager.addMetaDataAssignment(ah);
} else {
resourceManager.addAssignment(ah);
@@ -2324,7 +2324,7 @@ public class TabletServer extends Abstra
public void run() {
log.info(clientAddress + ": got assignment from master: " + extent);
- final boolean isMetaDataTablet =
extent.getTableId().toString().compareTo(Constants.METADATA_TABLE_ID) == 0;
+ final boolean isMetaDataTablet =
METASET.contains(extent.getTableId().toString());
synchronized (unopenedTablets) {
synchronized (openingTablets) {
Modified:
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1371431&r1=1371430&r2=1371431&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
(original)
+++
accumulo/branches/ACCUMULO-722/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Thu Aug 9 20:20:28 2012
@@ -19,8 +19,11 @@ package org.apache.accumulo.server.util;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
import java.util.UUID;
import jline.ConsoleReader;
@@ -39,6 +42,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeMissingPolicy;
@@ -60,8 +64,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
/**
* This class is used to setup the directory structure and the root tablet to
get an instance started
@@ -212,6 +218,7 @@ public class Initialize {
// create an instance id
fs.mkdirs(ServerConstants.getInstanceIdLocation());
fs.createNewFile(new Path(ServerConstants.getInstanceIdLocation(),
uuid.toString()));
+ fs.mkdirs(new Path(ServerConstants.getRecoveryDir()));
// initialize initial metadata config in zookeeper
initMetadataConfig();
@@ -248,8 +255,8 @@ public class Initialize {
// populate the root tablet with info about the default tablet
// the root tablet contains the key extent and locations of all the
// metadata tablets
- String initRootTabFile = ServerConstants.getMetadataTableDir() +
"/root_tablet/00000_00000."
- +
FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
+ String suffix =
FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
+ String initRootTabFile = ServerConstants.getMetadataTableDir() +
"/root_tablet/00000_00000." + suffix;
FileSKVWriter mfw =
FileOperations.getInstance().openWriter(initRootTabFile, fs, conf,
AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
@@ -267,6 +274,11 @@ public class Initialize {
// ----------] table tablet info
Text tableExtent = new Text(KeyExtent.getMetadataEntry(new
Text(Constants.METADATA_TABLE_ID),
Constants.METADATA_RESERVED_KEYSPACE_START_KEY.getRow()));
+ // add a file that will contain references to the initial namespace
tables
+ String default_file = Constants.TABLE_TABLET_LOCATION + "/00000_00001."
+ suffix;
+ Key defaultFileKey = new Key(tableExtent,
Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(default_file), 0);
+ mfw.append(defaultFileKey, new Value("1,0".getBytes()));
+
// table tablet's directory
Key tableDirKey = new Key(tableExtent,
Constants.METADATA_DIRECTORY_COLUMN.getColumnFamily(),
Constants.METADATA_DIRECTORY_COLUMN.getColumnQualifier(), 0);
mfw.append(tableDirKey, new
Value(Constants.TABLE_TABLET_LOCATION.getBytes()));
@@ -274,7 +286,7 @@ public class Initialize {
// table tablet time
Key tableTimeKey = new Key(tableExtent,
Constants.METADATA_TIME_COLUMN.getColumnFamily(),
Constants.METADATA_TIME_COLUMN.getColumnQualifier(), 0);
mfw.append(tableTimeKey, new Value((TabletTime.LOGICAL_TIME_ID +
"0").getBytes()));
-
+
// table tablet's prevrow
Key tablePrevRowKey = new Key(tableExtent,
Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(),
Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier(),
0);
@@ -296,7 +308,40 @@ public class Initialize {
Key defaultPrevRowKey = new Key(defaultExtent,
Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(),
Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier(), 0);
mfw.append(defaultPrevRowKey,
KeyExtent.encodePrevEndRow(Constants.METADATA_RESERVED_KEYSPACE_START_KEY.getRow()));
+ mfw.close();
+
+ String defaultTabletFile = ServerConstants.getMetadataTableDir() +
default_file;
+ mfw = FileOperations.getInstance().openWriter(defaultTabletFile, fs,
conf, AccumuloConfiguration.getDefaultConfiguration());
+ mfw.startDefaultLocalityGroup();
+ Map<Key, Value> defaultData = new TreeMap<Key, Value>();
+ String ids[] = "0,1,2".split(",");
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < ids.length; i++) {
+ Text extent = new Text(KeyExtent.getMetadataEntry(new Text(ids[i]),
null));
+ defaultData.put(new Key(extent,
Constants.METADATA_TIME_COLUMN.getColumnFamily(),
Constants.METADATA_TIME_COLUMN.getColumnQualifier(), now),
+ new Value((TabletTime.MILLIS_TIME_ID + "0").getBytes()));
+ Key dirKey = new Key(extent,
Constants.METADATA_DIRECTORY_COLUMN.getColumnFamily(),
Constants.METADATA_DIRECTORY_COLUMN.getColumnQualifier(), now);
+ defaultData.put(dirKey, new
Value(Constants.DEFAULT_TABLET_LOCATION.getBytes()));
+ defaultData.put(new Key(extent,
Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(),
Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier(), now),
+ new Value(new byte[]{0}));
+ fs.mkdirs(new Path(ServerConstants.getTablesDir() + "/" + ids[i] +
Constants.DEFAULT_TABLET_LOCATION));
+ }
+ // Add the file entry for the namespace root entry
+ Text extent = new Text(KeyExtent.getMetadataEntry(new Text("0"), null));
+ String rootNSFile = Constants.DEFAULT_TABLET_LOCATION + "/00000_00002."
+ suffix;
+ defaultData.put(new Key(extent,
Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(rootNSFile), 0), new
Value("1,0".getBytes()));
+ for (Entry<Key,Value> entry : defaultData.entrySet()) {
+ mfw.append(entry.getKey(), entry.getValue());
+ }
+ mfw.close();
+ // write out the namespace data for / in the DistributedNameNode
+ String rootNSDataFile = ServerConstants.getTablesDir() + "/0" +
rootNSFile;
+ mfw = FileOperations.getInstance().openWriter(rootNSDataFile, fs, conf,
AccumuloConfiguration.getDefaultConfiguration());
+ mfw.startDefaultLocalityGroup();
+ Text infoFam = new Text("info");
+ mfw.append(new Key(new Text("/"), infoFam, new Text("create_time"),
now), new Value(Long.toString(now).getBytes()));
+ mfw.append(new Key(new Text("/"), infoFam, new Text("isDir"), now), new
Value("Y".getBytes()));
mfw.close();
}
@@ -328,39 +373,59 @@ public class Initialize {
return;
}
}
+
}
private static void initZooKeeper(String uuid, String instanceNamePath)
throws KeeperException, InterruptedException {
// setup basic data in zookeeper
- IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT, new
byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
- ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT +
Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP,
Ids.OPEN_ACL_UNSAFE);
+ ZooUtil.putPersistentData(ZooReaderWriter.getInstance().getZooKeeper(),
Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
+ ZooUtil.putPersistentData(ZooReaderWriter.getInstance().getZooKeeper(),
Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP,
Ids.OPEN_ACL_UNSAFE);
// setup instance name
if (clearInstanceName)
- zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
- zoo.putPersistentData(instanceNamePath, uuid.getBytes(),
NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().recursiveDelete(instanceNamePath,
NodeMissingPolicy.SKIP);
+ ZooReaderWriter.getInstance().putPersistentData(instanceNamePath,
uuid.getBytes(), NodeExistsPolicy.FAIL);
+
+ ZooKeeper keeper = ZooReaderWriter.getInstance().getZooKeeper();
+ for (String path : "/dnn,/dnn/datanodes,/dnn/blocks".split(",")) {
+ try {
+ keeper.create(path, new byte[0], Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
+ } catch (KeeperException.NodeExistsException e) {
+ // ignored
+ }
+ }
+ // wait for some datanodes to show up in zookeeper
+ log.info("Waiting for DFS DataNodes to start");
+ do {
+ List<String> children = keeper.getChildren("/dnn/datanodes", null);
+ if (children != null && !children.isEmpty())
+ break;
+ UtilWaitThread.sleep(250);
+ } while (true);
// setup the instance
String zkInstanceRoot = Constants.ZROOT + "/" + uuid;
- zoo.putPersistentData(zkInstanceRoot, new byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES,
Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot, new
byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
TableManager.prepareNewTableState(uuid, Constants.METADATA_TABLE_ID,
Constants.METADATA_TABLE_NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZTSERVERS, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZROOT_TABLET, new
byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZROOT_TABLET_WALOGS, new
byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZLOGGERS, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, new
byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE,
MasterGoalState.NORMAL.toString().getBytes(), NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, new byte[0],
NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, new
byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, new
byte[0], NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, new byte[]
{'0'}, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, "0", "namespace",
TableState.ONLINE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, "1", "blocks", TableState.ONLINE,
NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, "2", "datanodes",
TableState.ONLINE, NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZTSERVERS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZPROBLEMS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZROOT_TABLET, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZROOT_TABLET_WALOGS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZLOGGERS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZTRACERS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZMASTERS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZMASTER_LOCK, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(),
NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZGC, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZGC_LOCK, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZCONFIG, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZTABLE_LOCKS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZHDFS_RESERVATIONS, new byte[0], NodeExistsPolicy.FAIL);
+ ZooReaderWriter.getInstance().putPersistentData(zkInstanceRoot +
Constants.ZNEXT_FILE, new byte[] {'1'}, NodeExistsPolicy.FAIL);
}
private static String getInstanceNamePath() throws IOException,
KeeperException, InterruptedException {
