Author: ecn
Date: Thu Apr 19 18:41:17 2012
New Revision: 1328076
URL: http://svn.apache.org/viewvc?rev=1328076&view=rev
Log:
ACCUMULO-542: just use a scanner to read the root tablet
Added:
accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py (with
props)
Modified:
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Modified:
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1328076&r1=1328075&r2=1328076&view=diff
==============================================================================
---
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Thu Apr 19 18:41:17 2012
@@ -1032,10 +1032,6 @@ public class Tablet {
if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
return null;
- } else if
(extent.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
- SortedSet<Column> columns = new TreeSet<Column>();
- columns.add(Constants.METADATA_TIME_COLUMN.toColumn());
- entries = MetadataTable.getRootMetadataDataEntries(extent, columns,
SecurityConstants.systemCredentials);
} else {
entries = new TreeMap<Key,Value>();
Text rowName = extent.getMetadataEntry();
@@ -1077,44 +1073,39 @@ public class Tablet {
SortedMap<Key,Value> datafilesMetadata;
- if (extent.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
- datafilesMetadata =
MetadataTable.getRootMetadataDataFileEntries(extent,
SecurityConstants.systemCredentials);
+ Text rowName = extent.getMetadataEntry();
+
+ if (tabletsKeyValues != null && tabletsKeyValues.size() > 0) {
+ datafilesMetadata = new TreeMap<Key,Value>();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().compareRow(rowName) == 0 &&
entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY)
== 0) {
+ datafilesMetadata.put(new Key(entry.getKey()), new
Value(entry.getValue()));
+ }
+ }
} else {
- Text rowName = extent.getMetadataEntry();
+ ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(),
SecurityConstants.systemCredentials, Constants.METADATA_TABLE_ID,
+ Constants.NO_AUTHS);
- if (tabletsKeyValues != null && tabletsKeyValues.size() > 0) {
- datafilesMetadata = new TreeMap<Key,Value>();
- for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
- if (entry.getKey().compareRow(rowName) == 0 &&
entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY)
== 0) {
- datafilesMetadata.put(new Key(entry.getKey()), new
Value(entry.getValue()));
- }
- }
- } else {
-
- ScannerImpl mdScanner = new
ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.systemCredentials,
Constants.METADATA_TABLE_ID,
- Constants.NO_AUTHS);
-
- // Commented out because when no data file is present, each tablet
will scan through metadata table and return nothing
- // reduced batch size to improve performance
- // changed here after endKeys were implemented from 10 to 1000
- mdScanner.setBatchSize(1000);
-
- // leave these in, again, now using endKey for safety
-
mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
-
- mdScanner.setRange(new Range(rowName));
-
- datafilesMetadata = new TreeMap<Key,Value>();
+ // Commented out because when no data file is present, each tablet
will scan through metadata table and return nothing
+ // reduced batch size to improve performance
+ // changed here after endKeys were implemented from 10 to 1000
+ mdScanner.setBatchSize(1000);
+
+ // leave these in, again, now using endKey for safety
+ mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+
+ mdScanner.setRange(new Range(rowName));
+
+ datafilesMetadata = new TreeMap<Key,Value>();
+
+ for (Entry<Key,Value> entry : mdScanner) {
- for (Entry<Key,Value> entry : mdScanner) {
-
- if (entry.getKey().compareRow(rowName) != 0) {
- break;
- }
-
- datafilesMetadata.put(new Key(entry.getKey()), new
Value(entry.getValue()));
+ if (entry.getKey().compareRow(rowName) != 0) {
+ break;
}
+
+ datafilesMetadata.put(new Key(entry.getKey()), new
Value(entry.getValue()));
}
}
Modified:
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1328076&r1=1328075&r2=1328076&view=diff
==============================================================================
---
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++
accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Thu Apr 19 18:41:17 2012
@@ -304,13 +304,13 @@ public class MetadataTable extends org.a
* convenience method for reading a metadata tablet's data file entries from
the root tablet
*
*/
- public static SortedMap<Key,Value> getRootMetadataDataFileEntries(KeyExtent
extent, AuthInfo credentials) {
+ private static SortedMap<Key,Value> getRootMetadataDataFileEntries(KeyExtent
extent, AuthInfo credentials) {
SortedSet<Column> columns = new TreeSet<Column>();
columns.add(new
Column(TextUtil.getBytes(Constants.METADATA_DATAFILE_COLUMN_FAMILY), null,
null));
return getRootMetadataDataEntries(extent, columns, credentials);
}
- public static SortedMap<Key,Value> getRootMetadataDataEntries(KeyExtent
extent, SortedSet<Column> columns, AuthInfo credentials) {
+ private static SortedMap<Key,Value> getRootMetadataDataEntries(KeyExtent
extent, SortedSet<Column> columns, AuthInfo credentials) {
try {
SortedMap<Key,Value> entries = new TreeMap<Key,Value>();
Added: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py?rev=1328076&view=auto
==============================================================================
--- accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py (added)
+++ accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py Thu Apr 19
18:41:17 2012
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import unittest
+
+from TestUtils import TestUtilsMixin
+
+log = logging.getLogger('test.auto')
+
+class BigRootTablet(unittest.TestCase, TestUtilsMixin):
+ "ACCUMULO-542: A large root tablet will fail to load if it does't fit in
the tserver scan buffers"
+
+ order = 80
+
+ settings = TestUtilsMixin.settings.copy()
+ settings['table.scan.max.memory'] = '1024'
+ settings['tserver.compaction.major.delay'] = '60m'
+
+ def setUp(self):
+ TestUtilsMixin.setUp(self);
+
+ def runTest(self):
+ cmd = 'table !METADATA\naddsplits 0 1 2 3 4 5 6 7 8 9 a\n'
+ for i in range(10):
+ cmd += 'createtable %s\nflush -t !METADATA\n' % i
+ self.shell(self.masterHost(), cmd)
+ self.stop_accumulo()
+ self.start_accumulo()
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTest(BigRootTablet())
+ return result
Propchange: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
------------------------------------------------------------------------------
svn:executable = *
