Author: kturner Date: Fri May 11 17:11:55 2012 New Revision: 1337287 URL: http://svn.apache.org/viewvc?rev=1337287&view=rev Log: ACCUMULO-591 clear tablet location cache before computing input splits (merged from trunk)
Modified: accumulo/branches/1.4/docs/config.html accumulo/branches/1.4/src/ (props changed) accumulo/branches/1.4/src/core/ (props changed) accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java accumulo/branches/1.4/src/server/ (props changed) Modified: accumulo/branches/1.4/docs/config.html URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/docs/config.html?rev=1337287&r1=1337286&r2=1337287&view=diff ============================================================================== --- accumulo/branches/1.4/docs/config.html (original) +++ accumulo/branches/1.4/docs/config.html Fri May 11 17:11:55 2012 @@ -202,7 +202,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>master.port.client</td> <td><b><a href='#PORT'>port</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the master</td> <td><pre>9999</pre></td> <td>The port used for handling client connections on the master</td> </tr> @@ -324,7 +324,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>tserver.compaction.major.concurrent.max</td> <td><b><a href='#COUNT'>count</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>3</pre></td> <td>The maximum number of concurrent major compactions for a tablet server</td> </tr> @@ -338,7 +338,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>tserver.compaction.major.thread.files.open.max</td> <td><b><a href='#COUNT'>count</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>10</pre></td> <td>Max number of files a major compaction thread can open at once. </td> </tr> @@ -380,7 +380,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>tserver.logger.count</td> <td><b><a href='#COUNT'>count</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>2</pre></td> <td>The number of loggers that each tablet server should use.</td> </tr> @@ -422,7 +422,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>tserver.memory.maps.native.enabled</td> <td><b><a href='#BOOLEAN'>boolean</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>true</pre></td> <td>An in-memory data store for accumulo implemented in c++ that increases the amount of data accumulo can hold in memory and avoids Java GC pauses.</td> </tr> @@ -457,7 +457,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr class='highlight'> <td>tserver.port.client</td> <td><b><a href='#PORT'>port</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>9997</pre></td> <td>The port used for handling client connections on the tablet servers</td> </tr> @@ -478,7 +478,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>tserver.scan.files.open.max</td> <td><b><a href='#COUNT'>count</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the tserver</td> <td><pre>100</pre></td> <td>Maximum total map files that all tablets in a tablet server can open for scans. </td> </tr> @@ -558,7 +558,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr > <td>logger.port.client</td> <td><b><a href='#PORT'>port</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the logger</td> <td><pre>11224</pre></td> <td>The port used for write-ahead logger services</td> </tr> @@ -617,7 +617,7 @@ $HADOOP_HOME/lib/[^.].*.jar, <tr class='highlight'> <td>gc.port.client</td> <td><b><a href='#PORT'>port</a></b></td> - <td>yes but requires restart</td> + <td>yes but requires restart of the gc</td> <td><pre>50091</pre></td> <td>The listening port for the garbage collector's monitor service</td> </tr> Propchange: accumulo/branches/1.4/src/ ------------------------------------------------------------------------------ Merged /accumulo/trunk:r1337210 Propchange: accumulo/branches/1.4/src/core/ ------------------------------------------------------------------------------ Merged /accumulo/trunk/core:r1337210 Modified: accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1337287&r1=1337286&r2=1337287&view=diff ============================================================================== --- accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java (original) +++ accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java Fri May 11 17:11:55 2012 @@ -850,6 +850,8 @@ public class TableOperationsImpl extends Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>(); String tableId = Tables.getTableId(instance, tableName); TabletLocator tl = TabletLocator.getInstance(instance, credentials, new Text(tableId)); + // its possible that the cache could contain complete, but old information about a tables tablets... so clear it + tl.invalidateCache(); while (!tl.binRanges(Collections.singletonList(range), binnedRanges).isEmpty()) { if (!Tables.exists(instance, tableId)) throw new TableDeletedException(tableId); @@ -860,6 +862,7 @@ public class TableOperationsImpl extends // sleep randomly between 100 and 200ms UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); binnedRanges.clear(); + tl.invalidateCache(); } // group key extents to get <= maxSplits Modified: accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1337287&r1=1337286&r2=1337287&view=diff ============================================================================== --- accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java (original) +++ accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java Fri May 11 17:11:55 2012 @@ -1342,6 +1342,8 @@ public abstract class InputFormatBase<K, Instance instance = getInstance(job.getConfiguration()); String tableId = null; tl = getTabletLocator(job.getConfiguration()); + // its possible that the cache could contain complete, but old information about a tables tablets... so clear it + tl.invalidateCache(); while (!tl.binRanges(ranges, binnedRanges).isEmpty()) { if (!(instance instanceof MockInstance)) { if (tableId == null) @@ -1354,6 +1356,7 @@ public abstract class InputFormatBase<K, binnedRanges.clear(); log.warn("Unable to locate bins for specified ranges. Retrying."); UtilWaitThread.sleep(100 + (int) (Math.random() * 100)); // sleep randomly between 100 and 200 ms + tl.invalidateCache(); } } } catch (Exception e) { Propchange: accumulo/branches/1.4/src/server/ ------------------------------------------------------------------------------ Merged /accumulo/trunk/server:r1337210