Repository: hbase Updated Branches: refs/heads/master 38e983ed4 -> 61d10feff
http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java index 45b7d3c..4890f0d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java @@ -23,10 +23,12 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CompareOperator; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceStability; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -41,13 +43,17 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; import org.apache.hadoop.hbase.filter.ByteArrayComparable; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.Service; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.Service; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall; /** * Regions store data for a certain region of a table. It stores all columns @@ -105,7 +111,7 @@ public interface Region extends ConfigurationObserver { * <p>Use with caution. Exposed for use of fixup utilities. * @return a list of the Stores managed by this region */ - List<Store> getStores(); + List<? extends Store> getStores(); /** * Return the Store for the given family @@ -115,7 +121,7 @@ public interface Region extends ConfigurationObserver { Store getStore(byte[] family); /** @return list of store file names for the given families */ - List<String> getStoreFileList(byte [][] columns); + List<String> getStoreFileList(byte[][] columns); /** * Check the region's underlying store files, open the files that have not @@ -753,6 +759,18 @@ public interface Region extends ConfigurationObserver { */ CompactionState getCompactionState(); + /** + * Request compaction on this region. + */ + void requestCompaction(String why, int priority, CompactionLifeCycleTracker tracker, User user) + throws IOException; + + /** + * Request compaction for the given family + */ + void requestCompaction(byte[] family, String why, int priority, + CompactionLifeCycleTracker tracker, User user) throws IOException; + /** Wait for all current flushes and compactions of the region to complete */ void waitForFlushesAndCompactions(); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index bed2a7a..f282766 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.Region.Operation; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; import org.apache.hadoop.hbase.security.User; @@ -499,18 +500,18 @@ public class RegionCoprocessorHost /** * See * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, - * InternalScanner, CompactionRequest, long)} + * InternalScanner, CompactionLifeCycleTracker, long)} */ - public InternalScanner preCompactScannerOpen(final Store store, - final List<StoreFileScanner> scanners, final ScanType scanType, final long earliestPutTs, - final CompactionRequest request, final User user, final long readPoint) throws IOException { + public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners, + ScanType scanType, long earliestPutTs, CompactionLifeCycleTracker tracker, User user, + long readPoint) throws IOException { return execOperationWithResult(null, coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>(user) { @Override public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException { setResult(oserver.preCompactScannerOpen(ctx, store, scanners, scanType, - earliestPutTs, getResult(), request, readPoint)); + earliestPutTs, getResult(), tracker, readPoint)); } }); } @@ -520,17 +521,17 @@ public class RegionCoprocessorHost * available candidates. * @param store The store where compaction is being requested * @param candidates The currently available store files - * @param request custom compaction request + * @param tracker used to track the life cycle of a compaction * @return If {@code true}, skip the normal selection process and use the current list * @throws IOException */ - public boolean preCompactSelection(final Store store, final List<StoreFile> candidates, - final CompactionRequest request, final User user) throws IOException { + public boolean preCompactSelection(Store store, List<StoreFile> candidates, + CompactionLifeCycleTracker tracker, User user) throws IOException { return execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) { @Override public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException { - oserver.preCompactSelection(ctx, store, candidates, request); + oserver.preCompactSelection(ctx, store, candidates, tracker); } }); } @@ -540,21 +541,17 @@ public class RegionCoprocessorHost * candidates. * @param store The store where compaction is being requested * @param selected The store files selected to compact - * @param request custom compaction + * @param tracker used to track the life cycle of a compaction */ - public void postCompactSelection(final Store store, final ImmutableList<StoreFile> selected, - final CompactionRequest request, final User user) { - try { - execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) { - @Override - public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) - throws IOException { - oserver.postCompactSelection(ctx, store, selected, request); - } - }); - } catch (IOException e) { - LOG.warn(e); - } + public void postCompactSelection(Store store, ImmutableList<StoreFile> selected, + CompactionLifeCycleTracker tracker, User user) throws IOException { + execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) { + @Override + public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) + throws IOException { + oserver.postCompactSelection(ctx, store, selected, tracker); + } + }); } /** @@ -562,18 +559,17 @@ public class RegionCoprocessorHost * @param store the store being compacted * @param scanner the scanner used to read store data during compaction * @param scanType type of Scan - * @param request the compaction that will be executed + * @param tracker used to track the life cycle of a compaction * @throws IOException */ - public InternalScanner preCompact(final Store store, final InternalScanner scanner, - final ScanType scanType, final CompactionRequest request, final User user) - throws IOException { + public InternalScanner preCompact(Store store, InternalScanner scanner, ScanType scanType, + CompactionLifeCycleTracker tracker, User user) throws IOException { return execOperationWithResult(false, scanner, coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>(user) { @Override public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException { - setResult(oserver.preCompact(ctx, store, getResult(), scanType, request)); + setResult(oserver.preCompact(ctx, store, getResult(), scanType, tracker)); } }); } @@ -582,16 +578,16 @@ public class RegionCoprocessorHost * Called after the store compaction has completed. * @param store the store being compacted * @param resultFile the new store file written during compaction - * @param request the compaction that is being executed + * @param tracker used to track the life cycle of a compaction * @throws IOException */ - public void postCompact(final Store store, final StoreFile resultFile, - final CompactionRequest request, final User user) throws IOException { + public void postCompact(Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, + User user) throws IOException { execOperation(coprocessors.isEmpty() ? null : new RegionOperation(user) { @Override public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException { - oserver.postCompact(ctx, store, resultFile, request); + oserver.postCompact(ctx, store, resultFile, tracker); } }); } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index b56c925..209fce8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -64,11 +64,6 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi List<WAL> getWALs() throws IOException; /** - * @return Implementation of {@link CompactionRequestor} or null. - */ - CompactionRequestor getCompactionRequester(); - - /** * @return Implementation of {@link FlushRequester} or null. */ FlushRequester getFlushRequester(); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java index 7281626..5ccd6e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java @@ -76,7 +76,7 @@ public abstract class RegionSplitPolicy extends Configured { if (explicitSplitPoint != null) { return explicitSplitPoint; } - List<Store> stores = region.getStores(); + List<HStore> stores = region.getStores(); byte[] splitPointFromLargestStore = null; long largestStoreSize = 0; http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 4bb31ae..b7e83bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.NavigableSet; +import java.util.Optional; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.CellComparator; @@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; @@ -51,7 +53,8 @@ import org.apache.hadoop.hbase.security.User; @InterfaceStability.Evolving public interface Store extends HeapSize, StoreConfigInformation, PropagatingConfigurationObserver { - /* The default priority for user-specified compaction requests. + /** + * The default priority for user-specified compaction requests. * The user gets top priority unless we have blocking compactions. (Pri <= 0) */ int PRIORITY_USER = 1; @@ -253,17 +256,12 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf */ CompactionProgress getCompactionProgress(); - CompactionContext requestCompaction() throws IOException; - - /** - * @deprecated see requestCompaction(int, CompactionRequest, User) - */ - @Deprecated - CompactionContext requestCompaction(int priority, CompactionRequest baseRequest) - throws IOException; + default Optional<CompactionContext> requestCompaction() throws IOException { + return requestCompaction(NO_PRIORITY, CompactionLifeCycleTracker.DUMMY, null); + } - CompactionContext requestCompaction(int priority, CompactionRequest baseRequest, User user) - throws IOException; + Optional<CompactionContext> requestCompaction(int priority, CompactionLifeCycleTracker tracker, + User user) throws IOException; void cancelRequestedCompaction(CompactionContext compaction); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java new file mode 100644 index 0000000..38fec7e --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionLifeCycleTracker.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; + +/** + * Used to track compaction execution. + */ [email protected](HBaseInterfaceAudience.COPROC) [email protected] +public interface CompactionLifeCycleTracker { + + static CompactionLifeCycleTracker DUMMY = new CompactionLifeCycleTracker() { + }; + + /** + * Called before compaction is executed by CompactSplitThread. + * <p> + * Requesting compaction on a region can lead to multiple compactions on different stores, so we + * will pass the {@link Store} in to tell you the store we operate on. + */ + default void beforeExecute(Store store) { + } + + /** + * Called after compaction is executed by CompactSplitThread. + * <p> + * Requesting compaction on a region can lead to multiple compactions on different stores, so we + * will pass the {@link Store} in to tell you the store we operate on. + */ + default void afterExecute(Store store) { + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java index 69e354b..e05c165 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java @@ -18,25 +18,21 @@ */ package org.apache.hadoop.hbase.regionserver.compactions; -import java.util.ArrayList; import java.util.Collection; import java.util.stream.Collectors; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceStability; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; -import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; +import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions; /** * This class holds all logical details necessary to run a compaction. */ [email protected]({ "coprocessor" }) [email protected] [email protected] public class CompactionRequest { // was this compaction promoted to an off-peak @@ -53,56 +49,18 @@ public class CompactionRequest { private String regionName = ""; private String storeName = ""; private long totalSize = -1L; + private CompactionLifeCycleTracker tracker = CompactionLifeCycleTracker.DUMMY; - /** - * This ctor should be used by coprocessors that want to subclass CompactionRequest. - */ - public CompactionRequest() { + public CompactionRequest(Collection<StoreFile> files) { this.selectionTime = EnvironmentEdgeManager.currentTime(); this.timeInNanos = System.nanoTime(); - } - - public CompactionRequest(Collection<StoreFile> files) { - this(); - Preconditions.checkNotNull(files); - this.filesToCompact = files; + this.filesToCompact = Preconditions.checkNotNull(files, "files for compaction can not null"); recalculateSize(); } public void updateFiles(Collection<StoreFile> files) { - this.filesToCompact = files; - recalculateSize(); - } - - /** - * Called before compaction is executed by CompactSplitThread; for use by coproc subclasses. - */ - public void beforeExecute() {} - - /** - * Called after compaction is executed by CompactSplitThread; for use by coproc subclasses. - */ - public void afterExecute() {} - - /** - * Combines the request with other request. Coprocessors subclassing CR may override - * this if they want to do clever things based on CompactionPolicy selection that - * is passed to this method via "other". The default implementation just does a copy. - * @param other Request to combine with. - * @return The result (may be "this" or "other"). - */ - public CompactionRequest combineWith(CompactionRequest other) { - this.filesToCompact = new ArrayList<>(other.getFiles()); - this.isOffPeak = other.isOffPeak; - this.isMajor = other.isMajor; - this.priority = other.priority; - this.selectionTime = other.selectionTime; - this.timeInNanos = other.timeInNanos; - this.regionName = other.regionName; - this.storeName = other.storeName; - this.totalSize = other.totalSize; + this.filesToCompact = Preconditions.checkNotNull(files, "files for compaction can not null"); recalculateSize(); - return this; } public Collection<StoreFile> getFiles() { @@ -168,6 +126,14 @@ public class CompactionRequest { : (isMajor ? DisplayCompactionType.MAJOR : DisplayCompactionType.ALL_FILES); } + public void setTracker(CompactionLifeCycleTracker tracker) { + this.tracker = tracker; + } + + public CompactionLifeCycleTracker getTracker() { + return tracker; + } + @Override public String toString() { String fsList = filesToCompact.stream().filter(f -> f.getReader() != null) @@ -186,12 +152,7 @@ public class CompactionRequest { * @param files files that should be included in the compaction */ private void recalculateSize() { - long sz = 0; - for (StoreFile sf : this.filesToCompact) { - StoreFileReader r = sf.getReader(); - sz += r == null ? 0 : r.length(); - } - this.totalSize = sz; + this.totalSize = filesToCompact.stream().map(StoreFile::getReader) + .mapToLong(r -> r != null ? r.length() : 0L).sum(); } } - http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index 0ca925e..e6d1935 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -338,14 +338,14 @@ public abstract class Compactor<T extends CellSink> { * @param readPoint the read point to help create scanner by Coprocessor if required. * @return Scanner override by coprocessor; null if not overriding. */ - protected InternalScanner preCreateCoprocScanner(final CompactionRequest request, - final ScanType scanType, final long earliestPutTs, final List<StoreFileScanner> scanners, - User user, final long readPoint) throws IOException { + protected InternalScanner preCreateCoprocScanner(CompactionRequest request, ScanType scanType, + long earliestPutTs, List<StoreFileScanner> scanners, User user, long readPoint) + throws IOException { if (store.getCoprocessorHost() == null) { return null; } return store.getCoprocessorHost().preCompactScannerOpen(store, scanners, scanType, - earliestPutTs, request, user, readPoint); + earliestPutTs, request.getTracker(), user, readPoint); } /** @@ -355,12 +355,13 @@ public abstract class Compactor<T extends CellSink> { * @param scanner The default scanner created for compaction. * @return Scanner scanner to use (usually the default); null if compaction should not proceed. */ - protected InternalScanner postCreateCoprocScanner(final CompactionRequest request, - final ScanType scanType, final InternalScanner scanner, User user) throws IOException { + protected InternalScanner postCreateCoprocScanner(CompactionRequest request, ScanType scanType, + InternalScanner scanner, User user) throws IOException { if (store.getCoprocessorHost() == null) { return scanner; } - return store.getCoprocessorHost().preCompact(store, scanner, scanType, request, user); + return store.getCoprocessorHost().preCompact(store, scanner, scanType, request.getTracker(), + user); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index f7d70d2..c34fc6d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.replication.ReplicationEndpoint; @@ -1532,7 +1533,8 @@ public class AccessController implements MasterObserver, RegionObserver, RegionS @Override public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, - InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException { + InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) + throws IOException { requirePermission(getActiveUser(c), "compact", getTableName(c.getEnvironment()), null, null, Action.ADMIN, Action.CREATE); return scanner; http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp index c1e5aae..04aafa7 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/region.jsp @@ -92,7 +92,7 @@ </div> <% if(region != null) { // - List<Store> stores = region.getStores(); + List<? extends Store> stores = region.getStores(); for (Store store : stores) { String cf = store.getColumnFamilyName(); Collection<StoreFile> storeFiles = store.getStorefiles(); %> http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java index a99345b..76580f1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase; +import com.google.protobuf.Service; + import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collections; @@ -36,11 +38,8 @@ import org.apache.hadoop.hbase.client.locking.EntityLock; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.ipc.RpcServerInterface; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager; import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager; -import org.apache.hadoop.hbase.regionserver.CompactionRequestor; import org.apache.hadoop.hbase.regionserver.FlushRequester; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager; import org.apache.hadoop.hbase.regionserver.Leases; @@ -51,14 +50,14 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager; import org.apache.hadoop.hbase.regionserver.ServerNonceManager; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; -import com.google.protobuf.Service; - /** * Basic mock region server services. Should only be instantiated by HBaseTestingUtility.b */ @@ -160,11 +159,6 @@ public class MockRegionServerServices implements RegionServerServices { } @Override - public CompactionRequestor getCompactionRequester() { - return null; - } - - @Override public ClusterConnection getConnection() { return null; } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index 94d1cf6..85f65e8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -124,7 +124,7 @@ public class TestIOFencing { } @Override - public boolean compact(CompactionContext compaction, Store store, + public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController) throws IOException { try { return super.compact(compaction, store, throughputController); @@ -134,7 +134,7 @@ public class TestIOFencing { } @Override - public boolean compact(CompactionContext compaction, Store store, + public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController, User user) throws IOException { try { return super.compact(compaction, store, throughputController, user); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java index aff3d99..ac404bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreScanner; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -256,7 +256,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks { @Override public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, - InternalScanner s, CompactionRequest request, long readPoint) throws IOException { + InternalScanner s, CompactionLifeCycleTracker request, long readPoint) throws IOException { return createCompactorScanner(store, scanners, scanType, earliestPutTs); } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java index d5f3358..08ec09b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java @@ -784,7 +784,7 @@ public class TestBlockEvictionFromClient { } private BlockCache setCacheProperties(Region region) { - Iterator<Store> strItr = region.getStores().iterator(); + Iterator<? extends Store> strItr = region.getStores().iterator(); BlockCache cache = null; while (strItr.hasNext()) { Store store = strItr.next(); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index bef6f6b..023f8fc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -203,20 +204,20 @@ public class SimpleRegionObserver implements RegionObserver { @Override public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, - List<StoreFile> candidates, CompactionRequest request) throws IOException { + List<StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException { ctPreCompactSelect.incrementAndGet(); } @Override public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, - ImmutableList<StoreFile> selected, CompactionRequest request) { + ImmutableList<StoreFile> selected, CompactionLifeCycleTracker tracker) { ctPostCompactSelect.incrementAndGet(); } - @Override public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, - InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException { + InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) + throws IOException { ctPreCompact.incrementAndGet(); return scanner; } @@ -224,14 +225,14 @@ public class SimpleRegionObserver implements RegionObserver { @Override public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, - InternalScanner s, CompactionRequest request, long readPoint) throws IOException { + InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException { ctPreCompactScanner.incrementAndGet(); return s; } @Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, - StoreFile resultFile, CompactionRequest request) throws IOException { + StoreFile resultFile, CompactionLifeCycleTracker tracker) throws IOException { ctPostCompact.incrementAndGet(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java index 97c45ab..5cf0bb3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -194,13 +195,13 @@ public class TestCoprocessorInterface { } @Override public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, - Store store, InternalScanner scanner, ScanType scanType, CompactionRequest request) { + Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) { preCompactCalled = true; return scanner; } @Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, - Store store, StoreFile resultFile, CompactionRequest request) { + Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker) { postCompactCalled = true; } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java index 07fc179..0641b56 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -71,7 +71,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; @@ -417,7 +417,7 @@ public class TestRegionObserverInterface { @Override public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, - final InternalScanner scanner, final ScanType scanType, CompactionRequest request) { + InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) { return new InternalScanner() { @Override public boolean next(List<Cell> results) throws IOException { @@ -456,7 +456,7 @@ public class TestRegionObserverInterface { @Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, - StoreFile resultFile, CompactionRequest request) { + StoreFile resultFile, CompactionLifeCycleTracker tracker) { lastCompaction = EnvironmentEdgeManager.currentTime(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java index 8ab0175..9c06c3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl; @@ -61,6 +62,7 @@ import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; @@ -155,7 +157,7 @@ public class TestRegionObserverScannerOpenHook { @Override public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, - long earliestPutTs, InternalScanner s, CompactionRequest request, long readPoint) + long earliestPutTs, InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException { scanners.forEach(KeyValueScanner::close); return NO_DATA; @@ -252,7 +254,7 @@ public class TestRegionObserverScannerOpenHook { } @Override - public boolean compact(CompactionContext compaction, Store store, + public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController) throws IOException { boolean ret = super.compact(compaction, store, throughputController); if (ret) compactionStateChangeLatch.countDown(); @@ -260,7 +262,7 @@ public class TestRegionObserverScannerOpenHook { } @Override - public boolean compact(CompactionContext compaction, Store store, + public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController, User user) throws IOException { boolean ret = super.compact(compaction, store, throughputController, user); if (ret) compactionStateChangeLatch.countDown(); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 3ffa61b..ba18299 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -47,6 +47,27 @@ import org.apache.hadoop.hbase.client.locking.EntityLock; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.ipc.RpcServerInterface; +import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager; +import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager; +import org.apache.hadoop.hbase.regionserver.FlushRequester; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HeapMemoryManager; +import org.apache.hadoop.hbase.regionserver.Leases; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServer; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager; +import org.apache.hadoop.hbase.regionserver.ServerNonceManager; +import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; @@ -61,10 +82,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegion import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest; @@ -102,27 +123,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRespon import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; -import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager; -import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager; -import org.apache.hadoop.hbase.regionserver.CompactionRequestor; -import org.apache.hadoop.hbase.regionserver.FlushRequester; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HeapMemoryManager; -import org.apache.hadoop.hbase.regionserver.Leases; -import org.apache.hadoop.hbase.regionserver.MetricsRegionServer; -import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; -import org.apache.hadoop.hbase.regionserver.RegionServerServices; -import org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager; -import org.apache.hadoop.hbase.regionserver.ServerNonceManager; -import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; /** * A mock RegionServer implementation. @@ -315,12 +315,6 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { } @Override - public CompactionRequestor getCompactionRequester() { - // TODO Auto-generated method stub - return null; - } - - @Override public FlushRequester getFlushRequester() { // TODO Auto-generated method stub return null; http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 398d14d..12aed50 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -86,7 +86,7 @@ import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.security.EncryptionUtil; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -716,14 +716,14 @@ public class TestMobCompactor { } /** - * This copro overwrites the default compaction policy. It always chooses two latest - * hfiles and compacts them into a new one. + * This copro overwrites the default compaction policy. It always chooses two latest hfiles and + * compacts them into a new one. */ public static class CompactTwoLatestHfilesCopro implements RegionObserver { + @Override - public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c, - final Store store, final List<StoreFile> candidates, final CompactionRequest request) - throws IOException { + public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, + List<StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException { int count = candidates.size(); if (count >= 2) { http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index 5179b84..8015115 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -461,7 +462,7 @@ public class TestNamespaceAuditor { @Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, - StoreFile resultFile, CompactionRequest request) throws IOException { + StoreFile resultFile, CompactionLifeCycleTracker tracker) throws IOException { postCompact.countDown(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java index 823b1f7..0d07e1a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileSystemUtilizationChore.java @@ -36,9 +36,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; -import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -324,7 +324,7 @@ public class TestFileSystemUtilizationChore { final HRegionInfo info = mock(HRegionInfo.class); when(r.getRegionInfo()).thenReturn(info); List<Store> stores = new ArrayList<>(); - when(r.getStores()).thenReturn(stores); + when(r.getStores()).thenReturn((List) stores); for (Long storeSize : storeSizes) { final Store s = mock(Store.class); stores.add(s); @@ -338,7 +338,7 @@ public class TestFileSystemUtilizationChore { final HRegionInfo info = mock(HRegionInfo.class); when(r.getRegionInfo()).thenReturn(info); List<Store> stores = new ArrayList<>(); - when(r.getStores()).thenReturn(stores); + when(r.getStores()).thenReturn((List) stores); assertEquals( "Logic error, storeSizes and linkSizes must be equal in size", storeSizes.size(), hfileSizes.size()); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java index fcc3d4a..36c2e19 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.TestFromClientSideWithCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; /** @@ -61,7 +62,7 @@ public class NoOpScanPolicyObserver implements RegionObserver { public InternalScanner preCompactScannerOpen( final ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, - InternalScanner s, CompactionRequest request, long readPoint) throws IOException { + InternalScanner s, CompactionLifeCycleTracker tracker, long readPoint) throws IOException { // this demonstrates how to override the scanners default behavior ScanInfo oldSI = store.getScanInfo(); ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java index 0526462..a1fe87b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java @@ -18,11 +18,16 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.mockito.Matchers.*; -import static org.mockito.Mockito.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Optional; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.security.User; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -33,15 +38,23 @@ import org.mockito.stubbing.Answer; */ public class StatefulStoreMockMaker { // Add and expand the methods and answers as needed. - public CompactionContext selectCompaction() { return null; } - public void cancelCompaction(Object originalContext) {} - public int getPriority() { return 0; } + public Optional<CompactionContext> selectCompaction() { + return Optional.empty(); + } - private class SelectAnswer implements Answer<CompactionContext> { - public CompactionContext answer(InvocationOnMock invocation) throws Throwable { + public void cancelCompaction(Object originalContext) { + } + + public int getPriority() { + return 0; + } + + private class SelectAnswer implements Answer<Optional<CompactionContext>> { + public Optional<CompactionContext> answer(InvocationOnMock invocation) throws Throwable { return selectCompaction(); } } + private class PriorityAnswer implements Answer<Integer> { public Integer answer(InvocationOnMock invocation) throws Throwable { return getPriority(); @@ -53,15 +66,13 @@ public class StatefulStoreMockMaker { } } - public Store createStoreMock(String name) throws Exception { - Store store = mock(Store.class, name); - when(store.requestCompaction( - anyInt(), isNull(CompactionRequest.class))).then(new SelectAnswer()); - when(store.requestCompaction( - anyInt(), isNull(CompactionRequest.class), any(User.class))).then(new SelectAnswer()); + public HStore createStoreMock(String name) throws Exception { + HStore store = mock(HStore.class, name); + when(store.requestCompaction(anyInt(), any(CompactionLifeCycleTracker.class), any(User.class))) + .then(new SelectAnswer()); when(store.getCompactPriority()).then(new PriorityAnswer()); - doAnswer(new CancelAnswer()).when( - store).cancelRequestedCompaction(any(CompactionContext.class)); + doAnswer(new CancelAnswer()).when(store) + .cancelRequestedCompaction(any(CompactionContext.class)); return store; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index be078f2..3649823 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -33,8 +33,8 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor; import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; @@ -65,7 +66,6 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.junit.After; @@ -298,15 +298,16 @@ public class TestCompaction { Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files - Store store = r.getStore(COLUMN_FAMILY); + HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } CountDownLatch latch = new CountDownLatch(1); - TrackableCompactionRequest request = new TrackableCompactionRequest(latch); - thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request,null); + Tracker tracker = new Tracker(latch); + thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, tracker, + null); // wait for the latch to complete. latch.await(); @@ -322,7 +323,7 @@ public class TestCompaction { Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files - Store store = r.getStore(COLUMN_FAMILY); + HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) { createStoreFile(r); @@ -337,9 +338,9 @@ public class TestCompaction { long preFailedCount = metricsWrapper.getNumCompactionsFailed(); CountDownLatch latch = new CountDownLatch(1); - TrackableCompactionRequest request = new TrackableCompactionRequest(latch); + Tracker tracker = new Tracker(latch); thread.requestCompaction(mockRegion, store, "test custom comapction", Store.PRIORITY_USER, - request, null); + tracker, null); // wait for the latch to complete. latch.await(120, TimeUnit.SECONDS); @@ -370,20 +371,17 @@ public class TestCompaction { // setup a region/store with some files int numStores = r.getStores().size(); - List<Pair<CompactionRequest, Store>> requests = new ArrayList<>(numStores); CountDownLatch latch = new CountDownLatch(numStores); + Tracker tracker = new Tracker(latch); // create some store files and setup requests for each store on which we want to do a // compaction - for (Store store : r.getStores()) { + for (HStore store : r.getStores()) { createStoreFile(r, store.getColumnFamilyName()); createStoreFile(r, store.getColumnFamilyName()); createStoreFile(r, store.getColumnFamilyName()); - requests.add(new Pair<>(new TrackableCompactionRequest(latch), store)); + thread.requestCompaction(r, store, "test mulitple custom comapctions", Store.PRIORITY_USER, + tracker, null); } - - thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER, - Collections.unmodifiableList(requests), null); - // wait for the latch to complete. latch.await(); @@ -428,7 +426,7 @@ public class TestCompaction { } @Override - public synchronized CompactionContext selectCompaction() { + public synchronized Optional<CompactionContext> selectCompaction() { CompactionContext ctx = new TestCompactionContext(new ArrayList<>(notCompacting)); compacting.addAll(notCompacting); notCompacting.clear(); @@ -437,7 +435,7 @@ public class TestCompaction { } catch (IOException ex) { fail("Shouldn't happen"); } - return ctx; + return Optional.of(ctx); } @Override @@ -499,14 +497,14 @@ public class TestCompaction { } @Override - public CompactionContext selectCompaction() { + public Optional<CompactionContext> selectCompaction() { this.blocked = new BlockingCompactionContext(); try { this.blocked.select(null, false, false, false); } catch (IOException ex) { fail("Shouldn't happen"); } - return this.blocked; + return Optional.of(blocked); } @Override @@ -527,13 +525,13 @@ public class TestCompaction { } @Override - public Store createStoreMock(String name) throws Exception { + public HStore createStoreMock(String name) throws Exception { return createStoreMock(Integer.MIN_VALUE, name); } - public Store createStoreMock(int priority, String name) throws Exception { + public HStore createStoreMock(int priority, String name) throws Exception { // Override the mock to always return the specified priority. - Store s = super.createStoreMock(name); + HStore s = super.createStoreMock(name); when(s.getCompactPriority()).thenReturn(priority); return s; } @@ -555,7 +553,7 @@ public class TestCompaction { // Set up the region mock that redirects compactions. HRegion r = mock(HRegion.class); when( - r.compact(any(CompactionContext.class), any(Store.class), + r.compact(any(CompactionContext.class), any(HStore.class), any(ThroughputController.class), any(User.class))).then(new Answer<Boolean>() { @Override public Boolean answer(InvocationOnMock invocation) throws Throwable { @@ -568,7 +566,7 @@ public class TestCompaction { // Set up store mocks for 2 "real" stores and the one we use for blocking CST. ArrayList<Integer> results = new ArrayList<>(); StoreMockMaker sm = new StoreMockMaker(results), sm2 = new StoreMockMaker(results); - Store store = sm.createStoreMock("store1"), store2 = sm2.createStoreMock("store2"); + HStore store = sm.createStoreMock("store1"), store2 = sm2.createStoreMock("store2"); BlockingStoreMockMaker blocker = new BlockingStoreMockMaker(); // First, block the compaction thread so that we could muck with queue. @@ -691,24 +689,20 @@ public class TestCompaction { } /** - * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes. + * Simple {@link CompactionLifeCycleTracker} on which you can wait until the requested compaction + * finishes. */ - public static class TrackableCompactionRequest extends CompactionRequest { - private CountDownLatch done; - - /** - * Constructor for a custom compaction. Uses the setXXX methods to update the state of the - * compaction before being used. - */ - public TrackableCompactionRequest(CountDownLatch finished) { - super(); - this.done = finished; + public static class Tracker implements CompactionLifeCycleTracker { + + private final CountDownLatch done; + + public Tracker(CountDownLatch done) { + this.done = done; } @Override - public void afterExecute() { - super.afterExecute(); - this.done.countDown(); + public void afterExecute(Store store) { + done.countDown(); } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java index 2cea121..603203a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java @@ -28,6 +28,7 @@ import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.NavigableSet; +import java.util.Optional; import java.util.concurrent.ConcurrentSkipListSet; import javax.crypto.spec.SecretKeySpec; @@ -60,6 +61,7 @@ import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.security.EncryptionUtil; import org.apache.hadoop.hbase.security.User; @@ -536,8 +538,9 @@ public class TestHMobStore { // Trigger major compaction this.store.triggerMajorCompaction(); - CompactionContext requestCompaction = this.store.requestCompaction(1, null); - this.store.compact(requestCompaction, NoLimitThroughputController.INSTANCE, null); + Optional<CompactionContext> requestCompaction = + this.store.requestCompaction(Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); + this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null); Assert.assertEquals(1, this.store.getStorefiles().size()); //Check encryption after compaction http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java index ea14962..97f8ce3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.wal.TestWALActionsListener; import org.apache.hadoop.hbase.wal.WALEdit; @@ -254,7 +255,8 @@ public class TestHRegionServerBulkLoad { static int sleepDuration; @Override public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, - InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException { + InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker) + throws IOException { try { Thread.sleep(sleepDuration); } catch (InterruptedException ie) { http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java index f45c76c..707540a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy; @@ -417,7 +418,7 @@ public class TestMajorCompaction { } store.triggerMajorCompaction(); - CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null).getRequest(); + CompactionRequest request = store.requestCompaction().get().getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "System-requested major compaction should not occur if there are too many store files", @@ -436,7 +437,9 @@ public class TestMajorCompaction { createStoreFile(r); } store.triggerMajorCompaction(); - CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null).getRequest(); + CompactionRequest request = + store.requestCompaction(Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get() + .getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "User-requested major compaction should always occur, even if there are too many store files", http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 69bc9a7..3ba2299 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -31,6 +31,7 @@ import java.io.InterruptedIOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -70,7 +71,11 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; -import org.apache.hadoop.hbase.master.*; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterRpcServices; +import org.apache.hadoop.hbase.master.NoSuchProcedureException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.RegionStates; @@ -234,7 +239,7 @@ public class TestSplitTransactionOnCluster { assertEquals(1, cluster.getRegions(tableName).size()); HRegion region = cluster.getRegions(tableName).get(0); - Store store = region.getStore(cf); + HStore store = region.getStore(cf); int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName()); HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); @@ -246,8 +251,8 @@ public class TestSplitTransactionOnCluster { int fileNum = store.getStorefiles().size(); // 0, Compaction Request store.triggerMajorCompaction(); - CompactionContext cc = store.requestCompaction(); - assertNotNull(cc); + Optional<CompactionContext> cc = store.requestCompaction(); + assertTrue(cc.isPresent()); // 1, A timeout split // 1.1 close region assertEquals(2, region.close(false).get(cf).size()); @@ -255,7 +260,7 @@ public class TestSplitTransactionOnCluster { region.initialize(); // 2, Run Compaction cc - assertFalse(region.compact(cc, store, NoLimitThroughputController.INSTANCE)); + assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE)); assertTrue(fileNum > store.getStorefiles().size()); // 3, Split http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java index c959a22..78c5330 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java @@ -117,7 +117,7 @@ public class TestSplitWalDataLoss { } }).when(spiedRegion).internalFlushCacheAndCommit(Matchers.<WAL> any(), Matchers.<MonitoredTask> any(), Matchers.<PrepareFlushResult> any(), - Matchers.<Collection<Store>> any()); + Matchers.<Collection<HStore>> any()); // Find region key; don't pick up key for hbase:meta by mistake. String key = null; for (Map.Entry<String, Region> entry: rs.onlineRegions.entrySet()) { http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 3674303..2095dcd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.spy; @@ -47,7 +46,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -74,6 +72,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -105,12 +106,6 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.mockito.Mockito; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.FilterBase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - /** * Test class for the Store */ @@ -371,7 +366,7 @@ public class TestStore { // There will be no compaction due to threshold above. Last file will not be replaced. for (int i = 1; i <= storeFileNum - 1; i++) { // verify the expired store file. - assertNull(this.store.requestCompaction()); + assertFalse(this.store.requestCompaction().isPresent()); Collection<StoreFile> sfs = this.store.getStorefiles(); // Ensure i files are gone. if (minVersions == 0) { @@ -386,7 +381,7 @@ public class TestStore { // Let the next store file expired. edge.incrementTime(sleepTime); } - assertNull(this.store.requestCompaction()); + assertFalse(this.store.requestCompaction().isPresent()); Collection<StoreFile> sfs = this.store.getStorefiles(); // Assert the last expired file is not removed. @@ -422,7 +417,7 @@ public class TestStore { Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp - store.compact(store.requestCompaction(), NoLimitThroughputController.INSTANCE, null); + store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 54073bc..d25829d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -823,12 +823,12 @@ public abstract class AbstractTestWALReplay { final HRegion region = new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) { @Override - protected FlushResult internalFlushcache(final WAL wal, final long myseqid, - final Collection<Store> storesToFlush, MonitoredTask status, + protected FlushResultImpl internalFlushcache(final WAL wal, final long myseqid, + final Collection<HStore> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker) throws IOException { LOG.info("InternalFlushCache Invoked"); - FlushResult fs = super.internalFlushcache(wal, myseqid, storesToFlush, + FlushResultImpl fs = super.internalFlushcache(wal, myseqid, storesToFlush, Mockito.mock(MonitoredTask.class), writeFlushWalMarker); flushcount.incrementAndGet(); return fs; http://git-wip-us.apache.org/repos/asf/hbase/blob/61d10fef/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java index 656a0c7..d4d22b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreScanner; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -263,7 +264,7 @@ public class TestCoprocessorScanPolicy { public InternalScanner preCompactScannerOpen( final ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, - InternalScanner s, CompactionRequest request, long readPoint) throws IOException { + InternalScanner s,CompactionLifeCycleTracker tracker, long readPoint) throws IOException { Long newTtl = ttls.get(store.getTableName()); Integer newVersions = versions.get(store.getTableName()); ScanInfo oldSI = store.getScanInfo();
