[
https://issues.apache.org/jira/browse/HBASE-16270?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15434902#comment-15434902
]
Enis Soztutar edited comment on HBASE-16270 at 8/24/16 1:14 PM:
----------------------------------------------------------------
LICENSE.vm issue is back for Hadoop-2.6.1+:
{code}
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ hbase-assembly
---
[ERROR] Error invoking method 'get(java.lang.Integer)' in java.util.ArrayList
at META-INF/LICENSE.vm[line 1671, column 8]
java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor141.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.doInvoke(UberspectImpl.java:395)
at
org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:384)
at
org.apache.velocity.runtime.parser.node.ASTIndex.execute(ASTIndex.java:149)
at
org.apache.velocity.runtime.parser.node.ASTReference.execute(ASTReference.java:280)
at
org.apache.velocity.runtime.parser.node.ASTReference.render(ASTReference.java:369)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.directive.Foreach.render(Foreach.java:420)
at
org.apache.velocity.runtime.parser.node.ASTDirective.render(ASTDirective.java:207)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.SimpleNode.render(SimpleNode.java:342)
at org.apache.velocity.Template.merge(Template.java:356)
at org.apache.velocity.Template.merge(Template.java:260)
at
org.apache.velocity.app.VelocityEngine.mergeTemplate(VelocityEngine.java:354)
at
org.apache.maven.plugin.resources.remote.ProcessRemoteResourcesMojo.processResourceBundles(ProcessRemoteResourcesMojo.java:1164)
at
org.apache.maven.plugin.resources.remote.ProcessRemoteResourcesMojo.execute(ProcessRemoteResourcesMojo.java:520)
at
org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:101)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:209)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145)
at
org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:84)
at
org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:59)
at
org.apache.maven.lifecycle.internal.LifecycleStarter.singleThreadedBuild(LifecycleStarter.java:183)
at
org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:161)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:320)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:156)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:537)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:196)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:289)
at
org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:229)
at
org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:415)
at
org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:356)
Caused by: java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at java.util.ArrayList.rangeCheck(ArrayList.java:635)
at java.util.ArrayList.get(ArrayList.java:411)
... 46 more
{code}
Fortunately does not affect this patch. I'll commit this shortly.
was (Author: enis):
LICENSE.vm issue is back for Hadoop-2.6.2+:
{code}
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ hbase-assembly
---
[ERROR] Error invoking method 'get(java.lang.Integer)' in java.util.ArrayList
at META-INF/LICENSE.vm[line 1671, column 8]
java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor141.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.doInvoke(UberspectImpl.java:395)
at
org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:384)
at
org.apache.velocity.runtime.parser.node.ASTIndex.execute(ASTIndex.java:149)
at
org.apache.velocity.runtime.parser.node.ASTReference.execute(ASTReference.java:280)
at
org.apache.velocity.runtime.parser.node.ASTReference.render(ASTReference.java:369)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.directive.Foreach.render(Foreach.java:420)
at
org.apache.velocity.runtime.parser.node.ASTDirective.render(ASTDirective.java:207)
at
org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72)
at
org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87)
at
org.apache.velocity.runtime.parser.node.SimpleNode.render(SimpleNode.java:342)
at org.apache.velocity.Template.merge(Template.java:356)
at org.apache.velocity.Template.merge(Template.java:260)
at
org.apache.velocity.app.VelocityEngine.mergeTemplate(VelocityEngine.java:354)
at
org.apache.maven.plugin.resources.remote.ProcessRemoteResourcesMojo.processResourceBundles(ProcessRemoteResourcesMojo.java:1164)
at
org.apache.maven.plugin.resources.remote.ProcessRemoteResourcesMojo.execute(ProcessRemoteResourcesMojo.java:520)
at
org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:101)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:209)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153)
at
org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145)
at
org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:84)
at
org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:59)
at
org.apache.maven.lifecycle.internal.LifecycleStarter.singleThreadedBuild(LifecycleStarter.java:183)
at
org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:161)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:320)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:156)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:537)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:196)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:289)
at
org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:229)
at
org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:415)
at
org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:356)
Caused by: java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at java.util.ArrayList.rangeCheck(ArrayList.java:635)
at java.util.ArrayList.get(ArrayList.java:411)
... 46 more
{code}
Fortunately does not affect this patch. I'll commit this shortly.
> When using region replicas, get thousands of occurrences of
> UnexpectedStateException: Current snapshot id is -1
> ---------------------------------------------------------------------------------------------------------------
>
> Key: HBASE-16270
> URL: https://issues.apache.org/jira/browse/HBASE-16270
> Project: HBase
> Issue Type: Bug
> Components: Replication
> Affects Versions: 1.1.2
> Reporter: Robert Yokota
> Attachments: HBASE-16270-branch-1.2.patch, HBASE-16270-master.patch
>
>
> We have an HBase (1.1.2) production cluster with 58 region servers and a
> staging cluster with 6 region servers.
> For both clusters, we enabled region replicas with the following settings:
> hbase.regionserver.storefile.refresh.period = 0
> hbase.region.replica.replication.enabled = true
> hbase.region.replica.replication.memstore.enabled = true
> hbase.master.hfilecleaner.ttl = 3600000
> hbase.master.loadbalancer.class =
> org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer
> hbase.meta.replica.count = 3
> hbase.regionserver.meta.storefile.refresh.period = 30000
> hbase.region.replica.wait.for.primary.flush = true
> hbase.region.replica.storefile.refresh.memstore.multiplier = 4
> We then altered our HBase tables to have REGION_REPLICATION => 2
> Both clusters got into a state where a few region servers were spewing the
> following error below in the HBase logs. In one instance the error occurred
> over 70K times. At this time, these region servers would see 10x write
> traffic (the hadoop.HBase.RegionServer.Server.writeRequestCount metric) and
> in some instances would crash.
> At the same time, we would see secondary regions move and then go into the
> "reads are disabled" state for extended periods.
> It appears there is a race condition where the DefaultMemStore::clearSnapshot
> method might be called more than once in succession. The first call would set
> snapshotId to -1, but the second call would throw an exception. It seems the
> second call should just return if the snapshotId is already -1, rather than
> throwing an exception.
> Thu Jul 21 08:38:50 UTC 2016,
> RpcRetryingCaller{globalStartTime=1469090201543, pause=100, retries=35},
> org.apache.hadoop.hbase.regionserver.UnexpectedStateException:
> org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current
> snapshot id is -1,passed 1469085004304
> at
> org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:187)
> at
> org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1054)
> at
> org.apache.hadoop.hbase.regionserver.HStore.access$500(HStore.java:128)
> at
> org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2270)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4487)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4388)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4191)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:776)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1655)
> at
> org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
> at java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
