Anubhav Kale created CASSANDRA-10580:
----------------------------------------
Summary: When mutations are dropped, the column family should be
printed / have a counter per column family
Key: CASSANDRA-10580
URL: https://issues.apache.org/jira/browse/CASSANDRA-10580
Project: Cassandra
Issue Type: New Feature
Components: Core
Environment: Production
Reporter: Anubhav Kale
Priority: Minor
Fix For: 2.1.x
In our production cluster, we are seeing a large number of dropped mutations.
It would be really helpful to see which column families are really affected by
this (either through logs or through a dedicated counter for every column
family).
I have made a hack in StorageProxy (below) to help us with this. I am happy to
extend this to a better solution (print the CF affected in as logger.debug and
then manually grep) if experts agree this additional detail would be helpful in
general. Any other suggestions are welcome.
private static abstract class LocalMutationRunnable implements Runnable
{
private final long constructionTime = System.currentTimeMillis();
private IMutation mutation;
public final void run()
{
if (System.currentTimeMillis() > constructionTime + 2000L)
{
long timeTaken = System.currentTimeMillis() - constructionTime;
logger.warn("Anubhav LocalMutationRunnable thread ran after " +
timeTaken);
try
{
for(ColumnFamily family :
this.mutation.getColumnFamilies())
{
if
(family.toString().toLowerCase().contains("udsuserdailysnapshot"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.USERDAILY);
}
else if
(family.toString().toLowerCase().contains("udsuserhourlysnapshot"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.USERHOURLY);
}
else if
(family.toString().toLowerCase().contains("udstenantdailysnapshot"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.TENANTDAILY);
}
else if
(family.toString().toLowerCase().contains("udstenanthourlysnapshot"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.TENANTHOURLY);
}
else if
(family.toString().toLowerCase().contains("userdatasetraw"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.USERDSRAW);
}
else if
(family.toString().toLowerCase().contains("tenants"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.TENANTS);
}
else if
(family.toString().toLowerCase().contains("users"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.USERS);
}
else if
(family.toString().toLowerCase().contains("tenantactivity"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.TENANTACTIVITY);
}
else if
(family.getKeySpaceName().toLowerCase().contains("system"))
{
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.SYSTEMKS);
}
else
{
logger.warn("Anubhav LocalMutationRunnable
updating mutations for " + family.toString().toLowerCase());
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.OTHERTBL);
}
}
}
catch (Exception e)
{
logger.error("Anubhav LocalMutationRunnable Exception
", e);
}
MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.MUTATION);
HintRunnable runnable = new
HintRunnable(FBUtilities.getBroadcastAddress())
{
protected void runMayThrow() throws Exception
{
LocalMutationRunnable.this.runMayThrow();
}
};
submitHint(runnable);
return;
}
try
{
runMayThrow();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public LocalMutationRunnable(IMutation mutation)
{
this.mutation = mutation;
}
abstract protected void runMayThrow() throws Exception;
}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
