keith-turner commented on a change in pull request #168:
URL: https://github.com/apache/accumulo-testing/pull/168#discussion_r758692308
##########
File path:
src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
##########
@@ -86,51 +82,49 @@ private static int getFlushEntries(Properties props) {
return
Integer.parseInt(props.getProperty(TestProps.CI_INGEST_FLUSH_ENTRIES,
"1000000"));
}
- private static void pauseCheck(Properties props, Random rand) throws
InterruptedException {
- if (pauseEnabled(props)) {
+ private static void pauseCheck(Random rand) throws InterruptedException {
+ if (pauseEnabled) {
long elapsedNano = System.nanoTime() - lastPauseNs;
if (elapsedNano > (TimeUnit.SECONDS.toNanos(pauseWaitSec))) {
- long pauseDurationSec = getPause(props, rand,
CI_INGEST_PAUSE_DURATION_MIN,
- CI_INGEST_PAUSE_DURATION_MAX);
- log.info("PAUSING for " + pauseDurationSec + "s");
+ long pauseDurationSec = getPause(rand);
+ log.info("PAUSING for {}s", pauseDurationSec);
Thread.sleep(TimeUnit.SECONDS.toMillis(pauseDurationSec));
lastPauseNs = System.nanoTime();
- pauseWaitSec = getPause(props, rand, CI_INGEST_PAUSE_WAIT_MIN,
CI_INGEST_PAUSE_WAIT_MAX);
- log.info("INGESTING for " + pauseWaitSec + "s");
+ pauseWaitSec = getPause(rand);
+ log.info("INGESTING for {}s", pauseWaitSec);
}
}
}
public static void main(String[] args) throws Exception {
- try (ContinuousEnv env = new ContinuousEnv(args)) {
+ try (ContinuousEnv env = new ContinuousEnv(args);
+ AccumuloClient client = env.getAccumuloClient()) {
- visibilities =
parseVisibilities(env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES));
-
- long rowMin = env.getRowMin();
- long rowMax = env.getRowMax();
+ final long rowMin = env.getRowMin();
+ final long rowMax = env.getRowMax();
if (rowMin < 0 || rowMax < 0 || rowMax <= rowMin) {
throw new IllegalArgumentException("bad min and max");
}
- AccumuloClient client = env.getAccumuloClient();
String tableName = env.getAccumuloTableName();
if (!client.tableOperations().exists(tableName)) {
throw new TableNotFoundException(null, tableName,
"Consult the README and create the table before starting ingest.");
}
- BatchWriter bw = client.createBatchWriter(tableName);
-
- Random r = new Random();
+ Random rand = new Random();
byte[] ingestInstanceId = UUID.randomUUID().toString().getBytes(UTF_8);
+ log.info("Ingest instance ID: {} current time: {}ms", new
String(ingestInstanceId, UTF_8),
Review comment:
> Can use that to find the ingest logs if those logs contain the UUID.
The following is a general overview how one might investigate missing data
reported by the continuous ingest verify job.
* Pick one of the entries that points to a missing entry.
* Get the UUID and count from the value of the entry that points to the
missing entry.
* Use that UUID to find the continuous ingest client log that ingested the
data.
* Use the count to find a time range in the continuous ingest client log in
which the missing data was ingested.
* For that time range inspect the manager logs to see where the tablet was
assigned.
* Look at the tserver(s) logs related to that tablet for that time range.
There is some code in accumulo-testing that attempts to automate some
aspects of the above. However, that code usually does not work when you try to
use it because it parses logs messages that inevitably change.
##########
File path:
src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
##########
@@ -141,99 +135,96 @@ public static void main(String[] args) throws Exception {
long lastFlushTime = System.currentTimeMillis();
- int maxColF = env.getMaxColF();
- int maxColQ = env.getMaxColQ();
- boolean checksum =
Boolean.parseBoolean(env.getTestProperty(TestProps.CI_INGEST_CHECKSUM));
- long numEntries =
Long.parseLong(env.getTestProperty(TestProps.CI_INGEST_CLIENT_ENTRIES));
+ final int maxColF = env.getMaxColF();
+ final int maxColQ = env.getMaxColQ();
+ final boolean checksum = Boolean
+ .parseBoolean(testProps.getProperty(TestProps.CI_INGEST_CHECKSUM));
+ final long numEntries = Long
+
.parseLong(testProps.getProperty(TestProps.CI_INGEST_CLIENT_ENTRIES));
+ log.info("Total entries to be written: {}", numEntries);
- Properties testProps = env.getTestProperties();
- if (pauseEnabled(testProps)) {
+ visibilities =
parseVisibilities(testProps.getProperty(TestProps.CI_INGEST_VISIBILITIES));
+
+ pauseEnabled = pauseEnabled(testProps);
+
+ pauseMin =
Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_PAUSE_WAIT_MIN));
+ pauseMax =
Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_PAUSE_WAIT_MAX));
+ Preconditions.checkState(pauseMax >= pauseMin && pauseMin > 0);
Review comment:
could add an error message to the checkState call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
