vikrambohra commented on code in PR #3701:
URL: https://github.com/apache/gobblin/pull/3701#discussion_r1230152890
##########
gobblin-completeness/src/main/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifier.java:
##########
@@ -90,23 +102,52 @@ private static AuditCountClient getAuditClient(State
state) {
}
}
+ public Map<CompletenessType, Boolean> calculateCompleteness(String
datasetName, long beginInMillis, long endInMillis)
+ throws IOException {
+ return calculateCompleteness(datasetName, beginInMillis, endInMillis,
this.threshold);
+ }
+
/**
* Compare source tier against reference tiers.
- * Compute completion percentage by srcCount/refCount. Return true iff the
highest percentages is greater than threshold.
+ * Compute completion percentage which is true iff the calculated
percentages is greater than threshold.
*
* @param datasetName A dataset short name like 'PageViewEvent'
* @param beginInMillis Unix timestamp in milliseconds
* @param endInMillis Unix timestamp in milliseconds
* @param threshold User defined threshold
+ *
+ * @return a map of completeness result by CompletenessType
*/
- public boolean isComplete(String datasetName, long beginInMillis, long
endInMillis, double threshold)
- throws IOException {
- return getCompletenessPercentage(datasetName, beginInMillis, endInMillis)
> threshold;
+ public Map<CompletenessType, Boolean> calculateCompleteness(String
datasetName, long beginInMillis, long endInMillis,
+ double threshold) throws IOException {
+ Map<String, Long> countsByTier = getTierAndCount(datasetName,
beginInMillis, endInMillis);
+ log.info(String.format("checkTierCounts: audit counts map for %s for range
[%s,%s]", datasetName, beginInMillis, endInMillis));
+ countsByTier.forEach((x,y) -> log.info(String.format(" %s : %s ", x, y)));
+
+ Map<CompletenessType, Boolean> result = new HashMap<>();
+ result.put(CompletenessType.ClassicCompleteness,
CalculateCompleteness(datasetName, beginInMillis, endInMillis,
+ CompletenessType.ClassicCompleteness, countsByTier) > threshold);
+ result.put(CompletenessType.TotalCountCompleteness,
CalculateCompleteness(datasetName, beginInMillis, endInMillis,
+ CompletenessType.TotalCountCompleteness, countsByTier) > threshold);
+ return result;
}
- public boolean isComplete(String datasetName, long beginInMillis, long
endInMillis)
- throws IOException {
- return isComplete(datasetName, beginInMillis, endInMillis, this.threshold);
+ private double CalculateCompleteness(String datasetName, long beginInMillis,
long endInMillis, CompletenessType type,
Review Comment:
calculateCompleteness?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
