VladRodionov commented on a change in pull request #921: HBASE-22749:
Distributed MOB compactions
URL: https://github.com/apache/hbase/pull/921#discussion_r369938008
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
##########
@@ -967,85 +673,40 @@ public static boolean
isMobFileExpired(ColumnFamilyDescriptor column, long curre
}
/**
- * fill out partition id based on compaction policy and date, threshold...
- * @param id Partition id to be filled out
- * @param firstDayOfCurrentMonth The first day in the current month
- * @param firstDayOfCurrentWeek The first day in the current week
- * @param dateStr Date string from the mob file
- * @param policy Mob compaction policy
- * @param calendar Calendar object
- * @param threshold Mob compaciton threshold configured
- * @return true if the file needs to be excluded from compaction
+ * Gets encoded region name from a MOB file name
+ * @param mobFileName MOB file name
+ * @return encoded region name or null
*/
- public static boolean fillPartitionId(final CompactionPartitionId id,
- final Date firstDayOfCurrentMonth, final Date firstDayOfCurrentWeek,
final String dateStr,
- final MobCompactPartitionPolicy policy, final Calendar calendar, final
long threshold) {
-
- boolean skipCompcation = false;
- id.setThreshold(threshold);
- if (threshold <= 0) {
- id.setDate(dateStr);
- return skipCompcation;
+ public static String getEncodedRegionName(String mobFileName) {
+ int index = mobFileName.lastIndexOf(MobFileName.REGION_SEP);
+ if (index < 0) {
+ return null;
}
+ return mobFileName.substring(index + 1);
+ }
- long finalThreshold;
- Date date;
- try {
- date = MobUtils.parseDate(dateStr);
- } catch (ParseException e) {
- LOG.warn("Failed to parse date " + dateStr, e);
- id.setDate(dateStr);
- return true;
- }
+ /**
+ * Get list of referenced MOB files from a given collection of store files
+ * @param storeFiles store files
+ * @param mobDir MOB file directory
+ * @return list of MOB file paths
+ */
- /* The algorithm works as follows:
- * For monthly policy:
- * 1). If the file's date is in past months, apply 4 * 7 * threshold
- * 2). If the file's date is in past weeks, apply 7 * threshold
- * 3). If the file's date is in current week, exclude it from the
compaction
- * For weekly policy:
- * 1). If the file's date is in past weeks, apply 7 * threshold
- * 2). If the file's date in currently, apply threshold
- * For daily policy:
- * 1). apply threshold
- */
- if (policy == MobCompactPartitionPolicy.MONTHLY) {
- if (date.before(firstDayOfCurrentMonth)) {
- // Check overflow
- if (threshold < (Long.MAX_VALUE / MONTHLY_THRESHOLD_MULTIPLIER)) {
- finalThreshold = MONTHLY_THRESHOLD_MULTIPLIER * threshold;
- } else {
- finalThreshold = Long.MAX_VALUE;
- }
- id.setThreshold(finalThreshold);
+ public static List<Path> getReferencedMobFiles(Collection<HStoreFile>
storeFiles, Path mobDir) {
- // set to the date for the first day of that month
- id.setDate(MobUtils.formatDate(MobUtils.getFirstDayOfMonth(calendar,
date)));
- return skipCompcation;
+ Set<String> mobSet = new HashSet<String>();
+ for (HStoreFile sf : storeFiles) {
+ byte[] value = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);
+ if (value != null && value.length > 1) {
+ String s = Bytes.toString(value);
+ String[] all = s.split(",");
+ Collections.addAll(mobSet, all);
}
}
-
- if ((policy == MobCompactPartitionPolicy.MONTHLY) ||
- (policy == MobCompactPartitionPolicy.WEEKLY)) {
- // Check if it needs to apply weekly multiplier
- if (date.before(firstDayOfCurrentWeek)) {
- // Check overflow
- if (threshold < (Long.MAX_VALUE / WEEKLY_THRESHOLD_MULTIPLIER)) {
- finalThreshold = WEEKLY_THRESHOLD_MULTIPLIER * threshold;
- } else {
- finalThreshold = Long.MAX_VALUE;
- }
- id.setThreshold(finalThreshold);
-
- id.setDate(MobUtils.formatDate(MobUtils.getFirstDayOfWeek(calendar,
date)));
- return skipCompcation;
- } else if (policy == MobCompactPartitionPolicy.MONTHLY) {
- skipCompcation = true;
- }
+ List<Path> retList = new ArrayList<Path>();
+ for (String name : mobSet) {
+ retList.add(new Path(mobDir, name));
Review comment:
Created: https://issues.apache.org/jira/browse/HBASE-23723
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
