jt2594838 commented on a change in pull request #372:
[IOTDB-198]Reimplementation sync module
URL: https://github.com/apache/incubator-iotdb/pull/372#discussion_r321550822
##########
File path:
server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
##########
@@ -950,11 +960,272 @@ protected void mergeEndAction(List<TsFileResource>
seqFiles, List<TsFileResource
logger.info("{} a merge task ends", storageGroupName);
}
+ /**
+ * Load a new tsfile to storage group processor
+ *
+ * Firstly, determine the loading type of the file, whether it needs to be
loaded in sequence list
+ * or unsequence list.
+ *
+ * Secondly, execute the loading process by the type.
+ *
+ * Finally, update the latestTimeForEachDevice and
latestFlushedTimeForEachDevice.
+ *
+ * @param newTsFile new tsfile
+ * @param newTsFileResource tsfile resource
+ * @UsedBy sync module.
+ */
+ public void loadNewTsFile(File newTsFile, TsFileResource newTsFileResource)
+ throws TsFileProcessorException {
+ writeLock();
+ mergeLock.writeLock().lock();
+ try {
+ boolean isOverlap = false;
+ int preIndex = -1, subsequentIndex = sequenceFileList.size();
+ // check new tsfile
+ outer:
+ for (int i = 0; i < sequenceFileList.size(); i++) {
+ if
(sequenceFileList.get(i).getFile().getName().equals(newTsFile.getName())) {
+ return;
+ }
+ if (i == sequenceFileList.size() - 1 &&
sequenceFileList.get(i).getEndTimeMap().isEmpty()) {
+ continue;
+ }
+ int preCnt = 0, subsequenceCnt = 0;
+ for (String device : newTsFileResource.getStartTimeMap().keySet()) {
+ if (sequenceFileList.get(i).getStartTimeMap().containsKey(device)) {
+ long startTime1 =
sequenceFileList.get(i).getStartTimeMap().get(device);
+ long endTime1 =
sequenceFileList.get(i).getEndTimeMap().get(device);
+ long startTime2 = newTsFileResource.getStartTimeMap().get(device);
+ long endTime2 = newTsFileResource.getEndTimeMap().get(device);
+ if (startTime1 > endTime2) {
+ subsequenceCnt++;
+ } else if (startTime2 > endTime1) {
+ preCnt++;
+ } else {
+ isOverlap = true;
+ break outer;
+ }
+ }
+ }
+ if (preCnt != 0 && subsequenceCnt != 0) {
+ isOverlap = true;
+ break;
+ }
+ if (preCnt == 0 && subsequenceCnt != 0) {
+ subsequentIndex = i;
+ break;
+ }
+ if (preCnt != 0 && subsequenceCnt == 0) {
+ preIndex = i;
+ }
+ }
+
+ // loading tsfile by type
+ if (isOverlap) {
+ loadTsFileByType(-1, newTsFile, newTsFileResource,
unSequenceFileList.size());
+ } else {
+ if (subsequentIndex != sequenceFileList.size()) {
+ loadTsFileByType(1, newTsFile, newTsFileResource, subsequentIndex);
+ } else {
+ if (preIndex != -1) {
+ loadTsFileByType(1, newTsFile, newTsFileResource, preIndex + 1);
+ } else {
+ loadTsFileByType(1, newTsFile, newTsFileResource,
sequenceFileList.size());
+ }
+ }
+ }
+
+ // update latest time map
+ updateLatestTimeMap(newTsFileResource);
+ } catch (TsFileProcessorException | DiskSpaceInsufficientException e) {
+ logger.error("Failed to append the tsfile {} to storage group processor
{}.",
+ newTsFile.getAbsolutePath(), newTsFile.getParentFile().getName());
+ throw new TsFileProcessorException(e);
+ } finally {
+ mergeLock.writeLock().unlock();
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Update latest time in latestTimeForEachDevice and
latestFlushedTimeForEachDevice.
+ *
+ * @UsedBy sync module
+ */
+ private void updateLatestTimeMap(TsFileResource newTsFileResource) {
+ for (Entry<String, Long> entry :
newTsFileResource.getEndTimeMap().entrySet()) {
+ String device = entry.getKey();
+ long endTime = newTsFileResource.getEndTimeMap().get(device);
+ if (!latestTimeForEachDevice.containsKey(device)
+ || latestTimeForEachDevice.get(device) < endTime) {
+ latestTimeForEachDevice.put(device, endTime);
+ }
+ if (!latestFlushedTimeForEachDevice.containsKey(device)
+ || latestFlushedTimeForEachDevice.get(device) < endTime) {
+ latestFlushedTimeForEachDevice.put(device, endTime);
+ }
+ }
+ }
+
+ /**
+ * Execute the loading process by the type.
+ *
+ * @param type load type: 1 sequence tsfile ; 2 unsequence tsfile
+ * @param tsFile tsfile to be loaded
+ * @param tsFileResource tsfile resource to be loaded
+ * @param index the index in sequenceFileList/unSequenceFileList
+ * @UsedBy sync module
+ */
+ private void loadTsFileByType(int type, File tsFile, TsFileResource
tsFileResource, int index)
+ throws TsFileProcessorException, DiskSpaceInsufficientException {
+ File targetFile;
+ if (type == -1) {
+ targetFile =
+ new
File(DirectoryManager.getInstance().getNextFolderForUnSequenceFile(),
+ tsFile.getParentFile().getName() + File.separatorChar +
tsFile.getName());
+ tsFileResource.setFile(targetFile);
+ unSequenceFileList.add(index, tsFileResource);
+ logger
+ .info("Load tsfile in unsequence list, move file from {} to {}",
tsFile.getAbsolutePath(),
+ targetFile.getAbsolutePath());
+ } else {
+ targetFile =
+ new
File(DirectoryManager.getInstance().getNextFolderForSequenceFile(),
+ tsFile.getParentFile().getName() + File.separatorChar +
getFileNameForLoadingFile(
+ tsFile.getName(), index));
+ tsFileResource.setFile(targetFile);
+ sequenceFileList.add(index, tsFileResource);
+ logger.info("Load tsfile in sequence list, move file from {} to {}",
tsFile.getAbsolutePath(),
+ targetFile.getAbsolutePath());
+ }
+
+ // move file from sync dir to data dir
+ if (!targetFile.getParentFile().exists()) {
+ targetFile.getParentFile().mkdirs();
+ }
+ if (tsFile.exists() && !targetFile.exists()) {
+ try {
+ FileUtils.moveFile(tsFile, targetFile);
+ } catch (IOException e) {
+ throw new TsFileProcessorException(String.format(
+ "File renaming failed when loading tsfile. Origin: %s, Target: %s",
+ tsFile.getAbsolutePath(), targetFile.getAbsolutePath()));
+ }
+ }
+ if (new File(tsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX).exists() && !new File(
+ targetFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX).exists()) {
+ try {
+ FileUtils.moveFile(new File(tsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX),
+ new File(targetFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX));
+ } catch (IOException e) {
+ throw new TsFileProcessorException(String.format(
+ "File renaming failed when loading .resource file. Origin: %s,
Target: %s",
+ new File(tsFile, TsFileResource.RESOURCE_SUFFIX).getAbsolutePath(),
+ new File(targetFile,
TsFileResource.RESOURCE_SUFFIX).getAbsolutePath()));
+ }
+ }
+ }
+
+ /**
+ * Get an appropriate filename to ensure the order between files
Review comment:
Define "appropriate" please.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
