subject:"\[GitHub\] \[carbondata\] vikramahuja1001 commented on a change in pull request #4005\: \[CARBONDATA\-3978\] Trash Folder support in carbondata"

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-12-01 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533628020



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath.DataFileUtil;
+
+import org.apache.log4j.Logger;
+
+/**
+ *This util provide clean stale data methods for clean files command
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+  throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegmentFiles = new ArrayList<>();
+List redundantSegmentFile = new ArrayList<>();
+getStaleSegmentFiles(carbonTable, staleSegmentFiles, redundantSegmentFile);
+for (String staleSegmentFile : staleSegmentFiles) {
+  String segmentNumber = 
DataFileUtil.getSegmentNoFromSegmentFile(staleSegmentFile);
+  SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+  staleSegmentFile);
+  Map locationMap = 
fileStore.getSegmentFile()
+  .getLocationMap();
+  if (locationMap != null) {
+if (locationMap.entrySet().iterator().next().getValue().isRelative()) {
+  CarbonFile segmentPath = 
FileFactory.getCarbonFile(CarbonTablePath.getSegmentPath(
+  carbonTable.getTablePath(), segmentNumber));
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentPath, 
TrashUtil.getCompleteTrashFolderPath(
+  carbonTable.getTablePath(), timeStampForTrashFolder, 
segmentNumber));
+  // Deleting the stale Segment folders and the segment file.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentPath);
+// delete the segment file as well
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+staleSegmentFile));
+for (String duplicateStaleSegmentFile : redundantSegmentFile) {
+  if 
(DataFileUtil.getSegmentNoFromSegmentFile(duplicateStaleSegmentFile)
+  .equals(segmentNumber)) {
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable
+.getTablePath(), duplicateStaleSegmentFile));
+  }
+}
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentPath + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+}
+  }
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+  throws IOException {
+long

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-12-01 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533611032



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath  the path from which to copy the file
+   * @param destinationPath the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy   the files which are to be moved to the 
trash folder
+   * @param trashFolderWithTimestamp timestamp, partition folder(if any) and 
segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+String destinationPath = trashFolderWithTimestamp + CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName();
+try {
+  if (!FileFactory.isFileExist(destinationPath)) {
+copyToTrashFolder(filePathToCopy, destinationPath);
+  }
+} catch (IOException e) {
+  // in case there is any issue while copying the file to the trash 
folder, we need to delete
+  // the complete segment folder from the trash folder. The 
trashFolderWithTimestamp contains
+  // the segment folder too. Delete the folder as it is.
+  FileFactory.deleteFile(trashFolderWithTimestamp);
+  LOGGER.error("Error while checking trash folder: " + destinationPath + " 
or copying" +
+  " file: " + filePathToCopy + " to the trash folder at path", e);
+  throw e;
+}
+  }
+
+  /**
+   * The below method copies the complete segment folder to the trash folder. 
Here, the data files
+   * in segment are listed and copied one by one to the trash folder.
+   *
+   * @param segmentPath  the folder which are to be moved to the 
trash folder
+   * @param trashFolderWithTimestamp trashfolderpath with complete timestamp 
and segment number
+   * @return

Review comment:
   done

##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-12-01 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533261640



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.log4j.Logger;
+
+/**
+ *This util provide clean stale data methods for clean files command
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegmentFiles = getStaleSegmentFiles(carbonTable);
+for (String staleSegmentFile : staleSegmentFiles) {
+  String segmentNumber = 
CarbonTablePath.DataFileUtil.getSegmentNoFromSegmentFile(
+  staleSegmentFile);
+  SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+  staleSegmentFile);
+  Map locationMap = 
fileStore.getSegmentFile()
+  .getLocationMap();
+  if (locationMap != null) {
+// segmentLocation is the location in Fact/part0 where the segment is 
stored.
+CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,49 @@ public int getMaxSIRepairLimit(String dbName, String 
tableName) {
 return Math.abs(Integer.parseInt(thresholdValue));
   }
 
+  /**
+   * The below method returns the time(in milliseconds) for which timestamp 
folder retention in
+   * trash folder will take place.
+   */
+  public long getTrashFolderRetentionTime() {
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long)validateTrashFolderRetentionTime() * milliSecondsInADay;

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,49 @@ public int getMaxSIRepairLimit(String dbName, String 
tableName) {
 return Math.abs(Integer.parseInt(thresholdValue));
   }
 
+  /**
+   * The below method returns the time(in milliseconds) for which timestamp 
folder retention in
+   * trash folder will take place.
+   */
+  public long getTrashFolderRetentionTime() {
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long)validateTrashFolderRetentionTime() * milliSecondsInADay;
+  }
+
+  /**
+   * The below method returns the time(in days) for which timestamp folder 
retention in trash
+   * folder will take place
+   */
+  private int validateTrashFolderRetentionTime() {

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
##
@@ -183,16 +186,18 @@ private static boolean 
checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad,
 return true;
   }
   long deletionTime =

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-30 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532749779



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath  the path from which to copy the file
+   * @param destinationPath the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy   the files which are to be moved to the 
trash folder
+   * @param trashFolderWithTimestamp timestamp, partition folder(if any) and 
segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  String destinationPath = trashFolderWithTimestamp + CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName();
+  if (!FileFactory.isFileExist(destinationPath)) {
+copyToTrashFolder(filePathToCopy, destinationPath);
+  }
+} catch (IOException e) {
+  // in case there is any issue while copying the file to the trash 
folder, we need to delete
+  // the complete segment folder from the trash folder. The 
trashFolderWithTimestamp contains
+  // the segment folder too. Delete the folder as it is.
+  FileFactory.deleteFile(trashFolderWithTimestamp);
+  LOGGER.error("Error while creating trash folder or copying data to the 
trash folder", e);

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-30 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532539140



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-30 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532538210



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder : " + e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for each segment we get the indexfile first, then we get the 
carbondata file. Move both
+// of

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-30 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532537780



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,41 @@ public int getMaxSIRepairLimit(String dbName, String 
tableName) {
 return Math.abs(Integer.parseInt(thresholdValue));
   }
 
+  /**
+   * The below method returns the time(in milliseconds) for which timestamp 
folder retention in
+   * trash folder will take place.
+   */
+  public long getTrashFolderRetentionTime() {
+String propertyValue = 
getProperty(CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS, Integer
+.toString(CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT));
+int configuredValue = 0;
+try {
+  configuredValue = Integer.parseInt(propertyValue);
+  if (configuredValue < 0 || configuredValue > CarbonCommonConstants
+  .CARBON_TRASH_RETENTION_DAYS_MAXIMUM) {
+LOGGER.warn("Value of " + 
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS + " is" +
+" invalid, taking default value instead");
+configuredValue = 
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT;
+  }
+} catch (NumberFormatException e) {
+  LOGGER.error("Invalid value configured for " + CarbonCommonConstants
+  .CARBON_TRASH_RETENTION_DAYS + ", considering the default value");
+  configuredValue = 
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT;
+}
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long) configuredValue * milliSecondsInADay;
+  }

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-30 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532412599



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532396937



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),

Review comment:
   added in the same try- catch





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532396566



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532396388



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath the path from which to copy the file
+   * @param destinationPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+  throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {
+  FileFactory.mkdirs(trashFolderWithTimestamp);
+}
+if (!FileFactory.isFileExist(trashFolderWithTimestamp + 
CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName())) {
+  copyToTrashFolder(filePathToCopy, trashFolderWithTimestamp + 
CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName());
+}
+  }
+} catch (IOException e) {
+  // in case there is any issue while copying the file to the trash 
folder, we need to delete
+  // the complete segment folder from the trash folder. The 
trashFolderWithTimestamp contains
+  // the segment folder too. Delete the folder as it is.
+  FileFactory.deleteFile(trashFolderWithTimestamp);
+  LOGGER.error("Error while creating trash folder or copying data to the 
trash folder", e);
+  throw e;
+}
+  }
+
+  /**
+   * The below method copies the complete segment folder to the trash folder. 
Here, the data files
+   * in segment are listed and copied one by one to the trash folder.
+   *
+   * @param segmentPath the folder which are to be moved to the trash folder
+   * @param trashFolderWithTimestamp trashfolderpath with complete timestamp 
and segment number
+   * @return
+   */
+  public static void copySegmentToTrash(CarbonFile segmentPath,
+  String trashFolderWithTimestamp) throws IOException {
+try {
+  List dataFiles =

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532396184



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath the path from which to copy the file
+   * @param destinationPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+  throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {
+  FileFactory.mkdirs(trashFolderWithTimestamp);
+}
+if (!FileFactory.isFileExist(trashFolderWithTimestamp + 
CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName())) {
+  copyToTrashFolder(filePathToCopy, trashFolderWithTimestamp + 
CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName());
+}
+  }
+} catch (IOException e) {
+  // in case there is any issue while copying the file to the trash 
folder, we need to delete
+  // the complete segment folder from the trash folder. The 
trashFolderWithTimestamp contains
+  // the segment folder too. Delete the folder as it is.
+  FileFactory.deleteFile(trashFolderWithTimestamp);
+  LOGGER.error("Error while creating trash folder or copying data to the 
trash folder", e);
+  throw e;
+}
+  }
+
+  /**
+   * The below method copies the complete segment folder to the trash folder. 
Here, the data files
+   * in segment are listed and copied one by one to the trash folder.
+   *
+   * @param segmentPath the folder which are to be moved to the trash folder
+   * @param trashFolderWithTimestamp trashfolderpath with complete timestamp 
and segment number
+   * @return
+   */
+  public static void copySegmentToTrash(CarbonFile segmentPath,
+  String trashFolderWithTimestamp) throws IOException {
+try {
+  List dataFiles =

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532395502



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath the path from which to copy the file
+   * @param destinationPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+  throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {
+  FileFactory.mkdirs(trashFolderWithTimestamp);
+}
+if (!FileFactory.isFileExist(trashFolderWithTimestamp + 
CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName())) {
+  copyToTrashFolder(filePathToCopy, trashFolderWithTimestamp + 
CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName());
+}

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532370448



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param sourcePath the path from which to copy the file
+   * @param destinationPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String sourcePath, String 
destinationPath)
+  throws IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(destinationPath);
+  dataInputStream = FileFactory.getDataInputStream(sourcePath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + sourcePath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532364299



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532364095



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532363133



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532360958



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532359426



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532357875



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532357801



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder. Please delete them manually : " + 
e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();

Review comment:
   yes, can be removed

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-29 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532357687



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531552014



##
File path: 
integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
##
@@ -91,6 +96,14 @@ case class CarbonCleanFilesCommand(
 OperationListenerBus.getInstance.fireEvent(cleanFilesPreEvent, 
operationContext)
 if (tableName.isDefined) {
   Checker.validateTableExists(databaseNameOp, tableName.get, sparkSession)
+  if (forceClean) {
+// empty the trash folder
+TrashUtil.emptyTrash(carbonTable.getTablePath)
+  } else {
+// clear trash based on timestamp
+TrashUtil.deleteExpiredDataFromTrash(carbonTable.getTablePath)
+  }
+  CleanFilesUtil.cleanStaleSegments(carbonTable)

Review comment:
   i don't understand , what you mean here. We are cleaning the stale 
segments here for the first time. We delete the data in trash first based on 
timestamp expiration and then proceed with cleaning of stale segments and then 
do regular clean files operation for MFD/Compacted segments





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531551083



##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFileCommand.scala
##
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFileCommand extends QueryTest with BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = CarbonTablePath.getTrashFolderPath(path)
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+assert(!FileFactory.isFileExist(trashFolderPath))
+// no carbondata file is added to the trash
+assert(getFileCountInTrashFolder(trashFolderPath) == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with Marked For Delete segments") 
{
+// do not send MFD folders to trash
+createTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = CarbonTablePath.getTrashFolderPath(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+sql(s"""Delete from table cleantest where segment.id in(1)""")
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 1)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+// no carbondata file is added to the trash
+assert(getFileCountInTrashFolder(trashFolderPath) == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with compaction") {
+// do not send compacted folders to trash
+createTable()
+loadData()
+sql(s"""ALTER TABLE CLEANTEST COMPACT "MINOR" """)
+
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = CarbonTablePath.getTrashFolderPath(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 4)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+// no carbondata file is added to the trash
+assert(getFileCountInTrashFolder(trashFolderPath) == 0)
+
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with stale segments") {
+createTable()
+loadData()
+sql(s"""alter table cleantest compact 'minor'""")
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+sql(s"""INSERT INTO CLEANTEST SELECT "abc", 2, "name)
+checkAnswer(sql(s"""select count(*) from cleantest"""),
+  Seq(Row(5)))
+val path = CarbonEnv.getCarbonTable(Some("default"),

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531550965



##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFilesCommandPartitionTable.scala
##
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFilesCommandPartitionTable extends QueryTest with 
BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+assert(!FileFactory.isFileExist(trashFolderPath))
+val list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with Marked For Delete segments") 
{
+// do not send MFD folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+assert(!FileFactory.isFileExist(trashFolderPath))
+sql(s"""Delete from table cleantest where segment.id in(1)""")
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 1)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+var list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with compaction") {
+// do not send compacted folders to trash
+createParitionTable()
+loadData()
+sql(s"""ALTER TABLE CLEANTEST COMPACT "MINOR" """)
+
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 4)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+val list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+
+
+  test("test trash folder with 2 segments with same segment number") {
+createParitionTable()
+sql(s"""INSERT INTO CLEANTEST SELECT 1, 2,"hello","abc)
+
+val path = CarbonEnv.getCarbonTable(Some("default"),

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531548951



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder : " + e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for each segment we get the indexfile first, then we get the 
carbondata file. Move both
+// of

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531495145



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder : " + e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for each segment we get the indexfile first, then we get the 
carbondata file. Move both
+// of

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531455699



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param fromPath the path from which to copy the file
+   * @param toPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String fromPath, String toPath) throws 
IOException {

Review comment:
   done

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
##
@@ -792,4 +795,9 @@ public static String getParentPath(String dataFilePath) {
   return dataFilePath;
 }
   }
+
+  public static String getTrashFolderPath(String carbonTablePath) {
+return carbonTablePath + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .TRASH_DIR;

Review comment:
   done

##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFileCommand.scala
##
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFileCommand extends QueryTest with BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = CarbonTablePath.getTrashFolderPath(path)
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531455538



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(
+  carbonTable.getTablePath()) + 
CarbonCommonConstants.FILE_SEPARATOR +
+  timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath
+  .SEGMENT_PREFIX + segmentNumber);
+  // Deleting the stale Segment folders.
+  try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+  } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + " 
from after moving" +
+" it to the trash folder : " + e.getMessage(), e);
+  }
+  // delete the segment file as well
+  
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+  staleSegment));
+}
+  }
+  staleSegments.clear();
+}
+  }
+
+  /**
+   * This method will clean all the stale segments for partition table, delete 
the source folders
+   * after copying the data to the trash and also remove the .segment files of 
the stale segments
+   */
+  public static void cleanStaleSegmentsForPartitionTable(CarbonTable 
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for each segment we get the indexfile first, then we get the 
carbondata file. Move both
+// of

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531455280



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for a table, delete the 
source folder after
+   * copying the data to the trash and also remove the .segment files of the 
stale segments
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = System.currentTimeMillis();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap = 
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+  CarbonFile segmentLocation = 
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+  CarbonCommonConstants.FILE_SEPARATOR + 
fileStore.getSegmentFile().getLocationMap()
+  .entrySet().iterator().next().getKey());
+  // copy the complete segment to the trash folder
+  TrashUtil.copySegmentToTrash(segmentLocation, 
CarbonTablePath.getTrashFolderPath(

Review comment:
   need to fail the clean files command, if copy fails. so directly 
throwing exception if anything fails.

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-27 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r531454877



##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFilesCommandPartitionTable.scala
##
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFilesCommandPartitionTable extends QueryTest with 
BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+var list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with Marked For Delete segments") 
{
+// do not send MFD folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+assert(!FileFactory.isFileExist(trashFolderPath))
+sql(s"""Delete from table cleantest where segment.id in(1)""")
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 1)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+var list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash

Review comment:
   changed

##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,34 @@ public int getMaxSIRepairLimit(String dbName, String 
tableName) {
 return Math.abs(Integer.parseInt(thresholdValue));
   }
 
+  /**
+   * The below method returns the time(in milliseconds) for which timestamp 
folder retention in
+   * trash folder will take place.
+   */
+  public long getTrashFolderRetentionTime() {
+String propertyValue = 
getProperty(CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS);

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530973967



##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFileCommand.scala
##
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFileCommand extends QueryTest with BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0

Review comment:
   yes, removed.





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530973423



##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFilesCommandPartitionTable.scala
##
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFilesCommandPartitionTable extends QueryTest with 
BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+editTableStatusFile(path)
+assert(!FileFactory.isFileExist(trashFolderPath))
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == 4)
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(0 == segmentNumber2)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+var list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with Marked For Delete segments") 
{
+// do not send MFD folders to trash
+createParitionTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+assert(!FileFactory.isFileExist(trashFolderPath))
+sql(s"""Delete from table cleantest where segment.id in(1)""")
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 1)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+var list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+  test("clean up table and test trash folder with compaction") {
+// do not send compacted folders to trash
+createParitionTable()
+loadData()
+sql(s"""ALTER TABLE CLEANTEST COMPACT "MINOR" """)
+
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR
+assert(!FileFactory.isFileExist(trashFolderPath))
+
+val segmentNumber1 = sql(s"""show segments for table cleantest""").count()
+sql(s"CLEAN FILES FOR TABLE cleantest").show
+val segmentNumber2 = sql(s"""show segments for table cleantest""").count()
+assert(segmentNumber1 == segmentNumber2 + 4)
+assert(!FileFactory.isFileExist(trashFolderPath))
+count = 0
+val list = getFileCountInTrashFolder(trashFolderPath)
+// no carbondata file is added to the trash
+assert(list == 0)
+sql("""DROP TABLE IF EXISTS CLEANTEST""")
+  }
+
+
+
+  test("test trash folder with 2 segments with same segment number") {
+createParitionTable()
+sql(s"""INSERT INTO CLEANTEST SELECT 1, 2,"hello","abc)
+
+val path =

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530970382



##
File path: docs/clean-files.md
##
@@ -0,0 +1,56 @@
+
+
+
+## CLEAN FILES
+
+Clean files command is used to remove the Compacted, Marked For Delete ,In 
Progress which are stale and partial(Segments which are missing from the table 
status file but their data is present)
+ segments from the store.
+
+ Clean Files Command
+   ```
+   CLEAN FILES FOR TABLE TABLE_NAME
+   ```
+
+
+### TRASH FOLDER
+
+  Carbondata supports a Trash Folder which is used as a redundant folder where 
all stale(segments whose entry is not in tablestatus file) carbondata segments 
are moved to during clean files operation.
+  This trash folder is mantained inside the table path and is a hidden 
folder(.Trash). The segments that are moved to the trash folder are mantained 
under a timestamp 
+  subfolder(each clean files operation is represented by a timestamp). This 
helps the user to list down segments in the trash folder by timestamp.  By 
default all the timestamp sub-directory have an expiration
+  time of 7 days(since the timestamp it was created) and it can be configured 
by the user using the following carbon property. The supported values are 
between 0 and 365(both included.)
+   ```
+   carbon.trash.retention.days = "Number of days"
+   ``` 
+  Once the timestamp subdirectory is expired as per the configured expiration 
day value, that subdirectory is deleted from the trash folder in the subsequent 
clean files command.
+
+### FORCE DELETE TRASH
+The force option with clean files command deletes all the files and folders 
from the trash folder.
+
+  ```
+  CLEAN FILES FOR TABLE TABLE_NAME options('force'='true')
+  ```
+
+### DATA RECOVERY FROM THE TRASH FOLDER
+
+The segments can be recovered from the trash folder by creating table from the 
desired segment location

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530969904



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for table given table. In 
this method, we first
+   * get the stale segments(segments whose entry is not in the table status, 
but are present in
+   * the metadata folder) or in case when table status is deleted. To identify 
the stale segments
+   * we compare the segment files in the metadata folder with table status 
file, if it exists. The
+   * identified stale segments are then copied to the trash folder and then 
their .segment files
+   * are also deleted from the metadata folder. We only compare with 
tablestatus file here, not
+   * with tablestatus history file.
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+String metaDataLocation = carbonTable.getMetadataPath();
+long timeStampForTrashFolder = System.currentTimeMillis();
+String segmentFilesLocation =
+CarbonTablePath.getSegmentFilesLocation(carbonTable.getTablePath());
+CarbonFile[] segmentFilesList = 
FileFactory.getCarbonFile(segmentFilesLocation).listFiles();
+// there are no segments present in the Metadata folder. Can return here
+if (segmentFilesList.length == 0) {
+  return;
+}
+LoadMetadataDetails[] details = 
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+List staleSegments = getStaleSegments(details, segmentFilesList);

Review comment:
   Changed. separated flow for normal table and partition table. In case of 
normal table, getting the segment path from the .segment file location map and 
moving complete segment. In case of partition table flow, moving it file by 
file.





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530964261



##
File path: 
core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
##
@@ -1414,6 +1414,23 @@ private CarbonCommonConstants() {
 
   public static final String BITSET_PIPE_LINE_DEFAULT = "true";
 
+  /**
+   * this is the user defined time(in days), timestamp subfolders in trash 
directory will take
+   * this value as retention time. They are deleted after this time.
+   */
+  @CarbonProperty
+  public static final String CARBON_TRASH_RETENTION_DAYS = 
"carbon.trash.retention.days";
+
+  /**
+   * Default retention time of a subdirectory in trash folder is 7 days.
+   */
+  public static final String CARBON_TRASH_RETENTION_DAYS_DEFAULT = "7";

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530957279



##
File path: 
integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/events/CleanFilesPostEventListener.scala
##
@@ -54,6 +52,12 @@ class CleanFilesPostEventListener extends 
OperationEventListener with Logging {
 val indexTables = CarbonIndexUtil
   .getIndexCarbonTables(carbonTable, cleanFilesPostEvent.sparkSession)
 indexTables.foreach { indexTable =>
+  if (cleanFilesPostEvent.force) {
+TrashUtil.emptyTrash(indexTable.getTablePath)
+  } else {
+TrashUtil.deleteExpiredDataFromTrash(indexTable.getTablePath)
+  }
+  CleanFilesUtil.cleanStaleSegments(indexTable)

Review comment:
   removed trash code flow from cleanfilespostevenlistener

##
File path: 
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cleanfiles/TestCleanFileCommand.scala
##
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.cleanfiles
+
+import java.io.{File, PrintWriter}
+
+import scala.io.Source
+
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestCleanFileCommand extends QueryTest with BeforeAndAfterAll {
+
+  var count = 0
+
+  test("clean up table and test trash folder with IN PROGRESS segments") {
+// do not send the segment folders to trash
+createTable()
+loadData()
+val path = CarbonEnv.getCarbonTable(Some("default"), 
"cleantest")(sqlContext.sparkSession)
+  .getTablePath
+val trashFolderPath = path + CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.TRASH_DIR

Review comment:
   done





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530956951



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param fromPath the path from which to copy the file
+   * @param toPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String fromPath, String toPath) throws 
IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(toPath);
+  dataInputStream = FileFactory.getDataInputStream(fromPath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + fromPath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {
+  FileFactory.mkdirs(trashFolderWithTimestamp);
+}
+if (!FileFactory.isFileExist(trashFolderWithTimestamp + 
CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName())) {
+  copyToTrashFolder(filePathToCopy, trashFolderWithTimestamp + 
CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName());
+}
+  }
+} catch (IOException e) {
+  LOGGER.error("Error while creating trash folder or copying data to the 
trash folder", e);
+  throw e;
+}
+  }
+
+  /**
+   * The below method copies the complete segment folder to the trash folder. 
Here, the data files
+   * in segment are listed and copied one by one to the trash folder.
+   *
+   * @param segmentPath the folder which are to be moved to the trash folder
+   * @param trashFolderWithTimestamp trashfolderpath with complete timestamp 
and segment number
+   * @return
+   */
+  public static void copySegmentToTrash(CarbonFile segmentPath,
+  String trashFolderWithTimestamp) throws IOException {
+try {
+  List dataFiles = 
FileFactory.getFolderList(segmentPath.getAbsolutePath());
+  for (CarbonFile carbonFile : dataFiles) {
+copyFileToTrashFolder(carbonFile.getAbsolutePath(), 
trashFolderWithTimestamp);
+  }
+  LOGGER.info("Segment: " + segmentPath.getAbsolutePath() + " has been 
copied to" +
+  " the trash folder successfully");
+} catch (IOException e) {
+  LOGGER.error("Error while getting folder list for the segment", e);
+  throw e;
+}
+  }
+

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530955940



##
File path: core/src/main/java/org/apache/carbondata/core/util/TrashUtil.java
##
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the trash folder in carbondata. This class has methods to copy 
data to the trash and
+ * remove data from the trash.
+ */
+public final class TrashUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(TrashUtil.class.getName());
+
+  /**
+   * Base method to copy the data to the trash folder.
+   *
+   * @param fromPath the path from which to copy the file
+   * @param toPath  the path where the file will be copied
+   * @return
+   */
+  private static void copyToTrashFolder(String fromPath, String toPath) throws 
IOException {
+DataOutputStream dataOutputStream = null;
+DataInputStream dataInputStream = null;
+try {
+  dataOutputStream = FileFactory.getDataOutputStream(toPath);
+  dataInputStream = FileFactory.getDataInputStream(fromPath);
+  IOUtils.copyBytes(dataInputStream, dataOutputStream, 
CarbonCommonConstants.BYTEBUFFER_SIZE);
+} catch (IOException exception) {
+  LOGGER.error("Unable to copy " + fromPath + " to the trash folder", 
exception);
+  throw exception;
+} finally {
+  CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+}
+  }
+
+  /**
+   * The below method copies the complete a file to the trash folder.
+   *
+   * @param filePathToCopy the files which are to be moved to the trash folder
+   * @param trashFolderWithTimestamptimestamp, partition folder(if any) 
and segment number
+   * @return
+   */
+  public static void copyFileToTrashFolder(String filePathToCopy,
+  String trashFolderWithTimestamp) throws IOException {
+CarbonFile carbonFileToCopy = FileFactory.getCarbonFile(filePathToCopy);
+try {
+  if (carbonFileToCopy.exists()) {
+if (!FileFactory.isFileExist(trashFolderWithTimestamp)) {
+  FileFactory.mkdirs(trashFolderWithTimestamp);
+}
+if (!FileFactory.isFileExist(trashFolderWithTimestamp + 
CarbonCommonConstants
+.FILE_SEPARATOR + carbonFileToCopy.getName())) {
+  copyToTrashFolder(filePathToCopy, trashFolderWithTimestamp + 
CarbonCommonConstants
+  .FILE_SEPARATOR + carbonFileToCopy.getName());
+}
+  }
+} catch (IOException e) {
+  LOGGER.error("Error while creating trash folder or copying data to the 
trash folder", e);
+  throw e;
+}
+  }
+
+  /**
+   * The below method copies the complete segment folder to the trash folder. 
Here, the data files
+   * in segment are listed and copied one by one to the trash folder.
+   *
+   * @param segmentPath the folder which are to be moved to the trash folder
+   * @param trashFolderWithTimestamp trashfolderpath with complete timestamp 
and segment number
+   * @return
+   */
+  public static void copySegmentToTrash(CarbonFile segmentPath,

Review comment:
   Is now being used for normal table clean stale segments  flow





This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-26 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r530955721



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for 
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+  /**
+   * This method will clean all the stale segments for table given table. In 
this method, we first
+   * get the stale segments(segments whose entry is not in the table status, 
but are present in
+   * the metadata folder) or in case when table status is deleted. To identify 
the stale segments
+   * we compare the segment files in the metadata folder with table status 
file, if it exists. The
+   * identified stale segments are then copied to the trash folder and then 
their .segment files
+   * are also deleted from the metadata folder. We only compare with 
tablestatus file here, not
+   * with tablestatus history file.
+   */
+  public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+String metaDataLocation = carbonTable.getMetadataPath();
+long timeStampForTrashFolder = System.currentTimeMillis();
+String segmentFilesLocation =
+CarbonTablePath.getSegmentFilesLocation(carbonTable.getTablePath());
+CarbonFile[] segmentFilesList = 
FileFactory.getCarbonFile(segmentFilesLocation).listFiles();
+// there are no segments present in the Metadata folder. Can return here
+if (segmentFilesList.length == 0) {
+  return;
+}
+LoadMetadataDetails[] details = 
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+List staleSegments = getStaleSegments(details, segmentFilesList);
+
+if (staleSegments.size() > 0) {
+  for (String staleSegment : staleSegments) {
+String segmentNumber = 
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for each segment we get the indexfile first, then we get the 
carbondata file. Move both
+// of those to trash folder
+List filesToDelete = new ArrayList<>();
+SegmentFileStore fileStore = new 
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+List indexOrMergeFiles = 
fileStore.readIndexFiles(SegmentStatus.SUCCESS, true,
+FileFactory.getConfiguration());
+for (String file : indexOrMergeFiles) {
+  // copy the index or merge file to the trash folder
+  TrashUtil.copyFileToTrashFolder(file, 
CarbonTablePath.getTrashFolderPath(carbonTable
+  .getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + 
timeStampForTrashFolder +
+  CarbonCommonConstants.FILE_SEPARATOR + 
CarbonTablePath.SEGMENT_PREFIX +
+  segmentNumber);
+  filesToDelete.add(FileFactory.getCarbonFile(file));
+}
+// get carbondata files from here
+Map> indexFilesMap = fileStore.getIndexFilesMap();
+for (Map.Entry> entry : indexFilesMap.entrySet()) 
{
+  for (String file : entry.getValue()) {
+// copy the carbondata file to trash
+TrashUtil.copyFileToTrashFolder(file,

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-19 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r527484932



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.log4j.Logger;
+
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CarbonUtil.class.getName());
+  private static List filesInPartitionFolder = new 
ArrayList();
+
+  /**
+   *
+   * This method will clean the stale segments and move them to the trash 
folder.
+   */
+  public static void cleanStaleSegmentsNonPartitionTable(CarbonTable 
carbonTable)
+ throws IOException {
+String metaDataLocation = carbonTable.getMetadataPath();
+String partitionPath = 
CarbonTablePath.getPartitionDir(carbonTable.getTablePath());
+String timeStampForTrashFolder = String.valueOf(new 
Timestamp(System.currentTimeMillis())
+.getTime());
+if (FileFactory.isFileExist(partitionPath)) {
+  // list all segments before reading tablestatus file.
+  CarbonFile[] allSegments = 
FileFactory.getCarbonFile(partitionPath).listFiles();
+  // there is no segment
+  if (allSegments == null || allSegments.length == 0) {
+return;
+  }
+  int retryCount = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT);
+  int maxTimeout = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT);
+  ICarbonLock carbonTableStatusLock = CarbonLockFactory
+  .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier(), 
LockUsage.TABLE_STATUS_LOCK);
+  try {
+if (carbonTableStatusLock.lockWithRetries(retryCount, maxTimeout)) {
+  LoadMetadataDetails[] details = 
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+  // there is no segment or failed to read tablestatus file.
+  // so it should stop immediately.
+  // Do not stop if the tablestatus file does not exist, it can have 
stale segments.
+  if (FileFactory.getCarbonFile(carbonTable.getMetadataPath() + 
CarbonCommonConstants
+  .FILE_SEPARATOR + CarbonTablePath.TABLE_STATUS_FILE).exists() && 
(details == null
+  || details.length == 0)) {
+return;
+  }
+  HashMap staleSegments = 
getStaleSegmentsNormalTable(details,
+  allSegments);
+  // move these segments one by one to the trash folder
+  for (Map.Entry entry : staleSegments.entrySet()) 
{
+TrashUtil.copySegmentToTrash(entry.getKey(), 
carbonTable.getTablePath(),
+timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR 
+
+CarbonCommonConstants.LOAD_FOLDER + entry.getValue());
+LOGGER.info("Deleting Segment: " + 
entry.getKey().getAbsolutePath());
+try {
+  CarbonUtil.deleteFoldersAndFiles(entry.getKey());
+  LOGGER.info("Deleting

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-19 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r527484205



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.log4j.Logger;
+
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CarbonUtil.class.getName());
+  private static List filesInPartitionFolder = new 
ArrayList();
+
+  /**
+   *
+   * This method will clean the stale segments and move them to the trash 
folder.
+   */
+  public static void cleanStaleSegmentsNonPartitionTable(CarbonTable 
carbonTable)
+ throws IOException {
+String metaDataLocation = carbonTable.getMetadataPath();
+String partitionPath = 
CarbonTablePath.getPartitionDir(carbonTable.getTablePath());
+String timeStampForTrashFolder = String.valueOf(new 
Timestamp(System.currentTimeMillis())
+.getTime());
+if (FileFactory.isFileExist(partitionPath)) {
+  // list all segments before reading tablestatus file.
+  CarbonFile[] allSegments = 
FileFactory.getCarbonFile(partitionPath).listFiles();
+  // there is no segment
+  if (allSegments == null || allSegments.length == 0) {
+return;
+  }
+  int retryCount = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT);
+  int maxTimeout = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT);
+  ICarbonLock carbonTableStatusLock = CarbonLockFactory
+  .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier(), 
LockUsage.TABLE_STATUS_LOCK);
+  try {
+if (carbonTableStatusLock.lockWithRetries(retryCount, maxTimeout)) {
+  LoadMetadataDetails[] details = 
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+  // there is no segment or failed to read tablestatus file.
+  // so it should stop immediately.
+  // Do not stop if the tablestatus file does not exist, it can have 
stale segments.
+  if (FileFactory.getCarbonFile(carbonTable.getMetadataPath() + 
CarbonCommonConstants
+  .FILE_SEPARATOR + CarbonTablePath.TABLE_STATUS_FILE).exists() && 
(details == null
+  || details.length == 0)) {
+return;
+  }
+  HashMap staleSegments = 
getStaleSegmentsNormalTable(details,
+  allSegments);
+  // move these segments one by one to the trash folder
+  for (Map.Entry entry : staleSegments.entrySet()) 
{
+TrashUtil.copySegmentToTrash(entry.getKey(), 
carbonTable.getTablePath(),
+timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR 
+
+CarbonCommonConstants.LOAD_FOLDER + entry.getValue());
+LOGGER.info("Deleting Segment: " + 
entry.getKey().getAbsolutePath());
+try {
+  CarbonUtil.deleteFoldersAndFiles(entry.getKey());
+  LOGGER.info("Deleting

[GitHub] [carbondata] vikramahuja1001 commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata

2020-11-19 Thread GitBox



vikramahuja1001 commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r527482978



##
File path: 
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.log4j.Logger;
+
+public class CleanFilesUtil {
+
+  private static final Logger LOGGER =
+  LogServiceFactory.getLogService(CarbonUtil.class.getName());
+  private static List filesInPartitionFolder = new 
ArrayList();
+
+  /**
+   *
+   * This method will clean the stale segments and move them to the trash 
folder.
+   */
+  public static void cleanStaleSegmentsNonPartitionTable(CarbonTable 
carbonTable)
+ throws IOException {
+String metaDataLocation = carbonTable.getMetadataPath();
+String partitionPath = 
CarbonTablePath.getPartitionDir(carbonTable.getTablePath());
+String timeStampForTrashFolder = String.valueOf(new 
Timestamp(System.currentTimeMillis())
+.getTime());
+if (FileFactory.isFileExist(partitionPath)) {
+  // list all segments before reading tablestatus file.
+  CarbonFile[] allSegments = 
FileFactory.getCarbonFile(partitionPath).listFiles();
+  // there is no segment
+  if (allSegments == null || allSegments.length == 0) {
+return;
+  }
+  int retryCount = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT);
+  int maxTimeout = CarbonLockUtil
+  
.getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK,
+  CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT);
+  ICarbonLock carbonTableStatusLock = CarbonLockFactory
+  .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier(), 
LockUsage.TABLE_STATUS_LOCK);
+  try {
+if (carbonTableStatusLock.lockWithRetries(retryCount, maxTimeout)) {
+  LoadMetadataDetails[] details = 
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+  // there is no segment or failed to read tablestatus file.
+  // so it should stop immediately.
+  // Do not stop if the tablestatus file does not exist, it can have 
stale segments.
+  if (FileFactory.getCarbonFile(carbonTable.getMetadataPath() + 
CarbonCommonConstants
+  .FILE_SEPARATOR + CarbonTablePath.TABLE_STATUS_FILE).exists() && 
(details == null
+  || details.length == 0)) {
+return;
+  }
+  HashMap staleSegments = 
getStaleSegmentsNormalTable(details,
+  allSegments);
+  // move these segments one by one to the trash folder
+  for (Map.Entry entry : staleSegments.entrySet()) 
{
+TrashUtil.copySegmentToTrash(entry.getKey(), 
carbonTable.getTablePath(),
+timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR 
+
+CarbonCommonConstants.LOAD_FOLDER + entry.getValue());
+LOGGER.info("Deleting Segment: " + 
entry.getKey().getAbsolutePath());
+try {
+  CarbonUtil.deleteFoldersAndFiles(entry.getKey());
+  LOGGER.info("Deleting

43 matches

Mail list logo