Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1782#discussion_r160909350
--- Diff:
integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
---
@@ -893,33 +893,53 @@ object CommonUtil {
/**
* Merge the carbonindex files with in the segment to carbonindexmerge
file inside same segment
+ *
+ * @param sparkContext
+ * @param segmentIds
+ * @param tablePath
+ * @param carbonTable
+ * @param mergeIndexProperty
+ * @param readFileFooterFromCarbonDataFile flag to read file footer
information from carbondata
+ * file. This will used in case
of upgrade from version
+ * which do not store the
blocklet info to current version
*/
def mergeIndexFiles(sparkContext: SparkContext,
segmentIds: Seq[String],
tablePath: String,
carbonTable: CarbonTable,
- mergeIndexProperty: Boolean): Unit = {
+ mergeIndexProperty: Boolean,
+ readFileFooterFromCarbonDataFile: Boolean = false): Unit = {
if (mergeIndexProperty) {
- new CarbonMergeFilesRDD(sparkContext,
AbsoluteTableIdentifier.from(tablePath,
- carbonTable.getDatabaseName,
carbonTable.getTableName).getTablePath,
- segmentIds).collect()
+ new CarbonMergeFilesRDD(
+ sparkContext,
+ AbsoluteTableIdentifier
+ .from(tablePath, carbonTable.getDatabaseName,
carbonTable.getTableName).getTablePath,
+ segmentIds,
+ readFileFooterFromCarbonDataFile).collect()
} else {
try {
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT).toBoolean
if (CarbonProperties.getInstance().getProperty(
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
- new CarbonMergeFilesRDD(sparkContext,
AbsoluteTableIdentifier.from(tablePath,
- carbonTable.getDatabaseName,
carbonTable.getTableName).getTablePath,
- segmentIds).collect()
+ new CarbonMergeFilesRDD(
+ sparkContext,
+ AbsoluteTableIdentifier
+ .from(tablePath, carbonTable.getDatabaseName,
carbonTable.getTableName).getTablePath,
--- End diff --
Please use like this
```
new CarbonMergeFilesRDD(
sparkContext,
AbsoluteTableIdentifier.from(
tablePath,
carbonTable.getDatabaseName,
carbonTable.getTableName).getTablePath,
segmentIds,
readFileFooterFromCarbonDataFile).collect()
```
---
