[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294353005
##
File path:
integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala
##
@@ -62,216 +67,366 @@ case class CarbonShowCacheCommand(tableIdentifier:
Option[TableIdentifier],
}
}
- override protected def opName: String = "SHOW CACHE"
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+if (tableIdentifier.isEmpty) {
+ /**
+ * Assemble result for database
+ */
+ getAllTablesCache(sparkSession)
+} else {
+ /**
+ * Assemble result for table
+ */
+ val carbonTable =
CarbonEnv.getCarbonTable(tableIdentifier.get)(sparkSession)
+ Checker
+.validateTableExists(tableIdentifier.get.database,
tableIdentifier.get.table, sparkSession)
+ val numberOfIndexFiles = CacheUtil.getAllIndexFiles(carbonTable).size
+ val driverRawResults = getTableCacheFromDriver(sparkSession,
carbonTable, numberOfIndexFiles)
+ val indexRawResults = if
(CarbonProperties.getInstance().isDistributedPruningEnabled
+
(tableIdentifier.get.database.getOrElse(sparkSession.catalog.currentDatabase),
+tableIdentifier.get.table)) {
+getTableCacheFromIndexServer(carbonTable,
numberOfIndexFiles)(sparkSession)
+ } else { Seq() }
+ val result = driverRawResults.slice(0, 2) ++
+ driverRawResults.drop(2).map { row =>
+ Row(row.get(0), row.getLong(1) + row.getLong(2),
row.get(3))
+ }
+ val serverResults = indexRawResults.slice(0, 2) ++
+ indexRawResults.drop(2).map { row =>
+Row(row.get(0), row.getLong(1) + row.getLong(2),
row.get(3))
+ }
+ Seq(Row("DRIVER CACHE", "", "")) ++ result.map {
+row =>
+ Row(row.get(0), bytesToDisplaySize(row.getLong(1)), row.get(2))
+ } ++ (serverResults match {
+case Nil => Seq()
+case list =>
+ Seq(Row("---", "---", "---"), Row("INDEX
CACHE", "", "")) ++
+ list.map {
+ row => Row(row.get(0), bytesToDisplaySize(row.getLong(1)),
row.get(2))
+}
+ })
+}
+ }
def getAllTablesCache(sparkSession: SparkSession): Seq[Row] = {
val currentDatabase = sparkSession.sessionState.catalog.getCurrentDatabase
val cache = CacheProvider.getInstance().getCarbonCache
-if (cache == null) {
- Seq(
-Row("ALL", "ALL", 0L, 0L, 0L),
-Row(currentDatabase, "ALL", 0L, 0L, 0L))
-} else {
- var carbonTables = mutable.ArrayBuffer[CarbonTable]()
- sparkSession.sessionState.catalog.listTables(currentDatabase).foreach {
-tableIdent =>
+val isDistributedPruningEnabled = CarbonProperties.getInstance()
+ .isDistributedPruningEnabled("", "")
+if (cache == null && !isDistributedPruningEnabled) {
+ return makeEmptyCacheRows(currentDatabase)
+}
+var carbonTables = mutable.ArrayBuffer[CarbonTable]()
+sparkSession.sessionState.catalog.listTables(currentDatabase).foreach {
+ tableIdent =>
+try {
+ val carbonTable = CarbonEnv.getCarbonTable(tableIdent)(sparkSession)
+ if (!carbonTable.isChildDataMap && !carbonTable.isChildTable) {
+carbonTables += carbonTable
+ }
+} catch {
+ case _: NoSuchTableException =>
+LOGGER.debug("Ignoring non-carbon table " + tableIdent.table)
+}
+}
+val indexServerRows = if (isDistributedPruningEnabled) {
+ carbonTables.flatMap {
+mainTable =>
try {
-val carbonTable =
CarbonEnv.getCarbonTable(tableIdent)(sparkSession)
-if (!carbonTable.isChildDataMap) {
- carbonTables += carbonTable
-}
+makeRows(getTableCacheFromIndexServer(mainTable)(sparkSession),
mainTable)
} catch {
-case ex: NoSuchTableException =>
- LOGGER.debug("Ignoring non-carbon table " + tableIdent.table)
+case ex: UnsupportedOperationException => Seq()
}
}
+} else { Seq() }
- // All tables of current database
- var (dbDatamapSize, dbDictSize) = (0L, 0L)
- val tableList = carbonTables.flatMap {
+val driverRows = if (cache != null) {
+ carbonTables.flatMap {
carbonTable =>
try {
-val tableResult = getTableCache(sparkSession, carbonTable)
-var (indexSize, datamapSize) = (tableResult(0).getLong(1), 0L)
-tableResult.drop(2).foreach {
- row =>
-indexSize += row.getLong(1)
-datamapSize += row.getLong(2)
-}
-val dictSize = tableResult(1).getLong(1)
-
-
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294353083
##
File path:
integration/spark2/src/main/scala/org/apache/carbondata/indexserver/DistributedShowCacheRDD.scala
##
@@ -43,15 +49,21 @@ class DistributedShowCacheRDD(@transient private val ss:
SparkSession, tableName
override def internalCompute(split: Partition, context: TaskContext):
Iterator[String] = {
val dataMaps = DataMapStoreManager.getInstance().getAllDataMaps.asScala
+val tableList = tableName.split(",").map(_.replace("-", "_"))
Review comment:
Why do you need replace tableName?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294351781
##
File path:
integration/spark2/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
##
@@ -47,7 +47,7 @@ trait ServerInterface {
/**
* Get the cache size for the specified table.
*/
- def showCache(tableName: String) : Array[String]
+ def showCache(tableNames: String) : Array[String]
Review comment:
Please optimize the description. table=> tables
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294348743
##
File path:
hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
##
@@ -565,7 +565,9 @@ public BlockMappingVO getBlockRowCount(Job job,
CarbonTable table,
allSegments.getInvalidSegments(), toBeCleanedSegments));
for (InputSplit extendedBlocklet : extendedBlocklets) {
CarbonInputSplit blocklet = (CarbonInputSplit) extendedBlocklet;
-blockletToRowCountMap.put(blocklet.getSegmentId() + "," +
blocklet.getFilePath(),
+String filePath = blocklet.getFilePath();
+String blockName = filePath.substring(filePath.lastIndexOf("/") +
1);
Review comment:
Whether have "\\" in filePath? like for windows?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294346701
##
File path:
datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
##
@@ -453,4 +453,19 @@ public DataMapMeta getMeta() {
public DataMapLevel getDataMapLevel() {
return DataMapLevel.CG;
}
+
+ @Override public String getCacheSize() {
Review comment:
Please optimize
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294346162
##
File path:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
##
@@ -302,14 +302,18 @@ public synchronized void clear() {
}
}
- @Override public String getCacheSize() throws IOException {
+ @Override public String getCacheSize() {
Review comment:
Please optimize this line to two lines:
```
@Override
public String getCacheSize() {
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294346162
##
File path:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
##
@@ -302,14 +302,18 @@ public synchronized void clear() {
}
}
- @Override public String getCacheSize() throws IOException {
+ @Override public String getCacheSize() {
Review comment:
Please optimize this line to two line:
```
@Override
public String getCacheSize() {
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [carbondata] xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled show cache for index server and MV
xubo245 commented on a change in pull request #3245: [CARBONDATA-3398] Handled
show cache for index server and MV
URL: https://github.com/apache/carbondata/pull/3245#discussion_r294346162
##
File path:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
##
@@ -302,14 +302,18 @@ public synchronized void clear() {
}
}
- @Override public String getCacheSize() throws IOException {
+ @Override public String getCacheSize() {
Review comment:
Please optimize this line to two line:
`
@Override
public String getCacheSize() {
`
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
