Github user xuchuanyin commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2299#discussion_r187768312
--- Diff:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
---
@@ -115,13 +123,55 @@ public DataMapBuilder createBuilder(Segment segment,
String shardName) {
Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
segmentMap.get(segment.getSegmentNo());
if (tableBlockIndexUniqueIdentifiers == null) {
+ CarbonTable carbonTable = this.getCarbonTable();
+ if (!carbonTable.getTableInfo().isTransactionalTable()) {
+ // For NonTransactional table, compare the schema of all index
files with inferred schema.
+ // If there is a mismatch throw exception. As all files must be of
same schema.
+ validateSchemaForNewTranscationalTableFiles(segment, carbonTable);
+ }
tableBlockIndexUniqueIdentifiers =
BlockletDataMapUtil.getTableBlockUniqueIdentifiers(segment);
segmentMap.put(segment.getSegmentNo(),
tableBlockIndexUniqueIdentifiers);
}
return tableBlockIndexUniqueIdentifiers;
}
+ private void validateSchemaForNewTranscationalTableFiles(Segment
segment, CarbonTable carbonTable)
+ throws IOException {
+ SchemaConverter schemaConverter = new
ThriftWrapperSchemaConverterImpl();
+ Map<String, String> indexFiles = segment.getCommittedIndexFile();
+ for (Map.Entry<String, String> indexFileEntry : indexFiles.entrySet())
{
+ Path indexFile = new Path(indexFileEntry.getKey());
+ org.apache.carbondata.format.TableInfo tableInfo =
CarbonUtil.inferSchemaFromIndexFile(
+ indexFile.toString(), carbonTable.getTableName());
+ TableInfo wrapperTableInfo =
schemaConverter.fromExternalToWrapperTableInfo(
+ tableInfo, identifier.getDatabaseName(),
+ identifier.getTableName(),
+ identifier.getTablePath());
+ List<ColumnSchema> indexFileColumnList =
+ wrapperTableInfo.getFactTable().getListOfColumns();
+ List<ColumnSchema> tableColumnList =
+ carbonTable.getTableInfo().getFactTable().getListOfColumns();
+ if (!compareColumnSchemaList(indexFileColumnList, tableColumnList)) {
+ LOG.error("Schema of " + indexFile.getName()
+ + " doesn't match with the table's schema");
+ throw new IOException("All the files doesn't have same schema. "
+ + "Unsupported operation on nonTransactional table. Check
logs.");
+ }
+ }
+ }
+
+ private boolean compareColumnSchemaList(List<ColumnSchema>
indexFileColumnList,
+ List<ColumnSchema> tableColumnList) {
+ if (indexFileColumnList.size() != tableColumnList.size()) {
+ return false;
--- End diff --
Can you add a log here and line170 to tell the reason.
---
