alexeykudinkin commented on code in PR #5428:
URL: https://github.com/apache/hudi/pull/5428#discussion_r924055733
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala:
##########
@@ -514,32 +594,66 @@ abstract class HoodieBaseRelation(val sqlContext:
SQLContext,
// t/h Spark Session configuration (for ex, for Spark SQL)
optParams.getOrElse(DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key,
DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.defaultValue.toString).toBoolean
||
- sparkSession.conf.get(DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key,
-
DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.defaultValue.toString).toBoolean
+ sparkSession.conf.get(DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key,
+
DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.defaultValue.toString).toBoolean
}
}
object HoodieBaseRelation extends SparkAdapterSupport {
private def generateUnsafeProjection(from: StructType, to: StructType) =
-
sparkAdapter.createCatalystExpressionUtils().generateUnsafeProjection(from, to)
+ sparkAdapter.getCatalystExpressionUtils().generateUnsafeProjection(from,
to)
def convertToAvroSchema(structSchema: StructType): Schema =
sparkAdapter.getAvroSchemaConverters.toAvroType(structSchema, nullable =
false, "Record")
def getPartitionPath(fileStatus: FileStatus): Path =
fileStatus.getPath.getParent
+ /**
+ * Projects provided schema by picking only required (projected) top-level
columns from it
+ *
+ * @param tableSchema schema to project (either of [[InternalSchema]] or
Avro's [[Schema]])
+ * @param requiredColumns required top-level columns to be projected
+ */
+ def projectSchema(tableSchema: Either[Schema, InternalSchema],
requiredColumns: Array[String]): (Schema, StructType, InternalSchema) = {
Review Comment:
Correct
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
