allisonwang-db commented on code in PR #43784:
URL: https://github.com/apache/spark/pull/43784#discussion_r1391823589
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala:
##########
@@ -75,3 +86,136 @@ class DataSourceManager {
dataSourceBuilders.containsKey(normalize(name))
}
}
+
+/**
+ * Data Source V1 wrapper for Python Data Source.
+ */
+object PythonDataSourceCodeGenerator extends Logging {
+ def generateClass(shortName: String): Class[_] = {
+ val ctx = newCodeGenContext()
+
+ val codeBody = s"""
+ private ${classOf[String].getName} source = "$shortName";
Review Comment:
This is very interesting. We basically dynamically generate a DSv1 wrapper
class when registering a Python data source. I find it a bit difficult to read
the code body. Just curious, can we replace `classOf[String].getName` to
`String` in this case?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala:
##########
@@ -75,3 +86,136 @@ class DataSourceManager {
dataSourceBuilders.containsKey(normalize(name))
}
}
+
+/**
+ * Data Source V1 wrapper for Python Data Source.
+ */
+object PythonDataSourceCodeGenerator extends Logging {
+ def generateClass(shortName: String): Class[_] = {
+ val ctx = newCodeGenContext()
+
+ val codeBody = s"""
+ private ${classOf[String].getName} source = "$shortName";
+
+ @Override
+ public String shortName() {
+ return source;
+ }
+
+ @Override
+ public ${classOf[BaseRelation].getName} createRelation(
+ ${classOf[SQLContext].getName} sqlContext,
+ ${classOf[Map[_, _]].getName}<
+ ${classOf[String].getName},
+ ${classOf[String].getName}
+ > parameters) {
+ return new PythonRelation(sqlContext, parameters, null, this.source);
+ }
+
+ @Override
+ public ${classOf[BaseRelation].getName} createRelation(
+ ${classOf[SQLContext].getName} sqlContext,
+ ${classOf[Map[_, _]].getName}<
+ ${classOf[String].getName},
+ ${classOf[String].getName}
+ > parameters,
+ ${classOf[StructType].getName} schema) {
+ return new PythonRelation(sqlContext, parameters, schema, this.source);
+ }
+
+ public class PythonRelation
+ extends ${classOf[BaseRelation].getName}
+ implements ${classOf[TableScan].getName} {
+ private ${classOf[SQLContext].getName} sqlContext;
+ ${classOf[Map[_, _]].getName}<
+ ${classOf[String].getName},
+ ${classOf[String].getName}
+ > parameters;
+ private ${classOf[StructType].getName} schema;
+ private ${classOf[String].getName} source;
+ private ${classOf[Dataset[_]].getName}<${classOf[Row].getName}> df;
+
+ public PythonRelation(
+ ${classOf[SQLContext].getName} sqlContext,
+ ${classOf[Map[_, _]].getName}<
+ ${classOf[String].getName},
+ ${classOf[String].getName}
+ > parameters,
+ ${classOf[StructType].getName} schema,
+ ${classOf[String].getName} source) {
+ this.sqlContext = sqlContext;
+ this.parameters = parameters;
+ this.source = source;
+ if (schema != null) {
+ this.schema = schema;
+ }
+ }
+
+ private ${classOf[Dataset[_]].getName}<${classOf[Row].getName}>
+ getOrCreateDataFrame() {
+ if (df == null) {
+ ${classOf[LogicalPlan].getName} plan =
(${classOf[LogicalPlan].getName}) this.sqlContext
+
.sparkSession().sharedState().dataSourceManager().lookupDataSource(source)
Review Comment:
Note: this will be changed to session-level.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala:
##########
@@ -185,9 +185,18 @@ class PythonDataSourceSuite extends QueryTest with
SharedSparkSession {
val dataSource = createUserDefinedPythonDataSource(dataSourceName,
dataSourceScript)
spark.dataSource.registerPython("test", dataSource)
+ // Test DataFrame
checkAnswer(spark.read.format("test").load(), Seq(Row(null, 1)))
checkAnswer(spark.read.format("test").load("1"), Seq(Row("1", 1)))
- checkAnswer(spark.read.format("test").load("1", "2"), Seq(Row("1", 1),
Row("2", 1)))
+ // DataSource V1 disallows multiple path reading.
Review Comment:
Ah I didn't know DSv1 does not support multiple path strings. I can update
this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
