milastdbx commented on code in PR #48625:
URL: https://github.com/apache/spark/pull/48625#discussion_r1818954177
##########
sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala:
##########
@@ -382,18 +382,11 @@ private case class PostgresDialect()
case Types.ARRAY =>
val tableName = rsmd.getTableName(columnIdx)
val columnName = rsmd.getColumnName(columnIdx)
- val query =
- s"""
- |SELECT pg_attribute.attndims
- |FROM pg_attribute
- | JOIN pg_class ON pg_attribute.attrelid = pg_class.oid
- | JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid
- |WHERE pg_class.relname = '$tableName' and pg_attribute.attname =
'$columnName'
- |""".stripMargin
+ val query = s"SELECT array_ndims($columnName) FROM $tableName LIMIT 1"
Review Comment:
Lets add comment
```
// Spark does not support different array dimensionality per row. With this
in mind, get dimensionality of any row.
// This might fail down the road, as Postgres array column might contain
different dimensionality.
```
##########
sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala:
##########
@@ -382,18 +382,11 @@ private case class PostgresDialect()
case Types.ARRAY =>
val tableName = rsmd.getTableName(columnIdx)
val columnName = rsmd.getColumnName(columnIdx)
- val query =
- s"""
- |SELECT pg_attribute.attndims
- |FROM pg_attribute
- | JOIN pg_class ON pg_attribute.attrelid = pg_class.oid
- | JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid
- |WHERE pg_class.relname = '$tableName' and pg_attribute.attname =
'$columnName'
- |""".stripMargin
+ val query = s"SELECT array_ndims($columnName) FROM $tableName LIMIT 1"
try {
Using.resource(conn.createStatement()) { stmt =>
Using.resource(stmt.executeQuery(query)) { rs =>
- if (rs.next()) metadata.putLong("arrayDimension", rs.getLong(1))
+ if (rs.next()) metadata.putLong("arrayDimension",
Math.max(rs.getLong(1), 1))
Review Comment:
Add comments
```
// Depending on how table is created dimensionality of array might not be
present in metadata.
// In these cases above pg metadata query returns zero for arrayDimension.
We default to 1 in this case.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
