This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2c5bcfad46e8 [SPARK-55580][SQL][TESTS] Use `spark` instead of
`sqlContext` in `docker-integration-tests`
2c5bcfad46e8 is described below
commit 2c5bcfad46e8fd7fea6728b47aa92aac322ee54e
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Feb 18 05:52:11 2026 -0800
[SPARK-55580][SQL][TESTS] Use `spark` instead of `sqlContext` in
`docker-integration-tests`
### What changes were proposed in this pull request?
This PR aims to use `spark` instead of `sqlContext` in
`docker-integration-tests`.
### Why are the changes needed?
To simplify the test code according to the current Apache Spark convention.
**BEFORE**
```bash
$ git grep sqlContext connector/docker-integration-tests/ | wc -l
56
```
**AFTER**
```bash
$ git grep sqlContext connector/docker-integration-tests/ | wc -l
0
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: `Gemini 3 Pro (High)` on `Antigravity`
Closes #54360 from dongjoon-hyun/SPARK-55580.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/jdbc/DB2IntegrationSuite.scala | 38 +++++++++++-----------
.../sql/jdbc/DockerKrbJDBCIntegrationSuite.scala | 4 +--
.../spark/sql/jdbc/MySQLIntegrationSuite.scala | 24 +++++++-------
.../spark/sql/jdbc/OracleIntegrationSuite.scala | 12 +++----
.../spark/sql/jdbc/PostgresIntegrationSuite.scala | 34 +++++++++----------
5 files changed, 56 insertions(+), 56 deletions(-)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index c2d83943e322..6a489ffb2d42 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -79,7 +79,7 @@ class DB2IntegrationSuite extends SharedJDBCIntegrationSuite {
}
test("Basic test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "tbl", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "tbl", new Properties)
val rows = df.collect()
assert(rows.length == 2)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -91,7 +91,7 @@ class DB2IntegrationSuite extends SharedJDBCIntegrationSuite {
test("Numeric types") {
Seq(true, false).foreach { legacy =>
withSQLConf(SQLConf.LEGACY_DB2_TIMESTAMP_MAPPING_ENABLED.key ->
legacy.toString) {
- val df = sqlContext.read.jdbc(jdbcUrl, "numbers", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
val rows = df.collect()
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -131,7 +131,7 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
test("Date types") {
withDefaultTimeZone(UTC) {
- val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
val rows = df.collect()
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -146,7 +146,7 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("String types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "strings", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "strings", new Properties)
val rows = df.collect()
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -164,20 +164,20 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
test("Basic write test") {
// cast decflt column with precision value of 38 to DB2 max decimal
precision value of 31.
- val df1 = sqlContext.read.jdbc(jdbcUrl, "numbers", new Properties)
+ val df1 = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
.selectExpr("small", "med", "big", "deci", "flt", "dbl", "real",
"cast(decflt as decimal(31, 5)) as decflt")
- val df2 = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
- val df3 = sqlContext.read.jdbc(jdbcUrl, "strings", new Properties)
+ val df2 = spark.read.jdbc(jdbcUrl, "dates", new Properties)
+ val df3 = spark.read.jdbc(jdbcUrl, "strings", new Properties)
df1.write.jdbc(jdbcUrl, "numberscopy", new Properties)
df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
// spark types that does not have exact matching db2 table types.
- val df4 = sqlContext.createDataFrame(
+ val df4 = spark.createDataFrame(
sparkContext.parallelize(Seq(Row("1".toShort, "20".toByte))),
new StructType().add("c1", ShortType).add("b", ByteType))
df4.write.jdbc(jdbcUrl, "otherscopy", new Properties)
- val rows = sqlContext.read.jdbc(jdbcUrl, "otherscopy", new
Properties).collect()
+ val rows = spark.read.jdbc(jdbcUrl, "otherscopy", new Properties).collect()
assert(rows(0).getShort(0) == 1)
assert(rows(0).getShort(1) == 20)
}
@@ -215,20 +215,20 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
).map { case (x, y) =>
Row(Integer.valueOf(x), String.valueOf(y))
}
- val df = sqlContext.read.jdbc(jdbcUrl, "tbl", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "tbl", new Properties)
for (_ <- 0 to 2) {
df.write.mode(SaveMode.Append).jdbc(jdbcUrl, "tblcopy", new Properties)
}
- assert(sqlContext.read.jdbc(jdbcUrl, "tblcopy", new Properties).count()
=== 6)
+ assert(spark.read.jdbc(jdbcUrl, "tblcopy", new Properties).count() === 6)
df.write.mode(SaveMode.Overwrite).option("truncate", true)
.jdbc(jdbcUrl, "tblcopy", new Properties)
- val actual = sqlContext.read.jdbc(jdbcUrl, "tblcopy", new
Properties).collect()
+ val actual = spark.read.jdbc(jdbcUrl, "tblcopy", new Properties).collect()
assert(actual.length === 2)
assert(actual.toSet === expectedResult)
}
test("SPARK-42534: DB2 Limit pushdown test") {
- val actual = sqlContext.read
+ val actual = spark.read
.format("jdbc")
.option("url", jdbcUrl)
.option("dbtable", "tbl")
@@ -238,7 +238,7 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
.orderBy("x")
.collect()
- val expected = sqlContext.read
+ val expected = spark.read
.format("jdbc")
.option("url", jdbcUrl)
.option("query", "SELECT x, y FROM tbl ORDER BY x FETCH FIRST 2 ROWS
ONLY")
@@ -249,23 +249,23 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-48269: boolean type") {
- val df = sqlContext.read.jdbc(jdbcUrl, "booleans", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "booleans", new Properties)
checkAnswer(df, Row(true))
Seq(true, false).foreach { legacy =>
withSQLConf(SQLConf.LEGACY_DB2_BOOLEAN_MAPPING_ENABLED.key ->
legacy.toString) {
val tbl = "booleanscopy" + legacy
df.write.jdbc(jdbcUrl, tbl, new Properties)
if (legacy) {
- checkAnswer(sqlContext.read.jdbc(jdbcUrl, tbl, new Properties),
Row("1"))
+ checkAnswer(spark.read.jdbc(jdbcUrl, tbl, new Properties), Row("1"))
} else {
- checkAnswer(sqlContext.read.jdbc(jdbcUrl, tbl, new Properties),
Row(true))
+ checkAnswer(spark.read.jdbc(jdbcUrl, tbl, new Properties), Row(true))
}
}
}
}
test("SPARK-48269: GRAPHIC types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "graphics", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "graphics", new Properties)
checkAnswer(df, Row("a".padTo(16, ' '), "b"))
// the padding happens in the source not because of reading as char type
assert(!df.schema.exists {
@@ -273,7 +273,7 @@ class DB2IntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-48269: binary types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "binarys", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "binarys", new Properties)
checkAnswer(df, Row(
"ABC".padTo(10, ' ').getBytes,
"ABC".getBytes,
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
index 30cca3bc1f25..850bc97d9395 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerKrbJDBCIntegrationSuite.scala
@@ -150,9 +150,9 @@ abstract class DockerKrbJDBCIntegrationSuite extends
DockerJDBCIntegrationSuite
props.setProperty("principal", principal)
val tableName = "write_test"
- sqlContext.createDataFrame(Seq(("foo", "bar")))
+ spark.createDataFrame(Seq(("foo", "bar")))
.write.jdbc(jdbcUrl, tableName, props)
- val df = sqlContext.read.jdbc(jdbcUrl, tableName, props)
+ val df = spark.read.jdbc(jdbcUrl, tableName, props)
val schema = df.schema
assert(schema.map(_.dataType).toSeq === Seq(StringType, StringType))
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index be17abbb9bff..345b66e07ae0 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -110,7 +110,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Basic test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "tbl", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "tbl", new Properties)
val rows = df.collect()
assert(rows.length == 2)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -120,7 +120,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Numeric types") {
- val row = sqlContext.read.jdbc(jdbcUrl, "numbers", new Properties).head()
+ val row = spark.read.jdbc(jdbcUrl, "numbers", new Properties).head()
assert(row.length === 10)
assert(row(0).isInstanceOf[Boolean])
assert(row(1).isInstanceOf[Array[Byte]])
@@ -145,14 +145,14 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
assert(row.getDouble(8) == 1.0000000000000002)
assert(row.getByte(9) == 0x80.toByte)
withSQLConf(SQLConf.LEGACY_MYSQL_BIT_ARRAY_MAPPING_ENABLED.key -> "true") {
- val row = sqlContext.read.jdbc(jdbcUrl, "numbers", new Properties).head()
+ val row = spark.read.jdbc(jdbcUrl, "numbers", new Properties).head()
assert(row(1).isInstanceOf[Long])
assert(row.getLong(1) == 0x225)
}
}
test("SPARK-47462: Unsigned numeric types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "unsigned_numbers", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "unsigned_numbers", new Properties)
val rows = df.head()
assert(rows.get(0).isInstanceOf[Short])
assert(rows.get(1).isInstanceOf[Integer])
@@ -184,7 +184,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
test("Date types") {
withDefaultTimeZone(UTC) {
- val df = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
checkAnswer(df, Row(
Date.valueOf("1991-11-09"),
Timestamp.valueOf("1970-01-01 13:31:24"),
@@ -203,7 +203,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
test("SPARK-47406: MySQL datetime types with preferTimestampNTZ") {
withDefaultTimeZone(UTC) {
- val df = sqlContext.read.option("preferTimestampNTZ", true)
+ val df = spark.read.option("preferTimestampNTZ", true)
.jdbc(jdbcUrl, "dates", new Properties)
checkAnswer(df, Row(
Date.valueOf("1991-11-09"),
@@ -216,7 +216,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("String types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "strings", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "strings", new Properties)
val rows = df.collect()
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -244,9 +244,9 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Basic write test") {
- val df1 = sqlContext.read.jdbc(jdbcUrl, "numbers", new Properties)
- val df2 = sqlContext.read.jdbc(jdbcUrl, "dates", new Properties)
- val df3 = sqlContext.read.jdbc(jdbcUrl, "strings", new Properties)
+ val df1 = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
+ val df2 = spark.read.jdbc(jdbcUrl, "dates", new Properties)
+ val df3 = spark.read.jdbc(jdbcUrl, "strings", new Properties)
df1.write.jdbc(jdbcUrl, "numberscopy", new Properties)
df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
@@ -280,7 +280,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
test("SPARK-47478: all boolean synonyms read-write roundtrip") {
- val df = sqlContext.read.jdbc(jdbcUrl, "bools", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "bools", new Properties)
checkAnswer(df, Row(true, true, true))
val properties0 = new Properties()
@@ -295,7 +295,7 @@ class MySQLIntegrationSuite extends
SharedJDBCIntegrationSuite {
val properties2 = new Properties()
properties2.setProperty("tinyInt1isBit", "false")
- checkAnswer(sqlContext.read.jdbc(jdbcUrl, "bools", properties2), Row(1,
true, 1))
+ checkAnswer(spark.read.jdbc(jdbcUrl, "bools", properties2), Row(1, true,
1))
df.write.mode("append").jdbc(jdbcUrl, "bools", new Properties)
checkAnswer(df, Seq(Row(true, true, true), Row(true, true, true)))
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index d44e2d1e748f..b76307a59d9c 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -186,7 +186,7 @@ class OracleIntegrationSuite extends
SharedJDBCIntegrationSuite
test("SPARK-16625: Importing Oracle numeric types") {
Seq("true", "false").foreach { flag =>
withSQLConf((SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED.key,
flag)) {
- val df = sqlContext.read.jdbc(jdbcUrl, "numerics", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "numerics", new Properties)
checkAnswer(df, Seq(Row(BigDecimal.valueOf(4),
BigDecimal.valueOf(1.23),
BigDecimal.valueOf(9999999999L), BigDecimal.valueOf(7456100))))
}
@@ -200,7 +200,7 @@ class OracleIntegrationSuite extends
SharedJDBCIntegrationSuite
// write the dataframe to the oracle table tbl
df1.write.jdbc(jdbcUrl, "tbl2", new Properties)
// read the table from the oracle
- val dfRead = sqlContext.read.jdbc(jdbcUrl, "tbl2", new Properties)
+ val dfRead = spark.read.jdbc(jdbcUrl, "tbl2", new Properties)
// get the rows
val rows = dfRead.collect()
// verify the data type is inserted
@@ -293,7 +293,7 @@ class OracleIntegrationSuite extends
SharedJDBCIntegrationSuite
}
test("SPARK-20557: column type TIMESTAMP with TIME ZONE should be
recognized") {
- val dfRead = sqlContext.read.jdbc(jdbcUrl, "ts_with_timezone", new
Properties)
+ val dfRead = spark.read.jdbc(jdbcUrl, "ts_with_timezone", new Properties)
val rows = dfRead.collect()
val types = rows(0).toSeq.map(x => x.getClass.toString)
assert(types(1).equals("class java.sql.Timestamp"))
@@ -309,14 +309,14 @@ class OracleIntegrationSuite extends
SharedJDBCIntegrationSuite
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key ->
localSessionTimeZone.getID) {
checkAnswer(
- sqlContext.read.jdbc(jdbcUrl, "ts_with_timezone", new Properties),
+ spark.read.jdbc(jdbcUrl, "ts_with_timezone", new Properties),
rsOfTsWithTimezone)
}
}
test("Column TIMESTAMP with TIME ZONE(JVM timezone)") {
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key ->
TimeZone.getDefault.getID) {
- val dfRead = sqlContext.read.jdbc(jdbcUrl, "ts_with_timezone", new
Properties)
+ val dfRead = spark.read.jdbc(jdbcUrl, "ts_with_timezone", new Properties)
Seq(PST, UTC).foreach(timeZone => {
withDefaultTimeZone(timeZone) {
checkAnswer(dfRead, rsOfTsWithTimezone)
@@ -415,7 +415,7 @@ class OracleIntegrationSuite extends
SharedJDBCIntegrationSuite
dfWrite.write.mode(SaveMode.Append).jdbc(jdbcUrl, tableName, props)
// read records from oracle_types
- val dfRead = sqlContext.read.jdbc(jdbcUrl, tableName, new Properties)
+ val dfRead = spark.read.jdbc(jdbcUrl, tableName, new Properties)
val rows = dfRead.collect()
assert(rows.length == 1)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 050f74f36903..abbb60bae6f2 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -186,7 +186,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Type mapping for various types") {
- val df = sqlContext.read.jdbc(jdbcUrl, "bar", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "bar", new Properties)
val rows = df.collect().sortBy(_.toString())
assert(rows.length == 2)
// Test the types, and values using the first row.
@@ -286,11 +286,11 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Basic write test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "bar", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "bar", new Properties)
// Test only that it doesn't crash.
df.write.jdbc(jdbcUrl, "public.barcopy", new Properties)
// Test that written numeric type has same DataType as input
- assert(sqlContext.read.jdbc(jdbcUrl, "public.barcopy", new
Properties).schema(13).dataType ==
+ assert(spark.read.jdbc(jdbcUrl, "public.barcopy", new
Properties).schema(13).dataType ==
ArrayType(DecimalType(2, 2), true))
// Test write null values.
df.select(df.queryExecution.analyzed.output.map { a =>
@@ -299,16 +299,16 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("Creating a table with shorts and floats") {
- sqlContext.createDataFrame(Seq((1.0f, 1.toShort)))
+ spark.createDataFrame(Seq((1.0f, 1.toShort)))
.write.jdbc(jdbcUrl, "shortfloat", new Properties)
- val schema = sqlContext.read.jdbc(jdbcUrl, "shortfloat", new
Properties).schema
+ val schema = spark.read.jdbc(jdbcUrl, "shortfloat", new Properties).schema
assert(schema(0).dataType == FloatType)
assert(schema(1).dataType == ShortType)
}
test("SPARK-47390: Convert TIMESTAMP/TIME WITH TIME ZONE regardless of
preferTimestampNTZ") {
Seq(true, false).foreach { prefer =>
- val df = sqlContext.read
+ val df = spark.read
.option("preferTimestampNTZ", prefer)
.jdbc(jdbcUrl, "ts_with_timezone", new Properties)
checkAnswer(df, Row(
@@ -320,7 +320,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
test("SPARK-22291: Conversion error when transforming array types of " +
"uuid, inet and cidr to StingType in PostgreSQL") {
- val df = sqlContext.read.jdbc(jdbcUrl, "st_with_array", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "st_with_array", new Properties)
val rows = df.collect()
assert(rows(0).getString(0) == "0a532531-cdf1-45e3-963d-5de90b6a30f1")
assert(rows(0).getString(1) == "172.168.22.1")
@@ -382,9 +382,9 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("write byte as smallint") {
- sqlContext.createDataFrame(Seq((1.toByte, 2.toShort)))
+ spark.createDataFrame(Seq((1.toByte, 2.toShort)))
.write.jdbc(jdbcUrl, "byte_to_smallint_test", new Properties)
- val df = sqlContext.read.jdbc(jdbcUrl, "byte_to_smallint_test", new
Properties)
+ val df = spark.read.jdbc(jdbcUrl, "byte_to_smallint_test", new Properties)
val schema = df.schema
assert(schema.head.dataType == ShortType)
assert(schema(1).dataType == ShortType)
@@ -395,18 +395,18 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("character type tests") {
- val df = sqlContext.read.jdbc(jdbcUrl, "char_types", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "char_types", new Properties)
checkAnswer(df, Row("abcd", "efgh", "ijkl", "mnop", "q", "eason", "c"))
}
test("SPARK-32576: character array type tests") {
- val df = sqlContext.read.jdbc(jdbcUrl, "char_array_types", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "char_array_types", new Properties)
checkAnswer(df, Row(Seq("a ", "bcd "), Seq("ef ", "gh "), Seq("i",
"j", "kl"),
Seq("mnop"), Seq("q", "r"), Seq("Eason", "Ethan")))
}
test("SPARK-34333: money type tests") {
- val df = sqlContext.read.jdbc(jdbcUrl, "money_types", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "money_types", new Properties)
val row = df.collect()
assert(row.length === 1)
assert(row(0).length === 1)
@@ -416,7 +416,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
test("SPARK-43040: timestamp_ntz read test") {
val prop = new Properties
prop.setProperty("preferTimestampNTZ", "true")
- val df = sqlContext.read.jdbc(jdbcUrl, "timestamp_ntz", prop)
+ val df = spark.read.jdbc(jdbcUrl, "timestamp_ntz", prop)
val row = df.collect()
assert(row.length === 3)
assert(row(0).length === 1)
@@ -444,7 +444,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-43267: user-defined column in array test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "custom_type", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "custom_type", new Properties)
val row = df.collect()
assert(row.length === 1)
assert(row(0).length === 2)
@@ -453,7 +453,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-44280: infinity timestamp test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "infinity_timestamp", new
Properties)
+ val df = spark.read.jdbc(jdbcUrl, "infinity_timestamp", new Properties)
val row = df.collect()
assert(row.length == 2)
@@ -470,7 +470,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-47501: infinity date test") {
- val df = sqlContext.read.jdbc(jdbcUrl, "infinity_dates", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "infinity_dates", new Properties)
val row = df.collect()
assert(row.length == 2)
@@ -496,7 +496,7 @@ class PostgresIntegrationSuite extends
SharedJDBCIntegrationSuite {
}
test("SPARK-47628: Fix reading bit array type") {
- val df = sqlContext.read.jdbc(jdbcUrl, "test_bit_array", new Properties)
+ val df = spark.read.jdbc(jdbcUrl, "test_bit_array", new Properties)
val expected = Row(Array(true, false), Array(
Array[Byte](48, 48, 48, 48, 49), Array[Byte](48, 48, 48, 49, 48)))
checkAnswer(df, expected)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
