[
https://issues.apache.org/jira/browse/SPARK-28256?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Genmao Yu updated SPARK-28256:
------------------------------
Description:
reproduce code
{code:sql}
CREATE TABLE `user_click_count` (`userId` STRING, `click` BIGINT)
USING org.apache.spark.sql.json
OPTIONS (path 'hdfs:///tmp/test');
{code}
error:
{code:java}
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at
org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:136)
at
org.apache.hadoop.fs.AbstractFileSystem.createFileSystem(AbstractFileSystem.java:165)
at
org.apache.hadoop.fs.AbstractFileSystem.get(AbstractFileSystem.java:250)
at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:342)
at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:339)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1911)
at
org.apache.hadoop.fs.FileContext.getAbstractFileSystem(FileContext.java:339)
at org.apache.hadoop.fs.FileContext.getFileContext(FileContext.java:456)
at
org.apache.spark.sql.execution.streaming.FileContextBasedCheckpointFileManager.<init>(CheckpointFileManager.scala:297)
at
org.apache.spark.sql.execution.streaming.CheckpointFileManager$.create(CheckpointFileManager.scala:189)
at
org.apache.spark.sql.execution.streaming.HDFSMetadataLog.<init>(HDFSMetadataLog.scala:63)
at
org.apache.spark.sql.execution.streaming.CompactibleFileStreamLog.<init>(CompactibleFileStreamLog.scala:46)
at
org.apache.spark.sql.execution.streaming.FileStreamSinkLog.<init>(FileStreamSinkLog.scala:85)
at
org.apache.spark.sql.execution.streaming.FileStreamSink.<init>(FileStreamSink.scala:98)
at
org.apache.spark.sql.execution.datasources.DataSource.createSink(DataSource.scala:297)
at
org.apache.spark.sql.execution.datasources.FindDataSourceTable$$anonfun$apply$2.applyOrElse(DataSourceStrategy.scala:379)
...
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:134)
... 67 more
Caused by: org.apache.hadoop.HadoopIllegalArgumentException: Uri without
authority: hdfs:/tmp/test/_spark_metadata
at
org.apache.hadoop.fs.AbstractFileSystem.getUri(AbstractFileSystem.java:313)
at
org.apache.hadoop.fs.AbstractFileSystem.<init>(AbstractFileSystem.java:266)
at org.apache.hadoop.fs.Hdfs.<init>(Hdfs.java:80)
... 72 more
{code}
was:
{code:java}
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at
org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:136)
at
org.apache.hadoop.fs.AbstractFileSystem.createFileSystem(AbstractFileSystem.java:165)
at
org.apache.hadoop.fs.AbstractFileSystem.get(AbstractFileSystem.java:250)
at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:342)
at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:339)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1911)
at
org.apache.hadoop.fs.FileContext.getAbstractFileSystem(FileContext.java:339)
at org.apache.hadoop.fs.FileContext.getFileContext(FileContext.java:456)
at
org.apache.spark.sql.execution.streaming.FileContextBasedCheckpointFileManager.<init>(CheckpointFileManager.scala:297)
at
org.apache.spark.sql.execution.streaming.CheckpointFileManager$.create(CheckpointFileManager.scala:189)
at
org.apache.spark.sql.execution.streaming.HDFSMetadataLog.<init>(HDFSMetadataLog.scala:63)
at
org.apache.spark.sql.execution.streaming.CompactibleFileStreamLog.<init>(CompactibleFileStreamLog.scala:46)
at
org.apache.spark.sql.execution.streaming.FileStreamSinkLog.<init>(FileStreamSinkLog.scala:85)
at
org.apache.spark.sql.execution.streaming.FileStreamSink.<init>(FileStreamSink.scala:98)
at
org.apache.spark.sql.execution.datasources.DataSource.createSink(DataSource.scala:297)
at
org.apache.spark.sql.execution.datasources.FindDataSourceTable$$anonfun$apply$2.applyOrElse(DataSourceStrategy.scala:379)
...
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:134)
... 67 more
Caused by: org.apache.hadoop.HadoopIllegalArgumentException: Uri without
authority: hdfs:/tmp/test/_spark_metadata
at
org.apache.hadoop.fs.AbstractFileSystem.getUri(AbstractFileSystem.java:313)
at
org.apache.hadoop.fs.AbstractFileSystem.<init>(AbstractFileSystem.java:266)
at org.apache.hadoop.fs.Hdfs.<init>(Hdfs.java:80)
... 72 more
{code}
> Failed to initialize FileContextBasedCheckpointFileManager with uri without
> authority
> -------------------------------------------------------------------------------------
>
> Key: SPARK-28256
> URL: https://issues.apache.org/jira/browse/SPARK-28256
> Project: Spark
> Issue Type: Improvement
> Components: Structured Streaming
> Affects Versions: 3.0.0
> Reporter: Genmao Yu
> Priority: Minor
>
> reproduce code
> {code:sql}
> CREATE TABLE `user_click_count` (`userId` STRING, `click` BIGINT)
> USING org.apache.spark.sql.json
> OPTIONS (path 'hdfs:///tmp/test');
> {code}
> error:
> {code:java}
> java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
> at
> org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:136)
> at
> org.apache.hadoop.fs.AbstractFileSystem.createFileSystem(AbstractFileSystem.java:165)
> at
> org.apache.hadoop.fs.AbstractFileSystem.get(AbstractFileSystem.java:250)
> at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:342)
> at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:339)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1911)
> at
> org.apache.hadoop.fs.FileContext.getAbstractFileSystem(FileContext.java:339)
> at org.apache.hadoop.fs.FileContext.getFileContext(FileContext.java:456)
> at
> org.apache.spark.sql.execution.streaming.FileContextBasedCheckpointFileManager.<init>(CheckpointFileManager.scala:297)
> at
> org.apache.spark.sql.execution.streaming.CheckpointFileManager$.create(CheckpointFileManager.scala:189)
> at
> org.apache.spark.sql.execution.streaming.HDFSMetadataLog.<init>(HDFSMetadataLog.scala:63)
> at
> org.apache.spark.sql.execution.streaming.CompactibleFileStreamLog.<init>(CompactibleFileStreamLog.scala:46)
> at
> org.apache.spark.sql.execution.streaming.FileStreamSinkLog.<init>(FileStreamSinkLog.scala:85)
> at
> org.apache.spark.sql.execution.streaming.FileStreamSink.<init>(FileStreamSink.scala:98)
> at
> org.apache.spark.sql.execution.datasources.DataSource.createSink(DataSource.scala:297)
> at
> org.apache.spark.sql.execution.datasources.FindDataSourceTable$$anonfun$apply$2.applyOrElse(DataSourceStrategy.scala:379)
> ...
> Caused by: java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at
> org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:134)
> ... 67 more
> Caused by: org.apache.hadoop.HadoopIllegalArgumentException: Uri without
> authority: hdfs:/tmp/test/_spark_metadata
> at
> org.apache.hadoop.fs.AbstractFileSystem.getUri(AbstractFileSystem.java:313)
> at
> org.apache.hadoop.fs.AbstractFileSystem.<init>(AbstractFileSystem.java:266)
> at org.apache.hadoop.fs.Hdfs.<init>(Hdfs.java:80)
> ... 72 more
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
