[
https://issues.apache.org/jira/browse/BEAM-9613?focusedWorklogId=575262&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-575262
]
ASF GitHub Bot logged work on BEAM-9613:
----------------------------------------
Author: ASF GitHub Bot
Created on: 01/Apr/21 00:55
Start Date: 01/Apr/21 00:55
Worklog Time Spent: 10m
Work Description: mouyang commented on pull request #14350:
URL: https://github.com/apache/beam/pull/14350#issuecomment-811561722
@TheNeuralBit thanks for looking into it for me. I didn't know how to avoid
vertical scroll hell previously, but I found out and posted what showed up in
the below. I couldn't find anything specific to a permission error or anything
specific for that matter.
In the mean time, I'll make the changes you hinted at and give it a another
blind go.
<details>
<summary>Error in Test Report</summary>
org.apache.beam.sdk.Pipeline$PipelineExecutionException:
java.lang.RuntimeException: Failed to create job with prefix
beam_bq_job_LOAD_bigqueryiowritetest0testwriteavro000maouyan03282032418eff7d5e_a5734b516aaf48229aaf8a974328bcf8_1af13eb133f5d1e510c8e70fecb762c7_00001_00000,
reached max retries: 3, last failed job: {
"configuration" : {
"load" : {
"createDisposition" : "CREATE_IF_NEEDED",
"destinationTable" : {
"datasetId" : "dataset-id",
"projectId" : "project-id",
"tableId" : "table-id"
},
"ignoreUnknownValues" : false,
"schemaUpdateOptions" : [ ],
"sourceFormat" : "AVRO",
"useAvroLogicalTypes" : false,
"writeDisposition" : "WRITE_EMPTY"
}
},
"jobReference" : {
"jobId" :
"beam_bq_job_LOAD_bigqueryiowritetest0testwriteavro000maouyan03282032418eff7d5e_a5734b516aaf48229aaf8a974328bcf8_1af13eb133f5d1e510c8e70fecb762c7_00001_00000-2",
"projectId" : "project-id"
},
"kind" : " bigquery#job",
"status" : {
"errorResult" : { },
"state" : "FAILED"
}
}.
at
org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.waitUntilFinish(DirectRunner.java:371)
at
org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.waitUntilFinish(DirectRunner.java:339)
at
org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:219)
at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:67)
at org.apache.beam.sdk.Pipeline.run(Pipeline.java:322)
at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:398)
at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:334)
at
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOWriteTest.testWriteAvro(BigQueryIOWriteTest.java:916)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at
org.apache.beam.sdk.testing.TestPipeline$1.evaluate(TestPipeline.java:322)
at
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOWriteTest$1$1.evaluate(BigQueryIOWriteTest.java:193)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at
org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:258)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:110)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38)
at
org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:62)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:119)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
at
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414)
at
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
at
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at
org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Failed to create job with prefix
beam_bq_job_LOAD_bigqueryiowritetest0testwriteavro000maouyan03282032418eff7d5e_a5734b516aaf48229aaf8a974328bcf8_1af13eb133f5d1e510c8e70fecb762c7_00001_00000,
reached max retries: 3, last failed job: {
"configuration" : {
"load" : {
"createDisposition" : "CREATE_IF_NEEDED",
"destinationTable" : {
"datasetId" : "dataset-id",
"projectId" : "project-id",
"tableId" : "table-id"
},
"ignoreUnknownValues" : false,
"schemaUpdateOptions" : [ ],
"sourceFormat" : "AVRO",
"useAvroLogicalTypes" : false,
"writeDisposition" : "WRITE_EMPTY"
}
},
"jobReference" : {
"jobId" :
"beam_bq_job_LOAD_bigqueryiowritetest0testwriteavro000maouyan03282032418eff7d5e_a5734b516aaf48229aaf8a974328bcf8_1af13eb133f5d1e510c8e70fecb762c7_00001_00000-2",
"projectId" : "project-id"
},
"kind" : " bigquery#job",
"status" : {
"errorResult" : { },
"state" : "FAILED"
}
}.
at
org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers$PendingJob.runJob(BigQueryHelpers.java:199)
at
org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers$PendingJobManager.waitForDone(BigQueryHelpers.java:152)
at
org.apache.beam.sdk.io.gcp.bigquery.WriteTables$WriteTablesDoFn.finishBundle(WriteTables.java:285)
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 575262)
Time Spent: 1h 40m (was: 1.5h)
> BigQuery IO not support convert double type for beam row
> --------------------------------------------------------
>
> Key: BEAM-9613
> URL: https://issues.apache.org/jira/browse/BEAM-9613
> Project: Beam
> Issue Type: Bug
> Components: io-java-gcp
> Affects Versions: 2.28.0
> Reporter: MAKSIM TSYGAN
> Assignee: Matthew Ouyang
> Priority: P3
> Fix For: 2.30.0
>
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> If execute query with double column via BigQueryIO.readFrom(), I get
> exception:
> Caused by: java.lang.UnsupportedOperationException: Converting BigQuery type
> 'class java.lang.Double' to 'FieldType\{typeName=DOUBLE, nullable=true,
> logicalType=null, collectionElementType=null, mapKeyType=null,
> mapValueType=null, rowSchema=null, metadata={}}' is not supported
> at
> org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toBeamValue(BigQueryUtils.java:532)
> at
> org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toBeamRowFieldValue(BigQueryUtils.java:483)
> at
> org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.lambda$toBeamRow$6(BigQueryUtils.java:469)
> at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
> at
> org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toBeamRow(BigQueryUtils.java:470)
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
