[
https://issues.apache.org/jira/browse/HUDI-7127?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan updated HUDI-7127:
--------------------------------------
Epic Link: HUDI-4302
Labels: (was: pull-request-available)
> Fix closure of Spark context in tests
> -------------------------------------
>
> Key: HUDI-7127
> URL: https://issues.apache.org/jira/browse/HUDI-7127
> Project: Apache Hudi
> Issue Type: Improvement
> Components: tests-ci
> Reporter: sivabalan narayanan
> Priority: Major
>
> For the past3 days, we are seeing CI tests failing due to spark context not
> properly shut down.
>
> Ref:
> [https://dev.azure.com/apache-hudi-ci-org/785b6ef4-2f42-4a89-8f0e-5f0d7039a0cc/_apis/build/builds/21029/logs/28]
>
>
> stacktrace:
> {code:java}
> 2023-11-21T00:34:10.1321683Z [INFO] Tests run: 4, Failures: 0, Errors: 0,
> Skipped: 0, Time elapsed: 12.375 s - in
> org.apache.hudi.TestHoodieParquetBloomFilter
> 2023-11-21T00:34:10.1327664Z [INFO] Running org.apache.hudi.util.TestPathUtils
> 2023-11-21T00:34:10.2175283Z [INFO] Tests run: 1, Failures: 0, Errors: 0,
> Skipped: 0, Time elapsed: 0.081 s - in org.apache.hudi.util.TestPathUtils
> 2023-11-21T00:34:10.2256379Z [INFO] Running
> org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle
> 2023-11-21T00:34:14.7733707Z [ERROR] Tests run: 5, Failures: 0, Errors: 5,
> Skipped: 0, Time elapsed: 4.53 s <<< FAILURE! - in
> org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle
> 2023-11-21T00:34:14.7735064Z [ERROR] testInstantiationFailure{boolean}[1]
> Time elapsed: 1.619 s <<< ERROR!
> 2023-11-21T00:34:14.7743733Z org.apache.spark.SparkException:
> 2023-11-21T00:34:14.7744752Z Only one SparkContext should be running in this
> JVM (see SPARK-2243).The currently running SparkContext was created at:
> 2023-11-21T00:34:14.7745761Z
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:943)
> 2023-11-21T00:34:14.7746644Z
> org.apache.hudi.TestHoodieParquetBloomFilter.initSparkContext(TestHoodieParquetBloom.scala:47)
> 2023-11-21T00:34:14.7747562Z
> org.apache.hudi.TestHoodieParquetBloomFilter.setUp(TestHoodieParquetBloom.scala:57)
> 2023-11-21T00:34:14.7748262Z
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2023-11-21T00:34:14.7748971Z
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2023-11-21T00:34:14.7749798Z
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2023-11-21T00:34:14.7750464Z java.lang.reflect.Method.invoke(Method.java:498)
> 2023-11-21T00:34:14.7751108Z
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> 2023-11-21T00:34:14.7752199Z
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> 2023-11-21T00:34:14.7753349Z
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> 2023-11-21T00:34:14.7754508Z
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> 2023-11-21T00:34:14.7755527Z
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> 2023-11-21T00:34:14.7756642Z
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> 2023-11-21T00:34:14.7758217Z
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> 2023-11-21T00:34:14.7759419Z
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> 2023-11-21T00:34:14.7760646Z
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> 2023-11-21T00:34:14.7761922Z
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> 2023-11-21T00:34:14.7763079Z
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> 2023-11-21T00:34:14.7764245Z
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> 2023-11-21T00:34:14.7765256Z
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> 2023-11-21T00:34:14.7766227Z at
> org.apache.spark.SparkContext$.$anonfun$assertNoOtherContextIsRunning$2(SparkContext.scala:2647)
> 2023-11-21T00:34:14.7766950Z at scala.Option.foreach(Option.scala:407)
> 2023-11-21T00:34:14.7767574Z at
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2644)
> 2023-11-21T00:34:14.7768453Z at
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2734)
> 2023-11-21T00:34:14.7769263Z at
> org.apache.spark.SparkContext.<init>(SparkContext.scala:95)
> 2023-11-21T00:34:14.7769965Z at
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> 2023-11-21T00:34:14.7770961Z at
> org.apache.hudi.testutils.HoodieSparkClientTestHarness.initSparkContexts(HoodieSparkClientTestHarness.java:194)
> 2023-11-21T00:34:14.7772104Z at
> org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle.setUp(TestHoodieRowCreateHandle.java:67)
> 2023-11-21T00:34:14.7772924Z at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2023-11-21T00:34:14.7773606Z at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2023-11-21T00:34:14.7774468Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2023-11-21T00:34:14.7775165Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2023-11-21T00:34:14.7775857Z at
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> 2023-11-21T00:34:14.7776774Z at
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> 2023-11-21T00:34:14.7777931Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> 2023-11-21T00:34:14.7779123Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> 2023-11-21T00:34:14.7780161Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> 2023-11-21T00:34:14.7781304Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> 2023-11-21T00:34:14.7782593Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> 2023-11-21T00:34:14.7783838Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> 2023-11-21T00:34:14.7785097Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> 2023-11-21T00:34:14.7786390Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> 2023-11-21T00:34:14.7787586Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> 2023-11-21T00:34:14.7788776Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> 2023-11-21T00:34:14.7789819Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> 2023-11-21T00:34:14.7790737Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> 2023-11-21T00:34:14.7795902Z at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:490)
> 2023-11-21T00:34:14.7797565Z at
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$19(ClassBasedTestDescriptor.java:475)
> 2023-11-21T00:34:14.7799096Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:167)
> 2023-11-21T00:34:14.7800722Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:195)
> 2023-11-21T00:34:14.7802173Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-11-21T00:34:14.7803561Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:195)
> 2023-11-21T00:34:14.7805069Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:164)
> 2023-11-21T00:34:14.7806305Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:127)
> 2023-11-21T00:34:14.7807594Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
> 2023-11-21T00:34:14.7808787Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> 2023-11-21T00:34:14.7809936Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-11-21T00:34:14.7811108Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2023-11-21T00:34:14.7812108Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2023-11-21T00:34:14.7813092Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2023-11-21T00:34:14.7814260Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2023-11-21T00:34:14.7815366Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2023-11-21T00:34:14.7816406Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2023-11-21T00:34:14.7817734Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> 2023-11-21T00:34:14.7819238Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
> 2023-11-21T00:34:14.7820538Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
> 2023-11-21T00:34:14.7821776Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
> 2023-11-21T00:34:14.7823022Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
> 2023-11-21T00:34:14.7824043Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2023-11-21T00:34:14.7824795Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> {code}
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
