sivabalan narayanan created HUDI-7127:
-----------------------------------------
Summary: Fix closure of Spark context in tests
Key: HUDI-7127
URL: https://issues.apache.org/jira/browse/HUDI-7127
Project: Apache Hudi
Issue Type: Improvement
Components: tests-ci
Reporter: sivabalan narayanan
For the past3 days, we are seeing CI tests failing due to spark context not
properly shut down.
Ref:
[https://dev.azure.com/apache-hudi-ci-org/785b6ef4-2f42-4a89-8f0e-5f0d7039a0cc/_apis/build/builds/21029/logs/28]
stacktrace:
{code:java}
2023-11-21T00:34:10.1321683Z [INFO] Tests run: 4, Failures: 0, Errors: 0,
Skipped: 0, Time elapsed: 12.375 s - in
org.apache.hudi.TestHoodieParquetBloomFilter
2023-11-21T00:34:10.1327664Z [INFO] Running org.apache.hudi.util.TestPathUtils
2023-11-21T00:34:10.2175283Z [INFO] Tests run: 1, Failures: 0, Errors: 0,
Skipped: 0, Time elapsed: 0.081 s - in org.apache.hudi.util.TestPathUtils
2023-11-21T00:34:10.2256379Z [INFO] Running
org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle
2023-11-21T00:34:14.7733707Z [ERROR] Tests run: 5, Failures: 0, Errors: 5,
Skipped: 0, Time elapsed: 4.53 s <<< FAILURE! - in
org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle
2023-11-21T00:34:14.7735064Z [ERROR] testInstantiationFailure{boolean}[1] Time
elapsed: 1.619 s <<< ERROR!
2023-11-21T00:34:14.7743733Z org.apache.spark.SparkException:
2023-11-21T00:34:14.7744752Z Only one SparkContext should be running in this
JVM (see SPARK-2243).The currently running SparkContext was created at:
2023-11-21T00:34:14.7745761Z
org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:943)
2023-11-21T00:34:14.7746644Z
org.apache.hudi.TestHoodieParquetBloomFilter.initSparkContext(TestHoodieParquetBloom.scala:47)
2023-11-21T00:34:14.7747562Z
org.apache.hudi.TestHoodieParquetBloomFilter.setUp(TestHoodieParquetBloom.scala:57)
2023-11-21T00:34:14.7748262Z
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2023-11-21T00:34:14.7748971Z
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2023-11-21T00:34:14.7749798Z
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2023-11-21T00:34:14.7750464Z java.lang.reflect.Method.invoke(Method.java:498)
2023-11-21T00:34:14.7751108Z
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
2023-11-21T00:34:14.7752199Z
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
2023-11-21T00:34:14.7753349Z
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
2023-11-21T00:34:14.7754508Z
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
2023-11-21T00:34:14.7755527Z
org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
2023-11-21T00:34:14.7756642Z
org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
2023-11-21T00:34:14.7758217Z
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
2023-11-21T00:34:14.7759419Z
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
2023-11-21T00:34:14.7760646Z
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
2023-11-21T00:34:14.7761922Z
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
2023-11-21T00:34:14.7763079Z
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
2023-11-21T00:34:14.7764245Z
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
2023-11-21T00:34:14.7765256Z
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
2023-11-21T00:34:14.7766227Z at
org.apache.spark.SparkContext$.$anonfun$assertNoOtherContextIsRunning$2(SparkContext.scala:2647)
2023-11-21T00:34:14.7766950Z at scala.Option.foreach(Option.scala:407)
2023-11-21T00:34:14.7767574Z at
org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2644)
2023-11-21T00:34:14.7768453Z at
org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2734)
2023-11-21T00:34:14.7769263Z at
org.apache.spark.SparkContext.<init>(SparkContext.scala:95)
2023-11-21T00:34:14.7769965Z at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
2023-11-21T00:34:14.7770961Z at
org.apache.hudi.testutils.HoodieSparkClientTestHarness.initSparkContexts(HoodieSparkClientTestHarness.java:194)
2023-11-21T00:34:14.7772104Z at
org.apache.hudi.io.storage.row.TestHoodieRowCreateHandle.setUp(TestHoodieRowCreateHandle.java:67)
2023-11-21T00:34:14.7772924Z at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2023-11-21T00:34:14.7773606Z at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2023-11-21T00:34:14.7774468Z at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2023-11-21T00:34:14.7775165Z at
java.lang.reflect.Method.invoke(Method.java:498)
2023-11-21T00:34:14.7775857Z at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
2023-11-21T00:34:14.7776774Z at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
2023-11-21T00:34:14.7777931Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
2023-11-21T00:34:14.7779123Z at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
2023-11-21T00:34:14.7780161Z at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
2023-11-21T00:34:14.7781304Z at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
2023-11-21T00:34:14.7782593Z at
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
2023-11-21T00:34:14.7783838Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
2023-11-21T00:34:14.7785097Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
2023-11-21T00:34:14.7786390Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
2023-11-21T00:34:14.7787586Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
2023-11-21T00:34:14.7788776Z at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
2023-11-21T00:34:14.7789819Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
2023-11-21T00:34:14.7790737Z at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
2023-11-21T00:34:14.7795902Z at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:490)
2023-11-21T00:34:14.7797565Z at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$19(ClassBasedTestDescriptor.java:475)
2023-11-21T00:34:14.7799096Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:167)
2023-11-21T00:34:14.7800722Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:195)
2023-11-21T00:34:14.7802173Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-11-21T00:34:14.7803561Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:195)
2023-11-21T00:34:14.7805069Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:164)
2023-11-21T00:34:14.7806305Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:127)
2023-11-21T00:34:14.7807594Z at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
2023-11-21T00:34:14.7808787Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
2023-11-21T00:34:14.7809936Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-11-21T00:34:14.7811108Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-11-21T00:34:14.7812108Z at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-11-21T00:34:14.7813092Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-11-21T00:34:14.7814260Z at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-11-21T00:34:14.7815366Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-11-21T00:34:14.7816406Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-11-21T00:34:14.7817734Z at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
2023-11-21T00:34:14.7819238Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
2023-11-21T00:34:14.7820538Z at
org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
2023-11-21T00:34:14.7821776Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
2023-11-21T00:34:14.7823022Z at
org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
2023-11-21T00:34:14.7824043Z at
java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-11-21T00:34:14.7824795Z at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
