radhikabajaj123 opened a new issue, #872: URL: https://github.com/apache/datafusion-comet/issues/872
Hello, I am trying to run the spark shell with comet enabled following the configurations specified at https://datafusion.apache.org/comet/user-guide/installation.html#installing-datafusion-comet, after creating a local build. I was previously able to launch the spark shell successfully after cloning the Comet project, however, it now gives this exception: ` at org.apache.spark.SparkContext.addLocalJarFile$1(SparkContext.scala:1968) at org.apache.spark.SparkContext.addJar(SparkContext.scala:2023) at org.apache.spark.SparkContext.$anonfun$new$12(SparkContext.scala:507) at org.apache.spark.SparkContext.$anonfun$new$12$adapted(SparkContext.scala:507) at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563) at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561) at scala.collection.AbstractIterable.foreach(Iterable.scala:926) at org.apache.spark.SparkContext.<init>(SparkContext.scala:507) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2740) at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:1026) at scala.Option.getOrElse(Option.scala:201) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:1020) at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114) at $line3.$read$$iw.<init>(<console>:5) at $line3.$read.<init>(<console>:4) at $line3.$read$.<clinit>(<console>:1) at $line3.$eval$.$print$lzycompute(<synthetic>:6) at $line3.$eval$.$print(<synthetic>:5) at $line3.$eval.$print(<synthetic>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670) at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1020) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506) at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36) at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116) at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43) at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501) at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924) at scala.collection.immutable.List.foreach(List.scala:333) at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1420) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954) at org.apache.spark.repl.Main$.doMain(Main.scala:84) at org.apache.spark.repl.Main$.main(Main.scala:59) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1020) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:192) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:215) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1111) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1120) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 24/08/26 17:08:39 ERROR SparkContext: Error initializing SparkContext. java.lang.ClassNotFoundException: org.apache.spark.CometPlugin at scala.reflect.internal.util.AbstractFileClassLoader.findClass(AbstractFileClassLoader.scala:75) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.spark.util.Utils$.classForName(Utils.scala:225) at org.apache.spark.util.Utils$.$anonfun$loadExtensions$1(Utils.scala:2946) at scala.collection.StrictOptimizedIterableOps.flatMap(StrictOptimizedIterableOps.scala:118) at scala.collection.StrictOptimizedIterableOps.flatMap$(StrictOptimizedIterableOps.scala:105) at scala.collection.immutable.ArraySeq.flatMap(ArraySeq.scala:35) at org.apache.spark.util.Utils$.loadExtensions(Utils.scala:2944) at org.apache.spark.internal.plugin.PluginContainer$.apply(PluginContainer.scala:207) at org.apache.spark.internal.plugin.PluginContainer$.apply(PluginContainer.scala:193) at org.apache.spark.SparkContext.<init>(SparkContext.scala:565) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2740) at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:1026) at scala.Option.getOrElse(Option.scala:201) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:1020) at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114) at $line3.$read$$iw.<init>(<console>:5) at $line3.$read.<init>(<console>:4) at $line3.$read$.<clinit>(<console>:1) at $line3.$eval$.$print$lzycompute(<synthetic>:6) at $line3.$eval$.$print(<synthetic>:5) at $line3.$eval.$print(<synthetic>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670) at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1020) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506) at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36) at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116) at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43) at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501) at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924) at scala.collection.immutable.List.foreach(List.scala:333) at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1420) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954) at org.apache.spark.repl.Main$.doMain(Main.scala:84) at org.apache.spark.repl.Main$.main(Main.scala:59) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1020) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:192) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:215) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1111) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1120) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 24/08/26 17:08:39 INFO SparkContext: SparkContext is stopping with exitCode 0. 24/08/26 17:08:39 INFO SparkUI: Stopped Spark web UI at http://n-chafqtlvmeh6ad1j7j7f3.workdaysuv.com:4040 24/08/26 17:08:39 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 24/08/26 17:08:39 INFO MemoryStore: MemoryStore cleared 24/08/26 17:08:39 INFO BlockManager: BlockManager stopped 24/08/26 17:08:39 INFO BlockManagerMaster: BlockManagerMaster stopped 24/08/26 17:08:39 WARN MetricsSystem: Stopping a MetricsSystem that is not running 24/08/26 17:08:39 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 24/08/26 17:08:39 INFO SparkContext: Successfully stopped SparkContext 24/08/26 17:08:39 ERROR Main: Failed to initialize Spark session. java.lang.ClassNotFoundException: org.apache.spark.CometPlugin at scala.reflect.internal.util.AbstractFileClassLoader.findClass(AbstractFileClassLoader.scala:75) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.spark.util.Utils$.classForName(Utils.scala:225) at org.apache.spark.util.Utils$.$anonfun$loadExtensions$1(Utils.scala:2946) at scala.collection.StrictOptimizedIterableOps.flatMap(StrictOptimizedIterableOps.scala:118) at scala.collection.StrictOptimizedIterableOps.flatMap$(StrictOptimizedIterableOps.scala:105) at scala.collection.immutable.ArraySeq.flatMap(ArraySeq.scala:35) at org.apache.spark.util.Utils$.loadExtensions(Utils.scala:2944) at org.apache.spark.internal.plugin.PluginContainer$.apply(PluginContainer.scala:207) at org.apache.spark.internal.plugin.PluginContainer$.apply(PluginContainer.scala:193) at org.apache.spark.SparkContext.<init>(SparkContext.scala:565) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2740) at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:1026) at scala.Option.getOrElse(Option.scala:201) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:1020) at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114) at $line3.$read$$iw.<init>(<console>:5) at $line3.$read.<init>(<console>:4) at $line3.$read$.<clinit>(<console>:1) at $line3.$eval$.$print$lzycompute(<synthetic>:6) at $line3.$eval$.$print(<synthetic>:5) at $line3.$eval.$print(<synthetic>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670) at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1020) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506) at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36) at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116) at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43) at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505) at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501) at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924) at scala.collection.immutable.List.foreach(List.scala:333) at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1420) at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64) at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954) at org.apache.spark.repl.Main$.doMain(Main.scala:84) at org.apache.spark.repl.Main$.main(Main.scala:59) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1020) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:192) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:215) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1111) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1120) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
