Github user vanzin commented on a diff in the pull request: https://github.com/apache/spark/pull/18801#discussion_r156240364 --- Diff: core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala --- @@ -1216,7 +1216,36 @@ private[spark] object SparkSubmitUtils { /** A nice function to use in tests as well. Values are dummy strings. */ def getModuleDescriptor: DefaultModuleDescriptor = DefaultModuleDescriptor.newDefaultInstance( - ModuleRevisionId.newInstance("org.apache.spark", "spark-submit-parent", "1.0")) + // Include timestamp in module name, so multiple clients resolving maven coordinates at the + // same time do not modify the same resolution file concurrently. + ModuleRevisionId.newInstance("org.apache.spark", + "spark-submit-parent-" + System.currentTimeMillis().toString, + "1.0")) + + /** + * clear ivy resolution from current launch. The resolution file is usually at + * ~/.ivy2/org.apache.spark-spark-submit-parent-$timestamp-default.xml, + * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$timestamp-1.0.xml, and + * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$timestamp-1.0.properties. + * Since each launch will have its own resolution files created, delete them after + * each resolution to prevent accumulation of these files in the ivy cache dir. + */ + private def clearIvyResolutionFiles( + mdId: ModuleRevisionId, + ivySettings: IvySettings, + ivyConfName: String): Unit = { + val currentResolutionFiles = Seq[File]( + new File(ivySettings.getDefaultCache, + s"${mdId.getOrganisation}-${mdId.getName}-$ivyConfName.xml"), + new File(ivySettings.getDefaultCache, + s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.xml"), + new File(ivySettings.getDefaultCache, + s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.properties") + ) + currentResolutionFiles.foreach{ file => --- End diff -- space before `{`
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org