stevedlawrence commented on code in PR #183:
URL: https://github.com/apache/daffodil-sbt/pull/183#discussion_r3258801656
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -887,123 +887,159 @@ object DaffodilPlugin extends AutoPlugin {
daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
daffodilFlattenResourceReferencePatterns := List(
- """schemaLocation=\"([^\"]*)\"""".r,
- "href=\"([^\"]*)\"".r,
- "document[(]'([^']*)'[)]".r),
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
daffodilFlattenSchemas / products := {
val logger = streams.value.log
- val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
val flatDir = target.value / "flatDir"
if (flatDir.exists())
IO.delete(flatDir)
IO.createDirectory(flatDir)
- val projectResources = (Compile / resourceDirectories).value
- .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
- .toMap
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
/**
* Create a URLClassLoader object with URLs to all resources used by the
* project. The class loader will be used to resolve references made
* within the flattened files.
*/
- val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
-
- val processed = scala.collection.mutable.Set[URI]()
- projectResources foreach { case (rootURL, urls) =>
- val unprocessed = scala.collection.mutable.Stack[URI]()
- val rootURI = rootURL.toURI
- val rootPath = Paths.get(rootURI)
- unprocessed.pushAll(urls.map(_.toURI))
- while (!unprocessed.isEmpty) {
- val contextURI = unprocessed.pop
- val bytes = contextURI.toURL.openStream().readAllBytes()
- val contextRelPath = contextURI.getScheme match {
- case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
- case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
- case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
- }
- val contextFlatPath = Paths.get(
- flatDir.toString,
- contextRelPath.toString.replaceAll("/", "__"))
- val bw = Files.newBufferedWriter(contextFlatPath)
- val fileAsString = new String(bytes, Charset.defaultCharset())
-
- val references = referenceRegexes.flatMap { re =>
- re.findAllIn(fileAsString).matchData.map(_.group(1))
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
Review Comment:
SBT supporst using slashes to build up file paths, so I think these kinds of
Paths.get can be simplified to
```scala
flatDir / cPath.toString.tail.replaceAll("/", "__")
```
Using Paths.XYZ or Files.XYZ is not very common in SBT plugins.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ ref -> Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ ref -> Option(classLoader.findResource(ref))
Review Comment:
This fallback currently only happens if `contextURI` is a file, but I think
we also need it if contextURI is a jar, since a file in a jar could also use
the deprecated behavior. Maybe change to have this match/case return a
Some/None for "file" and None for "jar", and then if the match/case returns
None then do the fall back?
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -887,123 +887,159 @@ object DaffodilPlugin extends AutoPlugin {
daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
daffodilFlattenResourceReferencePatterns := List(
- """schemaLocation=\"([^\"]*)\"""".r,
- "href=\"([^\"]*)\"".r,
- "document[(]'([^']*)'[)]".r),
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
daffodilFlattenSchemas / products := {
val logger = streams.value.log
- val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
val flatDir = target.value / "flatDir"
if (flatDir.exists())
IO.delete(flatDir)
IO.createDirectory(flatDir)
- val projectResources = (Compile / resourceDirectories).value
- .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
- .toMap
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
/**
* Create a URLClassLoader object with URLs to all resources used by the
* project. The class loader will be used to resolve references made
* within the flattened files.
*/
- val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
-
- val processed = scala.collection.mutable.Set[URI]()
- projectResources foreach { case (rootURL, urls) =>
- val unprocessed = scala.collection.mutable.Stack[URI]()
- val rootURI = rootURL.toURI
- val rootPath = Paths.get(rootURI)
- unprocessed.pushAll(urls.map(_.toURI))
- while (!unprocessed.isEmpty) {
- val contextURI = unprocessed.pop
- val bytes = contextURI.toURL.openStream().readAllBytes()
- val contextRelPath = contextURI.getScheme match {
- case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
- case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
- case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
- }
- val contextFlatPath = Paths.get(
- flatDir.toString,
- contextRelPath.toString.replaceAll("/", "__"))
- val bw = Files.newBufferedWriter(contextFlatPath)
- val fileAsString = new String(bytes, Charset.defaultCharset())
-
- val references = referenceRegexes.flatMap { re =>
- re.findAllIn(fileAsString).matchData.map(_.group(1))
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
Review Comment:
Can this be `startsWith` instead of `contains`? Should be slightly more
efficient since it wont' have to scan the whole string, and the contextURI
should start with one of the resource directories.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
Review Comment:
Thoughts on refactoring this to remove the duplicate `ref -> foo`'s
```scala
val resolvedRefs = references.map { ref =>
val optResolved = ... // all of this logic, returning only Some/None
ref -> optResolved
}
```
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
Review Comment:
I'm wondering if we should close the FileSystems we create at some point?
Plugins run in the same JVM as SBT, so I'm wondering if leaving FileSystems
open could cause potential issues. I guess SBT wont' have access to these
FileSystems so it wont' accidentally reference them, but I guess it could lead
file descriptors open or something? Note sure the best way do to that. Do we
keep track of all file systems we create and then close them all at the end. Or
just open and close for every processing loop? Feels like that might be
inefficient, but maybe it's fast enough?
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ ref -> Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ ref -> Option(classLoader.findResource(ref))
+ }
+ }
+ case _ => ref -> None
+ }
+ }
+ }
+ }.toMap
+
+ // Warn about unresolved references
+ resolvedRefs.filter(_._2 == None).map { case (ref, _) =>
+ logger.warn(s"Unable to resolve reference to $ref from source file
$contextURI")
+ }
+
+ val fullyResolved = resolvedRefs.filter(_._2 != None)
+
+ val updatedFileAsString = {
+ fullyResolved.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.get.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" => resolvedURI.toString.split("!")(1).tail
+ case _ => {
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ resolvedURI.toString.contains(root.toString)
+ )
+ resolvedRoot.get.toURI.relativize(resolvedURI).toString
+ }
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+ seen += contextURI
+ val unseen = fullyResolved.values.map(_.get.toURI).filterNot(seen)
Review Comment:
I think we need to add these unseen references to `seen` at this point.
Otherwise we might end up adding them to the unprocessed stack multiple times
and end up processing them multiple times. For example, say we have:
* A references B and C
* B references C
* C references nothing
And say A is a file and B and C come from jars.
1. We initialize both `seen` and `unprocessed` to contain A, since it's our
only file.
2. We pop A off the stack and process it, and find references to B and C. We
have not seen B or C and so push them to `unprocessed`. And assume we do not
add either to `seen` as you have now. So a this point after processing A,
`unprocessed` contains B and C, and `seen` contains A.
3. We loop around, and pop B off the stack. We find a reference to C. C is
not in `seen` so we push it to the `unprocessed` stack. But C is already on the
stack so there are two instances of it and we'll end up processing it twice.
If instead, at step 2 we add both B and C to `seen` , then when step 3 sees
C it will already be in `seen` and so won't get added to the `unprocessed`
stack a second time.
Since set.add returns a boolean if an element was added for the first time,
we can use that to add a file to `seen` *and* get a list of things that weren't
already in it. So I think this wants to be sometime tlike this (witha comment
that I think clarifies what's going on, since it's not super obvious what role
`seen` plays).
```scala
// We want to push each found reference to the unprocessed stack exactly
once.
// This avoids processing the same file multiple times if it's referenced
// across multiple files. We can't rely on the unprocessed stack itself to
// detect if we've added a reference before, since references are removed
from
// the stack once processed. Instead, we use the "seen" set as a record of
// everything that's ever been pushed to the stack. Note that seen.add
ignores
// duplicates and also returns true for references that are added to the set
// for the first time. So filtering on it gives us exactly the references
that
// haven't been seen before and so should be pushed to the unprocessed stack
// unprocessed stack
val unseen = fullyResolved.values.map(_.get.toURI).filter(seen.add)
unprocessed.pushAll(unseen)
```
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ ref -> Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ ref -> Option(classLoader.findResource(ref))
+ }
+ }
+ case _ => ref -> None
+ }
+ }
+ }
+ }.toMap
+
+ // Warn about unresolved references
+ resolvedRefs.filter(_._2 == None).map { case (ref, _) =>
+ logger.warn(s"Unable to resolve reference to $ref from source file
$contextURI")
+ }
+
+ val fullyResolved = resolvedRefs.filter(_._2 != None)
+
+ val updatedFileAsString = {
+ fullyResolved.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.get.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" => resolvedURI.toString.split("!")(1).tail
+ case _ => {
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ resolvedURI.toString.contains(root.toString)
+ )
+ resolvedRoot.get.toURI.relativize(resolvedURI).toString
+ }
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+ seen += contextURI
Review Comment:
I think this isn't needed? contextURI came from the `unprocessed` stack, and
anything added to the that stack should have already be added to the `seen` set.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ ref -> Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ ref -> Option(classLoader.findResource(ref))
+ }
+ }
+ case _ => ref -> None
+ }
+ }
+ }
+ }.toMap
+
+ // Warn about unresolved references
+ resolvedRefs.filter(_._2 == None).map { case (ref, _) =>
+ logger.warn(s"Unable to resolve reference to $ref from source file
$contextURI")
+ }
+
+ val fullyResolved = resolvedRefs.filter(_._2 != None)
+
+ val updatedFileAsString = {
+ fullyResolved.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.get.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" => resolvedURI.toString.split("!")(1).tail
+ case _ => {
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ resolvedURI.toString.contains(root.toString)
+ )
+ resolvedRoot.get.toURI.relativize(resolvedURI).toString
+ }
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
Review Comment:
It feels like we still need to include the jar file in the name. The
FileSystem trick allows jars to resolve relative paths to files in the same
jar, but the resulting relativized paths could still be the same once flattened.
For example, say we have foo.jar and bar.jar jars with these files:
```
foo.jar!/foo.dfdl.xsd // contains relative
schemaLocation="defaults.dfdl.xsd"
foo.jar!/defaults.dfdl.xsd
bar.jar!/bar.dfdl.xsd // contains relative
schemaLocation="defaults.dfdl.xsd"
bar.jar!/defaults.dfdl.xsd
```
Once flattened, this becomes
```
foo.dfdl.xsd
bar.dfdl.xsd
defaults.dfdl.xsd
```
So we only have a single defaults.dfdl.xsd file and both the foo.dfdl.xsd
and bar.dfdl.xsd schemas reference it. But those defaults.dfdl.xsd files could
actually have different content.
If we prepened the jar name to the flattened paths, then the flattened files
become something like
```
foo__foo.dfdl.xsd
foo__defaults.dfdl.xsd
bar__bar.dfdl.xsd
bar__defaults.dfdl.xsd
```
And the foo and bar schemas can now reference their respective default
schema.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
Review Comment:
I think it's worth adding a comment about contextPath (maybe here or maybe
when you call resolveSibling/exists), that it's a Path to either a normal file
on the default FileSystem or a Path to a "jar" FileSystem, and that this allows
things like resolveSibling/exists to resolve paths relative whatever FileSystem
the path originated from. I didn't realize Path objects were tied to a file
system and things like resolveSibling and exists are still tied to that file
system, and even reading the javadoc that doesn't feel very obvious to me.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
Review Comment:
Remove this `ref -> None`, I 'm guessing you had this before you added the
below code and forgot to remove this
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
Review Comment:
Should this error if flatPath already exists? If a flat file already exists
that implies there are two different resolved URI's that flatten to the same
file and so one is going to overwrite the other. Those two files don't
necessarily have the same content so overwriting one might lead to issues. And
although we usually prefer to include paths that properly namespace files, we
might not always be consistent about it, and we certainly don't enforce it.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,215 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, path) => {
+ val fs = try {
+ FileSystems.getFileSystem(contextURI)
+ } catch {
+ case e: FileSystemNotFoundException =>
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ }
+ val cPath = fs.getPath(path)
+ (cPath, Paths.get(flatDir.toString,
cPath.toString.tail.replaceAll("/", "__")))
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(file => contextURI.toString.contains(file.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ ref -> Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ ref -> Some(refPath.toUri.toURL)
+ } else {
+ ref -> None
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ ref -> Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ ref -> Option(classLoader.findResource(ref))
+ }
+ }
+ case _ => ref -> None
+ }
+ }
+ }
+ }.toMap
+
+ // Warn about unresolved references
+ resolvedRefs.filter(_._2 == None).map { case (ref, _) =>
+ logger.warn(s"Unable to resolve reference to $ref from source file
$contextURI")
Review Comment:
Two thoughts on this:
1. Is it worth erroring instead of warning? It feels like if we fail to find
a reference the resulting flattened schema is very unlikely to work. Are there
cases where it will work anyways and it's difficult for users to use an
alternative reference?
2. It might simplify things to do this diagnostic in resolvedRefs with a
flatMap so we don't have to filter, e.g
```scala
val resolvedRefs = references.flatMap { ref =>
val optResolved = ... // all of this logic, returning only Some/None
if (optResolved.isEmpty) logger.warn/error(...)
optResolved.map(resolved => ref -> resolved)
}.toMap
```
This way the flatMap is responsible for removing None's and you don't
have the filter here or below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
