stevedlawrence commented on code in PR #183:
URL: https://github.com/apache/daffodil-sbt/pull/183#discussion_r3266273271
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,252 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*/(.*).jar)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val unresolved = scala.collection.mutable.ArrayBuffer[(URI, String)]()
+ val jarFileSystems = scala.collection.mutable.Map[String, FileSystem]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+
+ // Get a Java Path for the current file (whether it is in a JAR or
+ // just a regular file) and a Path for where the current file will be
+ // moved to in the flattened directory structure. Note that using a
+ // Java Path on a regular file will use the default FileSystem while
for
+ // JARs a FileSystem is created for interacting with the files inside
+ // the JAR. This allows all Path functions like resolveSibling/exists
to
+ // work with both regular files or files contained inside a JAR.
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, jarName, path) => {
+ val fs = jarFileSystems.getOrElseUpdate(
+ Paths.get(jarPath).toString,
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ )
+ val cPath = fs.getPath(path)
+ (
+ cPath,
+ Paths.get(
+ flatDir.toString,
+ s"${jarName}__${cPath.toString.tail.replaceAll("/", "__")}"
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(dir => contextURI.getPath.startsWith(dir.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ assert(
+ !Files.exists(flatPath),
+ s"File $flatPath already exists and would be overwritten, aborting!"
+ )
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.flatMap { ref =>
+ val optResolved = if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ Some(refPath.toUri.toURL)
+ } else {
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
Review Comment:
I took another look at this, and I think this needs a tweak. As an example
of the issue I'm seeing, say we have foo.dfdl.xsd at this path:
```
file:///format/resourceDirectoryA/com/example/foo.dfdl.xsd
```
And say that file has a reference to schemaLocation="bar.dfdl.xsd". Because
that is a relative path, it should be resolved relative to the above URI, and
so we expect it to be at
```
file:///format/resourceDirectoryA/com/example/bar.dfdl.xsd
```
But if bar.dfdl.xsd was in another resource directory, it would still have
the com/example/bar.dfdl.xsd path, just rooted in a different directory, e.g.
```
file:///format/resourceDirectoryB/com/example/bar.dfdl.xsd
```
So it feels like what we need to is to strip off the root directory from
contextURI, which would give us something like "com/example/foo.dfdl.xsd",
resolve our reference relative to that, which would give us
"com/example/bar.dfdl.xsd", and then to this code to find that path relative to
all of our possible resource directories. Might be other ways to do it, but
some thing along those lines.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,252 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*/(.*).jar)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val unresolved = scala.collection.mutable.ArrayBuffer[(URI, String)]()
+ val jarFileSystems = scala.collection.mutable.Map[String, FileSystem]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+
+ // Get a Java Path for the current file (whether it is in a JAR or
+ // just a regular file) and a Path for where the current file will be
+ // moved to in the flattened directory structure. Note that using a
+ // Java Path on a regular file will use the default FileSystem while
for
+ // JARs a FileSystem is created for interacting with the files inside
+ // the JAR. This allows all Path functions like resolveSibling/exists
to
+ // work with both regular files or files contained inside a JAR.
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, jarName, path) => {
+ val fs = jarFileSystems.getOrElseUpdate(
+ Paths.get(jarPath).toString,
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ )
+ val cPath = fs.getPath(path)
+ (
+ cPath,
+ Paths.get(
+ flatDir.toString,
+ s"${jarName}__${cPath.toString.tail.replaceAll("/", "__")}"
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(dir => contextURI.getPath.startsWith(dir.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ assert(
+ !Files.exists(flatPath),
+ s"File $flatPath already exists and would be overwritten, aborting!"
+ )
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.flatMap { ref =>
+ val optResolved = if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ Some(refPath.toUri.toURL)
+ } else {
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ Option(classLoader.findResource(ref))
+ }
+ }
+ case "jar" => {
+ // Maybe this path in the JAR is actually absolute, just
+ // missing the leading '/', try resolving it relative to the
+ // root of the JAR
+ val jarPath = contextPath.getRoot().resolve(ref)
+ if (Files.exists(jarPath))
+ Some(jarPath.toUri.toURL)
+ else
+ None
+ }
+ case _ => None
+ }
+ }
+ }
+ if (optResolved.isEmpty)
+ unresolved.append((contextURI, ref))
+ optResolved.map(resolved => ref -> resolved)
+ }.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" =>
+ resolvedURI.toString match {
+ case jarRegex(_, jarName, path) =>
+ s"${jarName}__${path.tail.replaceAll("/", "__")}"
+ }
+ case _ => {
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ resolvedURI.toString.contains(root.toString)
+ )
+ resolvedRoot.get.toURI.relativize(resolvedURI).toString
+ }
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+ val unseen = resolvedRefs.values.map(_.toURI).filter(seen.add(_))
+ unprocessed.pushAll(unseen)
+ }
+
+ // Close any open JAR FileSystems
+ jarFileSystems.values.foreach(_.close())
+
+ // Error unresolved references
+ unresolved.foreach { case (context, ref) =>
+ logger.error(s"Unable to resolve reference to $ref from source file
$context")
+ }
+
+ // Error out if we have any unresolved references
+ assert(unresolved.isEmpty)
Review Comment:
This is an error that could indicate a broken schema, so it feels like more
of a user error than an assert. In SBT that way to provider user level error
messages is with a MessageOnlyException. So this should be something like:
```scala
if (!unresolved.isEmpty)
throw new MessageOnlyException("Unable to resolve one or more references
while flattening")
```
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,252 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*/(.*).jar)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val unresolved = scala.collection.mutable.ArrayBuffer[(URI, String)]()
+ val jarFileSystems = scala.collection.mutable.Map[String, FileSystem]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+
+ // Get a Java Path for the current file (whether it is in a JAR or
+ // just a regular file) and a Path for where the current file will be
+ // moved to in the flattened directory structure. Note that using a
+ // Java Path on a regular file will use the default FileSystem while
for
+ // JARs a FileSystem is created for interacting with the files inside
+ // the JAR. This allows all Path functions like resolveSibling/exists
to
+ // work with both regular files or files contained inside a JAR.
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, jarName, path) => {
+ val fs = jarFileSystems.getOrElseUpdate(
+ Paths.get(jarPath).toString,
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ )
+ val cPath = fs.getPath(path)
+ (
+ cPath,
+ Paths.get(
+ flatDir.toString,
+ s"${jarName}__${cPath.toString.tail.replaceAll("/", "__")}"
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(dir => contextURI.getPath.startsWith(dir.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ assert(
+ !Files.exists(flatPath),
+ s"File $flatPath already exists and would be overwritten, aborting!"
+ )
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.flatMap { ref =>
+ val optResolved = if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ Some(refPath.toUri.toURL)
+ } else {
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ Option(classLoader.findResource(ref))
+ }
+ }
+ case "jar" => {
+ // Maybe this path in the JAR is actually absolute, just
+ // missing the leading '/', try resolving it relative to the
+ // root of the JAR
+ val jarPath = contextPath.getRoot().resolve(ref)
+ if (Files.exists(jarPath))
+ Some(jarPath.toUri.toURL)
+ else
+ None
+ }
+ case _ => None
+ }
+ }
+ }
+ if (optResolved.isEmpty)
+ unresolved.append((contextURI, ref))
+ optResolved.map(resolved => ref -> resolved)
+ }.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" =>
+ resolvedURI.toString match {
+ case jarRegex(_, jarName, path) =>
+ s"${jarName}__${path.tail.replaceAll("/", "__")}"
+ }
+ case _ => {
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ resolvedURI.toString.contains(root.toString)
Review Comment:
Above you updated something similar to `contextURI.getPath.startsWith`, can
this do something similar for consistency?
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,252 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*/(.*).jar)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val unresolved = scala.collection.mutable.ArrayBuffer[(URI, String)]()
+ val jarFileSystems = scala.collection.mutable.Map[String, FileSystem]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+
+ // Get a Java Path for the current file (whether it is in a JAR or
+ // just a regular file) and a Path for where the current file will be
+ // moved to in the flattened directory structure. Note that using a
+ // Java Path on a regular file will use the default FileSystem while
for
+ // JARs a FileSystem is created for interacting with the files inside
+ // the JAR. This allows all Path functions like resolveSibling/exists
to
+ // work with both regular files or files contained inside a JAR.
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, jarName, path) => {
+ val fs = jarFileSystems.getOrElseUpdate(
+ Paths.get(jarPath).toString,
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ )
+ val cPath = fs.getPath(path)
+ (
+ cPath,
+ Paths.get(
+ flatDir.toString,
+ s"${jarName}__${cPath.toString.tail.replaceAll("/", "__")}"
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(dir => contextURI.getPath.startsWith(dir.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ assert(
+ !Files.exists(flatPath),
+ s"File $flatPath already exists and would be overwritten, aborting!"
+ )
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.flatMap { ref =>
+ val optResolved = if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ Some(refPath.toUri.toURL)
+ } else {
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ Option(classLoader.findResource(ref))
+ }
+ }
+ case "jar" => {
+ // Maybe this path in the JAR is actually absolute, just
+ // missing the leading '/', try resolving it relative to the
+ // root of the JAR
+ val jarPath = contextPath.getRoot().resolve(ref)
+ if (Files.exists(jarPath))
+ Some(jarPath.toUri.toURL)
+ else
+ None
Review Comment:
The deprecated fallback behavior is the same for file and jar schems, in
both cases they should just look at the classpath as if the ref was intended to
be absolute but was missing the leading slash. To avoid duplication, you might
do something like this?
```scala
val optResolvedRelative = contextURI.getScheme match {
case "file" => {
// need to check other resourceDirectories since they would end up in
the same same jar
val resolvedRoot = ...
resolvedRoot.map(_.toURI.resolve(ref).toULR)
}
case "jar" => {
// nothing else to check, resolveSibling should have found it in the
same jar
None
}
}
optResolvedRelative.orElse {
// deprecated fallback, treat ref as if it were absolute just with the
slash missing
Option(classLoader.findResource(ref))
}
```
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,252 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r
+ ),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter =
+ (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectURLs = (Compile / resourceDirectories).value.map { root =>
+ (root ** filter).get.map(_.toURI.toURL).toList
+ }.flatten
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+ val jarRegex = "(.*/(.*).jar)!(.*)".r
+
+ val seen = scala.collection.mutable.Set[URI]()
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val unresolved = scala.collection.mutable.ArrayBuffer[(URI, String)]()
+ val jarFileSystems = scala.collection.mutable.Map[String, FileSystem]()
+ unprocessed.pushAll(projectURLs.map(_.toURI))
+ seen ++= projectURLs.map(_.toURI)
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+
+ // Get a Java Path for the current file (whether it is in a JAR or
+ // just a regular file) and a Path for where the current file will be
+ // moved to in the flattened directory structure. Note that using a
+ // Java Path on a regular file will use the default FileSystem while
for
+ // JARs a FileSystem is created for interacting with the files inside
+ // the JAR. This allows all Path functions like resolveSibling/exists
to
+ // work with both regular files or files contained inside a JAR.
+ val (contextPath, flatPath) = contextURI.getScheme match {
+ case "jar" =>
+ contextURI.toString match {
+ case jarRegex(jarPath, jarName, path) => {
+ val fs = jarFileSystems.getOrElseUpdate(
+ Paths.get(jarPath).toString,
+ FileSystems.newFileSystem(contextURI, new
java.util.HashMap[String, Any]())
+ )
+ val cPath = fs.getPath(path)
+ (
+ cPath,
+ Paths.get(
+ flatDir.toString,
+ s"${jarName}__${cPath.toString.tail.replaceAll("/", "__")}"
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unable to parse JAR URI:
$contextURI")
+ }
+ case "file" => {
+ val path = Paths.get(contextURI)
+ val root = (Compile / resourceDirectories).value
+ .find(dir => contextURI.getPath.startsWith(dir.toString))
+ .get
+ .toURI
+ (
+ path,
+ Paths.get(
+ flatDir.toString,
+ Paths.get(root).relativize(path).toString.replaceAll("/", "__")
+ )
+ )
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unrecognized URI scheme:
$contextURI")
+ }
+
+ assert(
+ !Files.exists(flatPath),
+ s"File $flatPath already exists and would be overwritten, aborting!"
+ )
+ val bw = Files.newBufferedWriter(flatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.flatMap { ref =>
+ val optResolved = if (ref.startsWith("/")) {
+ // Dealing with an absolute path
+ Option(classLoader.findResource(ref.tail))
+ } else {
+ // Relative path
+ val refPath = contextPath.resolveSibling(ref).normalize()
+ if (Files.exists(refPath)) {
+ // Referenced path exists in either the same root resource
+ // diretory or jar file as the context schema
+ Some(refPath.toUri.toURL)
+ } else {
+ contextURI.getScheme match {
+ case "file" => {
+ // Need to check other resource directories
+ val resolvedRoot = (Compile /
resourceDirectories).value.find(root =>
+ Files.exists(Paths.get(root.toURI.resolve(ref)))
+ )
+ if (resolvedRoot.isDefined)
+ Some(resolvedRoot.get.toURI.resolve(ref).toURL)
+ else {
+ // Maybe this path is actually absolute, just missing the
+ // leading '/', check the classpath
+ Option(classLoader.findResource(ref))
+ }
+ }
+ case "jar" => {
+ // Maybe this path in the JAR is actually absolute, just
+ // missing the leading '/', try resolving it relative to the
+ // root of the JAR
+ val jarPath = contextPath.getRoot().resolve(ref)
+ if (Files.exists(jarPath))
+ Some(jarPath.toUri.toURL)
+ else
+ None
+ }
+ case _ => None
+ }
+ }
+ }
+ if (optResolved.isEmpty)
+ unresolved.append((contextURI, ref))
+ optResolved.map(resolved => ref -> resolved)
+ }.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, optResolvedURL)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val resolvedURI = optResolvedURL.toURI
+ val relativized = resolvedURI.getScheme match {
+ case "jar" =>
+ resolvedURI.toString match {
+ case jarRegex(_, jarName, path) =>
+ s"${jarName}__${path.tail.replaceAll("/", "__")}"
Review Comment:
You do the replaceAll("/", "__") below. Maybe this wants to be
```
s"${jarName}/${path.tail}"
```
So kindof the implication is the `relatived` variable always looks like a
path (and jars just prepend another directory), and later logic actually
flattens that path to replace slashes with underscores? My thinking is if we
ever want to do something instead of double underscores it'll only be in one
place below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
