stevedlawrence commented on code in PR #183:
URL: https://github.com/apache/daffodil-sbt/pull/183#discussion_r3234778482
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
Review Comment:
Indengation is incorrect. Can you re-run scalafmt?
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
Review Comment:
Should we avoid matching xsi:schemaLocation since those have a different
syntax? So it's something like:
```scala
"""(?<!xsi:)schemaLocation=\"([^\"]*)\"""".r
```
Then we don't need this string split stuff below:
```
val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
```
I don't know if there's anything where we actually use xsi:schemaLocation.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
Review Comment:
This doesn't have the same logic as Daffodil. If contextURI is a
`jar:file:/..` and it contains a relative path, then this will look for the
relative path relative to `rootPath`. But instead it should look inside the
same jar as contextURI.
For example, if the contextURI is
```
jar:file:/path/to/foo.jar!/path/to/schema.dfdl.xsd
```
and that schema.dfdl.xsd references `schemaLocation="foo.dfdl.xsd"`, we
should be looking for the file at:
```
jar:file:/path/to/foo.jar!/path/to/foo.dfdl.xsd
```
But this logic will check in
```
resourceDirectoryRoot/path/to/foo.dfdl.xsd
```
If the context is a jar, we shouldn't even consider looking at the
filesystem.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
Review Comment:
If this is a jar we should prefix the jar name to the path. It doesn't hurt
anything and avoids potential issues where two jars have the same name.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
+ // Found the file on the regular filesystem
+ Some(origLocation -> rootURI.resolve(relPath.toString))
+ } else {
+ // Check the classpath for the current reference relative to
the
+ // current contextURI
+ val url = Option(classLoader.findResource(relPath.toString))
Review Comment:
Same here, this isn't quite the same logic that Daffodil uses. I think this
is handling the jar case, but it's not looking at the same jar that contextURI
is in, it's just looking at the classpath and so could find something in a
different jar.
##########
README.md:
##########
@@ -323,6 +323,36 @@ root `src/` directory, and all test source and resource
files to be in a root
`test/` directory. Source files are those that end with `*.scala` or `*.java`,
and resource files are anything else.
+### Flatten Schemas
+
+This plugin has functionality to flatten the directory structure of 1 or more
+schema projects, renaming the schema files and upating schemaLocation's as
+necessary.
+
+```bash
+sbt daffodilFlattenSchemas
+```
+
+The renaming works as follows:
+
+`org/apache/daffodil/xsd/main.dfdl.xsd`
+
+will be renamed to:
+
+`org__apache__daffodil__xsd__main.dfdl.xsd`
+
+Note: Original files are not modified, they are simply copied to the specified
+output directory with the new name and updated schemaLocation's.
+
+Many non-Daffodil XML/XSD programs (such as XML validators) do not resolve
Review Comment:
Nice explanation for why this is needed, I might recommend moving it to the
top of this section so it acts as an introduction as to why this task exists
and why things are renamed the way they are. Otherwise the above logic feels
kindof arbitrary until you get to this paragraph.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
Review Comment:
This is an issue with the using root mentioned above, it's possible that the
contetxURI is a different root that rootURI because the are multple possible
roots for a URI--it isn't necessarily the same one as this root we're looking
at.
So I think this wants to be something like this:
```scala
case "file" => {
val root = (Compile /
resourceDirectories).value.find(contextURI.startsWith).get
Paths.get(root.relativize(contextURI))
}
```
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
Review Comment:
In this review, I've thought of a new issue if the contextURI is a file://
and the origLocation is a relative path. In that case a relative resource path
could access a file in a different resourceDirectory since they are would be in
the same jar. So I think the logic for looking up relative paths when the
context is a file:// wants to be something like this:
```scala
val resolvedRoot = (Compile / resourceDirectories).value.find(root =>
File.exists(root.resolve(relPath)))
if (resolvedRoot.isDefined) {
Some(origLocation -> resolvedRoot.resolve(relPath))
}
```
So similar to what you have, but we have try to resolve the path relative to
all possible roots instead of the current root. Similar to above, that is
because a path relative to a file:// could resolve to any resource root
directory, not just the current one.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
+ // Found the file on the regular filesystem
+ Some(origLocation -> rootURI.resolve(relPath.toString))
+ } else {
+ // Check the classpath for the current reference relative to
the
+ // current contextURI
+ val url = Option(classLoader.findResource(relPath.toString))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ // Maybe this is actually an absolute path not within the
same
+ // context. Check the classpath for the original location
+ val url2 = Option(classLoader.findResource(origLocation))
+ if (url2.isDefined)
+ Some(origLocation -> url2.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve local reference to $ref
from source file $contextURI")
+ None
+ }
+ }
+ }
+ }
+ }.flatten.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, resolvedRef)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val relativized = resolvedRef.getScheme match {
+ case "jar" => resolvedRef.toString.split("!")(1).tail
+ case _ => rootURI.relativize(resolvedRef).toString
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
Review Comment:
Same as above, we should prefix the jar name if the schema is a jar. It
might be worth adding a function that converts a URI to a flat path, since we
need it in two places 1) above when creating the flat file 2) here when
creating a reference to a flat file. Maybe it's define at the top of this
setting or something since it's really only needed by this setting.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
Review Comment:
Our coding convention doesn't usually drop periods, e.g. this wants to be
`projectResources.foreach { ...
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
Review Comment:
I think including the root in this doesn't really work in the following
logic. Just because a specific resource file is resolved from some root
directory it doesn't that the thigns that file references use that same root.
They could be in a different root or a jar, so carrying around the root I don't
think really works.
Ithink this just wants to be
```scala
val projectResources = (Compile / resourceDirectories).value ** filter
```
And the below logic needs to figure out which root directory files are in
and strip it off.
Note that this would mean we no longer have `projectResource.foreach { ...
}`, I think you just push all the project resources to the unprocessed stack
and then process them as normal. Later code in other comments then has to
figure out which root a file:// resource actually came frome.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
+ // Found the file on the regular filesystem
+ Some(origLocation -> rootURI.resolve(relPath.toString))
+ } else {
+ // Check the classpath for the current reference relative to
the
+ // current contextURI
+ val url = Option(classLoader.findResource(relPath.toString))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ // Maybe this is actually an absolute path not within the
same
+ // context. Check the classpath for the original location
+ val url2 = Option(classLoader.findResource(origLocation))
+ if (url2.isDefined)
+ Some(origLocation -> url2.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve local reference to $ref
from source file $contextURI")
+ None
+ }
+ }
+ }
+ }
+ }.flatten.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, resolvedRef)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val relativized = resolvedRef.getScheme match {
+ case "jar" => resolvedRef.toString.split("!")(1).tail
+ case _ => rootURI.relativize(resolvedRef).toString
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+ processed += contextURI
+ unprocessed.pushAll((resolvedRefs.values.toSet diff
processed).filterNot(unprocessed contains _))
Review Comment:
I feel like it's a bit hard to make sense of the diff + filterNot. I think
this logic is could be simplified if instead of keeping track of processed vs
unprocessed, we keep track of processed vs "seen", where "seen" contains URIs
that have either been processed or are in the unprocessed stack. So it kindof
has this logic you have here already built-in to it. Note that this "seen"
logic is similar to the logic Daffodil uses when it resolves includes/imports
so it's kindof consistent.
So this logic becomes something like:
```scala
// add all resolved references to the seen Set. Any references we haven't
already seen
// must never have been referenced before and so we add them to the stack to
be processed
val unseen = resolvedReferences.filter(seen.add)
unprocessed.pushAll(unseen)
```
The only additional change needed with this approach is when we initialize
the unprocessed stack/queue we must also add those to the seen set, because
those are in the unprocessed queue and so will eventually be processed even if
something else references them.
This way when we see a reference, we don't ever have to check if it wasn't
already processed AND isn't already in the unprocessed stack. We just mark it
as seen, and if it's the first time seeing it we add it the to list to be
processed.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +858,198 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+ /* Paths in daffodilFlattenExcludes/Includes that are not globbed at the
start are
+ * generally only going to match paths within JAR files on the classpath.
+ * In order to deal with paths on the filesystem we need to glob the start
+ * of the path to account for different directory structure before the
+ * schema project path.
+ */
+ daffodilFlattenIncludes := "*.xsd" | "*.xsl" | "*.xslt" | "*.xml",
+ daffodilFlattenExcludes := HiddenFileFilter,
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+ daffodilFlattenSchemas := {
+
+ val logger = streams.value.log
+ val filter = daffodilFlattenIncludes.value --
daffodilFlattenExcludes.value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /* Get all dependency jars and resources specific to this project. Note
+ * that the "Test" configuration is used for JAR files in order to ensure
+ * we pull in XSD files from daffodil-lib, as in many schema projects the
+ * daffodil dependencies are only used for testing, not compiling. Also
+ * note that we want to use Test/externalDependencyClasspath to get
+ * dependency jars, and not something like Test/fullClasspath or
+ * Test/dependencyClasspath, since those could trigger expensive resource
+ * generators or compilation of internal test jars that we don't need--we
+ * only need Compile/resources from this project and Test/dependency jars
+ * from external projects
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Compile /
fullClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val schemaLocationPattern = """schemaLocation=\"([^\"]*)\"""".r
+ val hrefPattern = "href=\"([^\"]*)\"".r
+ val documentPattern = "document[(]'([^']*)'[)]".r
+ val referenceRegexes = List(schemaLocationPattern, hrefPattern,
documentPattern)
+
+ val referencedFiles = {
+ // This annotation simply warns if the compiler cannot enable tail
call optimization
+ @scala.annotation.tailrec
+ def getReferences(root: URL, parents: Seq[URL], acc:
ArrayBuffer[URL]): Seq[URL] = {
+ val rootPath = Paths.get(root.toURI)
+ parents match {
+ // Already processed this file (h), proceed with rest of files (t)
+ case h :: t if (acc.contains(h)) =>
+ getReferences(
+ root,
+ t,
+ acc
+ )
+ // Need to process this file (h)
+ case h :: t => {
+ val relPath = h.toString match {
+ case j if (j.startsWith("jar:")) =>
Paths.get(j.toString.split("!")(1).tail)
+ case f if (f.startsWith("file:")) =>
rootPath.relativize(Paths.get(h.toURI))
+ }
+ val bytes = h.openStream().readAllBytes()
+ val newPath = Paths.get(
+ flatDir.toString,
+ relPath.toString.replaceAll("/", "__")
+ )
+ val bw = Files.newBufferedWriter(newPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+ val references = {
+ referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+ }
+
+ // Resolve the location of all references on the actual file
+ // system
+ val resolvedReferences = references
+ .map { ref =>
+ val origLocation = {
+ if (ref.contains("urn:")) {
+ val urn = ref.split(" ")(1)
+ if (urn.startsWith("/"))
+ urn.tail
+ else
+ urn
+ } else if (ref.startsWith("/"))
+ ref.tail
+ else
+ ref
+ }
+ val res = {
+ val r = classLoader.findResource(origLocation)
+ if (r == null) {
+ val rrel = List(relPath.getParent,
origLocation).mkString("/")
+ classLoader.findResource(rrel)
+ } else
+ r
+ }
+ if (res == null)
+ logger.warn(s"Unable to resolve reference to $ref from
source file $h")
+ ref -> Option(res)
+ }
+ .toMap
+ .filter(e => e._2.isDefined)
+
+ // For each reference do a search and replace of the entire file
+ val updatedFileAsString = {
+ resolvedReferences.foldLeft(fileAsString) {
+ case (input, (ref, resolvedRef)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to
"__"
+ val rref = resolvedRef.get.toString
+ val relativized = rref match {
+ case j if (j.startsWith("jar")) => j.split("!")(1).tail
+ case f if (f.startsWith("file")) =>
rootPath.relativize(Paths.get(resolvedRef.get.toURI)).toString
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+
+ /* Have succesfully processed this file (h), call getReferences
+ * again on the rest of the list (t) + any references from this
+ * file. Add this file to the accumulator list
+ */
+ getReferences(
+ root,
Review Comment:
I believe some projects do exist that make use of resource generators. We
also don't know what other project might exists. They probably aren't common,
but they could exists. And in those cases we will have separate resource
directories and those files could reference each other. I feel it's important
to make sure this works correctly and I don't think adds too much complexity.
My latest review adds some comments about how we might be able to do this.
##########
src/main/scala/org/apache/daffodil/DaffodilPlugin.scala:
##########
@@ -840,6 +854,179 @@ object DaffodilPlugin extends AutoPlugin {
}
}
+ def daffodilFlattenSettings: Seq[Setting[_]] = Seq(
+ daffodilFlattenTarget := target.value /
s"${name.value}-${version.value}-flat.zip",
+
+ /**
+ * Whether or not to publish the flattened schemas zip. Defaults to false.
+ *
+ * If projects want to publish flattened schemas then they must explicitly
enable it by
+ * setting 'daffodilFlattenSchemas / publishArtifact := true'.
+ *
+ * If false, flattened schemas will not be created unless you explicitly
run the
+ * daffodilFlattenSchemas.
+ */
+ daffodilFlattenSchemas / publishArtifact := false,
+
+ daffodilFlattenSchemas / artifact := Artifact(
+ name.value,
+ "flat",
+ "zip",
+ Some("flat"),
+ Vector(),
+ None
+ ),
+
+ /**
+ * Only grab the file types that need to be flattened. These files are only
+ * grabbed from the root project. Any referenced files will be pulled in
+ * from the classpath. Note that XML files are commonly used in XSLT for
+ * settings.
+ */
+ daffodilFlattenSchemas / includeFilter := "*.xsd" | "*.xsl" | "*.xslt" |
"*.xml",
+ daffodilFlattenSchemas / excludeFilter := HiddenFileFilter,
+
+ daffodilFlattenResourceReferencePatterns := List(
+ """schemaLocation=\"([^\"]*)\"""".r,
+ "href=\"([^\"]*)\"".r,
+ "document[(]'([^']*)'[)]".r),
+
+ daffodilFlattenSchemas / products := {
+
+ val logger = streams.value.log
+ val filter = (daffodilFlattenSchemas / includeFilter).value --
(daffodilFlattenSchemas / excludeFilter).value
+
+ val flatDir = target.value / "flatDir"
+ if (flatDir.exists())
+ IO.delete(flatDir)
+ IO.createDirectory(flatDir)
+
+ val projectResources = (Compile / resourceDirectories).value
+ .map { root => root.toURI.toURL -> (root **
filter).get.map(_.toURI.toURL).toList }
+ .toMap
+
+ /**
+ * Create a URLClassLoader object with URLs to all resources used by the
+ * project. The class loader will be used to resolve references made
+ * within the flattened files.
+ */
+ val projectURLs = (Compile /
resourceDirectories).value.map(_.toURI.toURL)
+ val allClasspathURLs = (Test /
externalDependencyClasspath).value.map(_.data.toURI.toURL)
+ val classLoader = new URLClassLoader((projectURLs ++
allClasspathURLs).toArray, null)
+
+ val referenceRegexes = daffodilFlattenResourceReferencePatterns.value
+
+ val processed = scala.collection.mutable.Set[URI]()
+ projectResources foreach { case (rootURL, urls) =>
+ val unprocessed = scala.collection.mutable.Stack[URI]()
+ val rootURI = rootURL.toURI
+ val rootPath = Paths.get(rootURI)
+ unprocessed.pushAll(urls.map(_.toURI))
+ while (!unprocessed.isEmpty) {
+ val contextURI = unprocessed.pop
+ val bytes = contextURI.toURL.openStream().readAllBytes()
+ val contextRelPath = contextURI.getScheme match {
+ case "jar" => Paths.get(contextURI.toString.split("!")(1).tail)
+ case "file" => Paths.get(rootURI.relativize(contextURI).getPath)
+ case _ => throw new IllegalArgumentException(s"Unrecognized URI
scheme: $contextURI")
+ }
+ val contextFlatPath = Paths.get(
+ flatDir.toString,
+ contextRelPath.toString.replaceAll("/", "__"))
+ val bw = Files.newBufferedWriter(contextFlatPath)
+ val fileAsString = new String(bytes, Charset.defaultCharset())
+
+ val references = referenceRegexes.flatMap { re =>
+ re.findAllIn(fileAsString).matchData.map(_.group(1))
+ }
+
+ val resolvedRefs = references.map { ref =>
+ val origLocation = if (ref contains " ") ref.split(" ")(1) else ref
+ if (origLocation.startsWith("/")) {
+ // Dealing with an absolute path
+ val url = Option(classLoader.findResource(origLocation.tail))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve absolute reference to $ref
from source file $contextURI")
+ None
+ }
+ } else {
+ // Relative path
+ val relPath =
contextRelPath.resolveSibling(origLocation).normalize()
+ if (Files.exists(rootPath.resolve(relPath))) {
+ // Found the file on the regular filesystem
+ Some(origLocation -> rootURI.resolve(relPath.toString))
+ } else {
+ // Check the classpath for the current reference relative to
the
+ // current contextURI
+ val url = Option(classLoader.findResource(relPath.toString))
+ if (url.isDefined)
+ Some(origLocation -> url.get.toURI)
+ else {
+ // Maybe this is actually an absolute path not within the
same
+ // context. Check the classpath for the original location
+ val url2 = Option(classLoader.findResource(origLocation))
+ if (url2.isDefined)
+ Some(origLocation -> url2.get.toURI)
+ else {
+ logger.warn(s"Unable to resolve local reference to $ref
from source file $contextURI")
+ None
+ }
+ }
+ }
+ }
+ }.flatten.toMap
+
+ val updatedFileAsString = {
+ resolvedRefs.foldLeft(fileAsString) {
+ case (input, (ref, resolvedRef)) => {
+ // For each reference replace each instance of it in the
+ // file with the same reference but with "/" changed to "__"
+ val relativized = resolvedRef.getScheme match {
+ case "jar" => resolvedRef.toString.split("!")(1).tail
+ case _ => rootURI.relativize(resolvedRef).toString
+ }
+ input.replaceAll(ref, relativized.replaceAll("/", "__"))
+ }
+ }
+ }
+
+ bw.write(updatedFileAsString)
+ bw.close()
+ processed += contextURI
+ unprocessed.pushAll((resolvedRefs.values.toSet diff
processed).filterNot(unprocessed contains _))
+ }
+ }
+
+ /* Create zip file containing all flattened schemas */
+ val flattenedFiles = IO.listFiles(flatDir)
Review Comment:
Another though about reproducibility, can we sort the flattned files by name
so that the order is consistent. Just another way to possibly improve
reproducibilty.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
