This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 32dd924 SOLR-15867 Bring in build stuff from lucene (#491)
32dd924 is described below
commit 32dd924b14ee7160cb3a47b75bbe9f51ee36a483
Author: Jan Høydahl <[email protected]>
AuthorDate: Sat Jan 8 00:52:23 2022 +0100
SOLR-15867 Bring in build stuff from lucene (#491)
janhoy:
* Bring in improved build parts from lucene 9
* Add maven and docker folders to release dir
* Add instructions for building docker file to workflow.txt
* Package official dockerfile into release
houston:
* Add support for useGpg, slim down Docker release.
* Add ability to build a release with a non-signed Dockerfile
* tar -> tgz. Fixed official docker build. a bit messy still
Co-authored-by: Houston Putman <[email protected]>
---
build.gradle | 1 -
buildSrc/build.gradle | 15 +-
.../{build.gradle => scriptDepVersions.gradle} | 24 +--
.../java/org/apache/lucene/gradle/Checksum.java | 195 +++++++++++++++++++++
gradle/documentation/changes-to-html.gradle | 24 ++-
gradle/help.gradle | 2 +-
gradle/releasing.gradle | 55 ------
help/dependencies.txt | 4 +-
help/gpgSigning.txt | 70 --------
help/publishing.txt | 138 +++++++++++++++
help/workflow.txt | 15 +-
settings.gradle | 1 +
solr/CHANGES.txt | 2 +
solr/distribution/build.gradle | 160 +++++++++++++++++
solr/distribution/source-release.gradle | 50 ++++++
solr/docker/build.gradle | 24 ++-
solr/docker/gradle-help.txt | 2 +-
solr/docker/templates/Dockerfile.body.template | 2 +-
solr/packaging/build.gradle | 61 +------
19 files changed, 632 insertions(+), 213 deletions(-)
diff --git a/build.gradle b/build.gradle
index f4d3805..490c2f9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -127,7 +127,6 @@ apply from: file('gradle/java/jar-manifest.gradle')
// Publishing and releasing
apply from: file('gradle/maven/defaults-maven.gradle')
-apply from: file('gradle/releasing.gradle')
// IDE support, settings and specials.
apply from: file('gradle/ide/intellij-idea.gradle')
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index b1aee14..91f8874 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -15,15 +15,24 @@
* limitations under the License.
*/
+repositories {
+ mavenCentral()
+}
+
ext {
// Minimum Java version required to compile buildSrc.
minJavaVersion = JavaVersion.VERSION_11
}
+// Make sure the build environment is consistent.
+apply from: file('../gradle/validation/check-environment.gradle')
+
+// Load common buildSrc and script deps.
+apply from: file("scriptDepVersions.gradle")
+
dependencies {
implementation gradleApi()
implementation localGroovy()
-}
-// Make sure the build environment is consistent.
-apply from: file('../gradle/validation/check-environment.gradle')
+ implementation
"commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"
+}
diff --git a/buildSrc/build.gradle b/buildSrc/scriptDepVersions.gradle
similarity index 64%
copy from buildSrc/build.gradle
copy to buildSrc/scriptDepVersions.gradle
index b1aee14..5c27c51 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/scriptDepVersions.gradle
@@ -15,15 +15,19 @@
* limitations under the License.
*/
-ext {
- // Minimum Java version required to compile buildSrc.
- minJavaVersion = JavaVersion.VERSION_11
-}
+// Declare script dependency versions outside of palantir's
+// version unification control. These are not our main dependencies
+// but are reused in buildSrc and across applied scripts.
-dependencies {
- implementation gradleApi()
- implementation localGroovy()
+ext {
+ // TODO: Check if Solr needs all of these
+ scriptDepVersions = [
+ "apache-rat": "0.11",
+ "commons-codec": "1.13",
+ "ecj": "3.27.0",
+ "flexmark": "0.61.24",
+ "javacc": "7.0.4",
+ "jflex": "1.8.2",
+ "jgit": "5.9.0.202009080501-r",
+ ]
}
-
-// Make sure the build environment is consistent.
-apply from: file('../gradle/validation/check-environment.gradle')
diff --git a/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java
b/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java
new file mode 100644
index 0000000..0dab9dc
--- /dev/null
+++ b/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * A task that generates SHA512 checksums. Cloned from:
+ *
https://github.com/gradle/gradle-checksum/blob/03351de/src/main/java/org/gradle/crypto/checksum/Checksum.java
+ * with custom fixes to make the checksum palatable to shasum, see pending PR:
+ * https://github.com/gradle/gradle-checksum/pull/4
+ *
+ * Original license ASL:
+ * https://github.com/gradle/gradle-checksum/blob/03351de/LICENSE
+ */
+
+package org.apache.lucene.gradle;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.FileType;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.work.Incremental;
+import org.gradle.work.InputChanges;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Locale;
+
+public class Checksum extends DefaultTask {
+ private FileCollection files;
+ private File outputDir;
+ private Algorithm algorithm;
+
+ public enum Algorithm {
+ MD5(new DigestUtils(DigestUtils.getMd5Digest())),
+ SHA256(new DigestUtils(DigestUtils.getSha256Digest())),
+ SHA384(new DigestUtils(DigestUtils.getSha384Digest())),
+ SHA512(new DigestUtils(DigestUtils.getSha512Digest()));
+
+ private final DigestUtils hashFunction;
+
+ Algorithm(DigestUtils hashFunction) {
+ this.hashFunction = hashFunction;
+ }
+
+ public String getExtension() {
+ return name().toLowerCase(Locale.ROOT);
+ }
+ }
+
+ public Checksum() {
+ outputDir = new File(getProject().getBuildDir(), "checksums");
+ algorithm = Algorithm.SHA256;
+ }
+
+ @InputFiles
+ @Incremental
+ public FileCollection getFiles() {
+ return files;
+ }
+
+ public void setFiles(FileCollection files) {
+ this.files = files;
+ }
+
+ @Input
+ public Algorithm getAlgorithm() {
+ return algorithm;
+ }
+
+ public void setAlgorithm(Algorithm algorithm) {
+ this.algorithm = algorithm;
+ }
+
+ @OutputDirectory
+ public File getOutputDir() {
+ return outputDir;
+ }
+
+ public void setOutputDir(File outputDir) {
+ if (outputDir.exists() && !outputDir.isDirectory()) {
+ throw new IllegalArgumentException("Output directory must be a
directory.");
+ }
+ this.outputDir = outputDir;
+ }
+
+ @TaskAction
+ public void generateChecksumFiles(InputChanges inputChanges) throws
IOException {
+ if (!getOutputDir().exists()) {
+ if (!getOutputDir().mkdirs()) {
+ throw new IOException("Could not create directory:" + getOutputDir());
+ }
+ }
+
+ if (!inputChanges.isIncremental()) {
+ getProject().delete(allPossibleChecksumFiles());
+ }
+
+ inputChanges
+ .getFileChanges(getFiles())
+ .forEach(
+ fileChange -> {
+ if (fileChange.getFileType() == FileType.DIRECTORY) {
+ getProject()
+ .getLogger()
+ .warn("Checksums can't be applied to directories: " +
fileChange.getFile());
+ return;
+ }
+
+ File input = fileChange.getFile();
+ switch (fileChange.getChangeType()) {
+ case REMOVED:
+ if (input.isFile()) {
+ getProject().delete(outputFileFor(input));
+ }
+ break;
+
+ case ADDED:
+ case MODIFIED:
+ input = fileChange.getFile();
+ if (input.isFile()) {
+ File checksumFile = outputFileFor(input);
+
+ try {
+ String checksum =
algorithm.hashFunction.digestAsHex(input).trim();
+
+ /*
+ * https://man7.org/linux/man-pages/man1/sha1sum.1.html
+ *
+ * The default mode is to print a line with checksum, a
space, a character
+ * indicating input mode ('*' for binary, ' ' for text
or where
+ * binary is insignificant), and name for each FILE.
+ */
+ boolean binaryMode = true;
+
+ Files.writeString(
+ checksumFile.toPath(),
+ String.format(
+ Locale.ROOT,
+ "%s %s%s",
+ checksum,
+ binaryMode ? "*" : " ",
+ input.getName()),
+ StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new GradleException("Trouble creating checksum: "
+ e.getMessage(), e);
+ }
+ }
+ break;
+ default:
+ throw new RuntimeException();
+ }
+ });
+ }
+
+ private File outputFileFor(File inputFile) {
+ return new File(getOutputDir(), inputFile.getName() + "." +
algorithm.getExtension());
+ }
+
+ private FileCollection allPossibleChecksumFiles() {
+ FileCollection possibleFiles = null;
+ for (Algorithm algo : Algorithm.values()) {
+ if (possibleFiles == null) {
+ possibleFiles = filesFor(algo);
+ } else {
+ possibleFiles = possibleFiles.plus(filesFor(algo));
+ }
+ }
+ return possibleFiles;
+ }
+
+ private FileCollection filesFor(final Algorithm algo) {
+ return getProject()
+ .fileTree(getOutputDir(), files -> files.include("**/*." +
algo.toString().toLowerCase()));
+ }
+}
diff --git a/gradle/documentation/changes-to-html.gradle
b/gradle/documentation/changes-to-html.gradle
index e9692ab..3b4ca69 100644
--- a/gradle/documentation/changes-to-html.gradle
+++ b/gradle/documentation/changes-to-html.gradle
@@ -22,6 +22,18 @@ configure(project(':solr:documentation')) {
siteDir = resources
script = file("${resources}/changes2html.pl")
}
+
+ // Make the rendered HTML of changes available as a separate
+ // artifact for the distribution.
+ configurations {
+ changesHtml
+ }
+
+ artifacts {
+ changesHtml changesToHtml.targetDir, {
+ builtBy changesToHtml
+ }
+ }
}
// compile changes.txt into an html file
@@ -44,7 +56,7 @@ class ChangesToHtmlTask extends DefaultTask {
@OutputDirectory
final DirectoryProperty targetDir = project.objects.directoryProperty()
- .fileProvider(project.providers.provider {
project.file("${project.docroot}/changes") })
+ .fileProvider(project.providers.provider {
project.file("${project.docroot}/changes") })
@Input
def luceneDocUrl = "${-> project.luceneDocUrl }"
@@ -72,11 +84,11 @@ class ChangesToHtmlTask extends DefaultTask {
ignoreExitValue = true
args += [
- "-CSD",
- script,
- "${productName}",
- versionsFile.toString(),
- luceneDocUrl.concat('/') // slash required at end by perl script
+ "-CSD",
+ script,
+ "${productName}",
+ versionsFile.toString(),
+ luceneDocUrl.concat('/') // slash required at end by perl script
]
}
diff --git a/gradle/help.gradle b/gradle/help.gradle
index 7bbdaf5..3a354ed 100644
--- a/gradle/help.gradle
+++ b/gradle/help.gradle
@@ -30,7 +30,7 @@ configure(rootProject) {
["Git", "help/git.txt", "Git assistance and guides."],
["ValidateLogCalls", "help/validateLogCalls.txt", "How to use logging
calls efficiently."],
["IDEs", "help/IDEs.txt", "IDE support."],
- ["GpgSigning", "help/gpgSigning.txt", "Signing artifacts with GPG."],
+ ["Publishing", "help/publishing.txt", "Release publishing publishing,
signing, etc."],
["Docker", "solr/docker/gradle-help.txt", "Building Solr Docker
images."],
]
diff --git a/gradle/releasing.gradle b/gradle/releasing.gradle
deleted file mode 100644
index b05b826..0000000
--- a/gradle/releasing.gradle
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.commons.codec.digest.DigestUtils
-
-// We're using commons-codec for computing checksums.
-buildscript {
- repositories {
- mavenCentral()
- }
-
- dependencies {
- classpath
"commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"
- }
-}
-
-allprojects {
- plugins.withType(DistributionPlugin) {
- def checksum = {
- outputs.files.each { File file ->
- String sha512 = new
DigestUtils(DigestUtils.sha512Digest).digestAsHex(file).trim()
- new File(file.parent, file.name + ".sha512").write(sha512 + "
" + file.name, "UTF-8")
- }
- }
-
- distZip {
- doLast checksum
- }
-
- distTar {
- compression = Compression.GZIP
- doLast checksum
- }
-
- installDist {
- doLast {
- logger.lifecycle "Distribution assembled under:
${destinationDir}"
- }
- }
- }
-}
diff --git a/help/dependencies.txt b/help/dependencies.txt
index 2990db6..8cd78b7 100644
--- a/help/dependencies.txt
+++ b/help/dependencies.txt
@@ -13,8 +13,8 @@
https://docs.gradle.org/current/userguide/dependency_management_for_java_project
https://docs.gradle.org/current/userguide/java_library_plugin.html#sec:java_library_separation
https://docs.gradle.org/current/userguide/java_plugin.html#sec:java_plugin_and_dependency_management
-For the needs of Solr we will typically focus on three
-configurations and attach project dependencies to them:
+Solr typically uses three configurations and attach project
+dependencies to them:
api - makes a dependency available for main classes, tests and any
other modules importing the project (exportable dependency),
diff --git a/help/gpgSigning.txt b/help/gpgSigning.txt
deleted file mode 100644
index 80e45f0..0000000
--- a/help/gpgSigning.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-GPG Signing
-===========
-
-GPG Signing of distribution files (typically by a release manager) is done
with the 'signDist' command.
-
-The only required configuration property Gradle needs is the
'signing.gnupg.keyName' (aka: the fingerprint) of
-the key you wish to use:
-
-./gradlew signDist -Psigning.gnupg.keyName=YOUR_KEY_FINGERPRINT
-
-By default when you run this command, Gradle will delegate to the `gpg2`
command for managing the signing of each file, which (should)
-in turn use the `gpg-agent` to prompt you for your secret key only as needed
using a dialog box specific to your operating system and/or
-`gpg-agent` preferences.
-
-You may wish to put the `signing.gnupg.keyName` in your
`~/.gradle/gradle.properties` so it is set automatically any time you use gradle
-
-
-Additional Configuration
-------------------------
-
-The following additional properties -- specified either on the command line
via `-P...` or in your `~/.gradle/gradle.properties` may be
-useful/necessary in your system:
-
-signing.gnupg.useLegacyGpg=true # Changes the default
executable from `gpg2` to `gpg` and explicitly sets `--use-agent`
-signing.gnupg.executable=gpg # Allows explicit control
over what command executable used (ex: `gpg2`, `gpg`, `gpg.exe`, etc...)
-signing.gnupg.homeDir=/tmp/gnupg-home # overrides GnuPG's default
home directory (ex: `~/.gnupg/`)
-signing.gnupg.optionsFile=/tmp/gnupg-home/my.conf # overrides GnuPG's default
configuration file
-signing.gnupg.passphrase=... # Provide your passphrase
to gradle to hand off to gpg. *NOT RECOMMENDED*, see below.
-
-
-Notes About Error Messages
---------------------------
-
-
-### `gpg: signing failed: Inappropriate ioctl for device`
-
-This typically happens if your `gpg-agent` is configured (either globally for
your operating system, or personally in your
-`~/.gnupg/gpg-agent.conf`) to use a `pinentry` command which depends on using
the same `tty` as the `gpg` command (ex: `pinentry-curses`,
-or `pinentry-tty`, etc...).
-
-`tty` based `pinentry` implementations do not work when Gradle's
`SigningPlugin` is attempting to invoke `gpg` -- among other problems:
-Gradle is multi-threaded, and we sign multiple artifacts by default; so even
if the `SigningPlugin` didn't automatically force `--no-tty` when
-running `gpg` you could easily run into problems where a second `pinentry`
process wanted to read from the same `tty` in the middle of you
-typing in your passphrase to the first process.
-
-Developers are encouraged to configure a *non* `tty` based `pinentry` (ex:
`pinentry-gnome`, `pinentry-x11`, `pinentry-qt`, `pinentry-mac`,
-`pinentry-wsl-ps1`, etc...) either globally in your operating system, or
personally in your `~/.gnupg/gpg-agent.conf`, or in a new
-`gpg-agent.conf` file a new GnuPG configuration directory (containing a copy
of your private keys) that you direct gradle to via
-`signing.gnupg.homeDir`
-
-If none of these options are viable for you, then as a last resort you may
wish to consider using the `signing.gnupg.passphrase=...` property.
-This will expose your secret passphrase to the Gradle process, which will then
pass it directly to each `gpg-agent` instance using
-`--pinentry-mode=loopback`.
-
-
-### `gpg: signing failed: No such file or directory`
-
-This may mean that there is a problem preventing `gpg` from communicating
correctly with the `gpg-agent` (and/or invoking your `pinentry`
-program) that is independent of gradle. Try running `pkill gpg-agent` and
then retrying your `./gradlew` command
-
-
-### `No value has been specified for property 'signatory.keyId'.`
-
-Do not bother ever attempting to set a command line (or gradle.properties)
property named `signatory.keyId`. This is evidently the
-name of an internal property that the gradle `SigningPlugin` expects the
`GnupgSignatory` plugin we use to provide -- which it does
-as long as you have specified a valid value for `signing.gnupg.keyName`
-
-If you see this error, it means you did not properly set
`signing.gnupg.keyName` _AND_ you invoked a task which is attempting to use
-the `SigningPlugin`, but does not depend on the custom
`failUnlessGpgKeyProperty` to report the error correctly. Please file a Jira
-noting what `./gradlew` command you attempted to run so we can fix it's
dependencies, and try again after setting `signing.gnupg.keyName`.
diff --git a/help/publishing.txt b/help/publishing.txt
new file mode 100644
index 0000000..5b8bc5d
--- /dev/null
+++ b/help/publishing.txt
@@ -0,0 +1,138 @@
+Distribution and artifact publishing
+====================================
+
+
+See all distribution-related tasks by running:
+gradlew tasks --group distribution
+
+
+Maven
+-----
+
+To publish Solr Maven artifacts to a local ~/.m2 repository, run:
+
+gradlew mavenToLocalRepo
+
+To generate a local build/maven-local folder, run:
+
+gradlew mavenToLocalFolder
+
+To publish Solr Maven artifacts to Apache repositories
+(CI or release manager's job, typically!), run:
+
+gradlew mavenToApacheSnapshots -PasfNexusUsername= -PasfNexusPassword=
+gradlew mavenToApacheReleases -PasfNexusUsername= -PasfNexusPassword=
[optional signing options]
+
+See artifact signing section below if you plan to use mavenToApacheReleases.
+
+It is a good idea to avoid passing passwords on command line. CI jobs have
+these properties saved in ~/.gradle/gradle.properties - this way they
+are read automatically.
+
+Apache Releases repository will not accept snapshots.
+
+
+Release (distribution) artifacts
+--------------------------------
+
+To collect all release artifacts, and optionally sign them, run:
+
+gradlew assembleRelease [optional signing options]
+
+All distribution artifacts will be placed under:
+
+solr/distribution/build/release
+
+Artifact signing is optional (but required if you're really making a release).
+
+
+Artifact signing
+----------------
+
+Certain tasks may optionally sign artifacts or require artifacts to be signed:
+
+ assembleRelease
+ mavenToApacheReleases
+
+Signing can be enabled by adding the "-Psign" option, for example:
+
+gradlew assembleRelease mavenToApacheReleases -Psign
+
+By default gradle uses a Java-based implementation of PGP for signing, which
requieres
+several "signing.*" properties via either ~/.gradle/gradle.properties or
command-line options:
+
+https://docs.gradle.org/current/userguide/signing_plugin.html#sec:signatory_credentials
+
+An example full command-line that assembles signed artifacts could look like
this:
+
+gradlew assembleRelease mavenToApacheReleases -Psign -Psigning.keyId=...
-Psigning.password=... -Psigning.secretKeyRingFile=...
+
+The keyId is the last 8 digits of your key (gpg -k will print your keys).
Gradle documentation has more options
+of secure passing of private key information and passwords.
+
+
+Artifact signing using an external GPG with GPG Agent
+-----------------------------------------------------
+
+You can use an external GPG command to deal with signing artifacts, with out
needing to give gradle your passphrase,
+by adding a "-PuseGpg" option, but this changes the properties you must
specify:
+
+For gpg2:
+gradlew [tasks] -Psign -PuseGpg -Psigning.gnupg.keyName=...
+
+For gpg:
+gradlew [tasks] -Psign -PuseGpg -Psigning.gnupg.keyName=...
-Psigning.gnupg.useLegacyGpg=true
+
+The keyName is the last 8 digits of your key (gpg -k will print your keys).
+
+There are additional (optional) "signing.gnupg.*" properties which exist that
may be useful/necessary in your system:
+
+signing.gnupg.useLegacyGpg=true # Changes the default
executable from `gpg2` to `gpg` and explicitly sets `--use-agent`
+signing.gnupg.executable=gpg # Allows explicit control
over what command executable used (ex: `gpg2`, `gpg`, `gpg.exe`, etc...)
+signing.gnupg.homeDir=/tmp/gnupg-home # overrides GnuPG's default
home directory (ex: `~/.gnupg/`)
+signing.gnupg.optionsFile=/tmp/gnupg-home/my.conf # overrides GnuPG's default
configuration file
+signing.gnupg.passphrase=... # Provide your passphrase
to gradle to hand off to gpg. *NOT RECOMMENDED*, see below.
+
+If in doubt, consult gradle's signing plugin documentation:
+https://docs.gradle.org/current/userguide/signing_plugin.html#sec:using_gpg_agent
+
+"signing.gnupg.passphrase" is not recommended because there is no advantage to
using an external GPG process if you use it. If you
+are comfortable giving gradle your passphrase, then there is no reason to use
an external GPG process via '-PuseGpg'. Just use the
+"signing.*" options described previously to let gradle deal with your key
directly.
+
+Because of how Gradle's signing plugin invokes GPG, using an external GPG
process *only* works if your GPG configuration uses a
+GPG agent (required by gpg2) and if the "pinentry" for your GPG agent does not
require access to the tty to prompt you for a password.
+
+If you the following command fails with your GPG configuration, you can not
use an external GPG process with gradle:
+
+echo foo | gpg --batch --no-tty --armor --detach-sign --use-agent --local-user
YOUR_KEY_NAME
+
+
+Notes About GPG Error Messages
+------------------------------
+
+### `gpg: signing failed: Inappropriate ioctl for device` or `Invalid IPC
response`
+
+This typically happens if your `gpg-agent` is configured (either globally for
your operating system, or personally in your
+`~/.gnupg/gpg-agent.conf`) to use a `pinentry` command which depends on using
the same `tty` as the `gpg` command (ex: `pinentry-curses`,
+or `pinentry-tty`, etc...).
+
+`tty` based `pinentry` implementations do not work when Gradle's signing
plugin is attempting to invoke `gpg` -- among other problems:
+Gradle is multi-threaded and we sign multiple artifacts by default. Even if
you use "--max-workers 1" to force single-threaded execution,
+the signing plugin invokes gpg with `--batch --no-tty`, making it impossible
for gpg (or a tty based pinentry) to prompt you for your passphrase
+in the same terminal where you run Gradle.
+
+Developers are encouraged to configure a *non* `tty` based `pinentry` (ex:
`pinentry-gnome`, `pinentry-x11`, `pinentry-qt`, `pinentry-mac`,
+`pinentry-wsl-ps1`, etc...) either globally in your operating system, or
personally in your `~/.gnupg/gpg-agent.conf`, or in a new
+`gpg-agent.conf` file a new GnuPG configuration directory (containing a copy
of your private keys) that you direct gradle to via
+`signing.gnupg.homeDir`
+
+If this is not possible, then you should avoid using an external GPG process,
and use the default (pure java) Artifact signing support
+
+
+### `gpg: signing failed: No such file or directory`
+
+This may mean that there is a problem preventing `gpg` from communicating
correctly with the `gpg-agent` (and/or invoking your `pinentry`
+program) that is independent of gradle. Try running `pkill gpg-agent` and
then retrying your `./gradlew` command
+
+
diff --git a/help/workflow.txt b/help/workflow.txt
index 2b22c3c..c59d77d 100644
--- a/help/workflow.txt
+++ b/help/workflow.txt
@@ -27,11 +27,22 @@ local maven repository for inspection:
gradlew mavenLocal
ls -R build/maven-local/
-Put together Solr distribution:
+Put together a local Solr binary "distribution" folder:
gradlew -p solr/packaging assemble
-ls solr/packaging/build/distributions/solr-* # release archives
ls solr/packaging/build/solr-* # expanded directory
+For quick local development
+gradlew -p solr/packaging dev
+ls solr/packaging/build/dev # expanded directory
+
+Generate the release tar and zip archives (see publishing.txt for details)
+gradlew -p solr/distribution assembleRelease
+ls solr/distribution/build/release # release archives
+
+Build a docker image from the local repository (see docker/gradle-help.txt for
more)
+gradlew dockerBuild dockerTag
+docker run --rm -p 8983:8983 apache/solr:9.0.0-SNAPSHOT
+
Other validation and checks
===========================
diff --git a/settings.gradle b/settings.gradle
index a04cd7c..88eab39 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -47,6 +47,7 @@ include "solr:solr-ref-guide"
include "solr:example"
include "solr:documentation"
include "solr:packaging"
+include "solr:distribution"
include "solr:docker"
// Configures development for joint Lucene/ Solr composite build.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 62cf2f1..11a0253 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -230,6 +230,8 @@ Build
* SOLR-15793: Pin http_parser.rb to specific version to allow Solr Ref Guide
on Jekyll to run (Eric Pugh)
+* SOLR-15867: Make the gradle build ready for 9.0 release (janhoy, Houston
Putman, Dawid Weiss)
+
Other Changes
----------------------
* SOLR-15603: Add an option to activate Gradle build cache, build task
cleanups (Alexis Tual, Dawid Weiss)
diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle
new file mode 100644
index 0000000..a586b09
--- /dev/null
+++ b/solr/distribution/build.gradle
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.gradle.Checksum
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+
+plugins {
+ id 'signing'
+}
+
+// This project puts together the Solr "distribution", assembling bits and
pieces
+// from across the project structure into release artifacts.
+
+ext {
+ releaseDir = file("${buildDir}/release")
+ withSignedArtifacts = { ->
+ def propValue = propertyOrDefault("sign", null)
+ // Allow -Psign to work as a shorthand for -Psign=true
+ return propValue != null && (propValue.isBlank() ||
Boolean.parseBoolean(propValue))
+ }.call()
+ useGpgForSigning = { ->
+ def propValue = propertyOrDefault("useGpg", null)
+ // Allow -PuseGpg to work as a shorthand for -PuseGpg=true
+ return propValue != null && (propValue.isBlank() ||
Boolean.parseBoolean(propValue))
+ }.call()
+}
+
+if (project.ext.useGpgForSigning) {
+ signing {
+ useGpgCmd()
+ }
+}
+
+// Prepare the "source" distribution artifact.
+apply from:
buildscript.sourceFile.toPath().resolveSibling("source-release.gradle")
+
+
+// Set up the HTML-rendered "changes" distribution artifact by linking to
documentation's output.
+configurations {
+ changesHtml
+ docker
+}
+
+dependencies {
+ changesHtml project(path: ":solr:documentation", configuration:
"changesHtml")
+ docker project(path: ':solr:docker', configuration:
project.ext.withSignedArtifacts ? 'packagingOfficial' : 'packaging')
+}
+
+def distTarTask = rootProject.getTasksByName("distTar", true)[0]
+def distZipTask = rootProject.getTasksByName("distZip", true)[0]
+
+// Compute checksums for release archives.
+task computeChecksums(type: Checksum) {
+ algorithm = Checksum.Algorithm.SHA512
+
+ files = objects.fileCollection()
+ [
+ tasks.assembleSourceTgz,
+ distTarTask,
+ distZipTask,
+ ].each { dep ->
+ dependsOn dep
+ files += dep.outputs.files
+ }
+
+ outputDir = file("${buildDir}/checksums")
+}
+
+task signBinaryTgz(type: Sign) {
+ sign distTarTask
+}
+task signBinaryZip(type: Sign) {
+ sign distZipTask
+}
+task signSourceTgz(type: Sign) {
+ // The source tgz is not an archive task so be explicit about the outputs to
sign.
+ dependsOn tasks.assembleSourceTgz
+ sign tasks.assembleSourceTgz.destination
+}
+
+task signReleaseArchives(type: Sync) {
+ from tasks.signBinaryTgz
+ from tasks.signBinaryZip
+ from tasks.signSourceTgz
+
+ into "${buildDir}/signatures"
+}
+
+task prepareGitRev() {
+ dependsOn ":gitStatus"
+
+ ext.outputFile = file("${buildDir}/.gitrev")
+
+ outputs.file(ext.outputFile)
+ inputs.property("gitrev", provider { -> rootProject.ext.gitRev })
+
+ doFirst {
+ Files.writeString(ext.outputFile.toPath(), rootProject.ext.gitRev,
StandardCharsets.UTF_8)
+ }
+}
+
+
+// Assemble everything needed in the release folder structure.
+task assembleRelease(type: Sync) {
+ description "Assemble all Solr artifacts for a release."
+ dependsOn ":mavenToLocalFolder"
+
+ from(configurations.changesHtml, {
+ into "changes"
+ })
+
+ from(configurations.docker, {
+ include 'Dockerfile.*'
+ into "docker"
+ })
+
+ from(rootProject.mavenLocalDir, {
+ into "maven"
+ })
+
+ from tasks.prepareGitRev
+ from tasks.assembleSourceTgz
+ from distTarTask
+ from distZipTask
+
+ from tasks.computeChecksums
+
+ // Conditionally, attach signatures of all the release archives.
+ if (project.ext.withSignedArtifacts) {
+ from tasks.signReleaseArchives
+ }
+
+ into releaseDir
+}
+
+
+// Add the description and task group to some of the tasks that make
+// sense at the user-level help.
+tasks.matching {it.name in [
+ "assembleSourceTgz",
+ "assembleRelease",
+]}.all {
+ group "distribution"
+}
diff --git a/solr/distribution/source-release.gradle
b/solr/distribution/source-release.gradle
new file mode 100644
index 0000000..9e0c11b
--- /dev/null
+++ b/solr/distribution/source-release.gradle
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+// Prepare the "source" distribution artifact. We use raw git export, no
additional complexity needed.
+
+configure(project(":solr:distribution")) {
+ task assembleSourceTgz() {
+ description "Assemble source Solr artifact as a .tgz file."
+
+ // Make sure no unstaged/ dirty files are present.
+ dependsOn ":gitStatus", ":checkWorkingCopyClean"
+
+ ext {
+ destination = file("${buildDir}/packages/solr-${version}-src.tgz")
+ }
+
+ inputs.property("git-revision", { -> rootProject.ext.gitRev })
+ outputs.file destination
+
+ doFirst {
+ quietExec {
+ executable = project.externalTool("git")
+ workingDir = project.rootDir
+
+ args += [
+ "archive",
+ "--format", "tgz",
+ "--prefix", "solr-${version}/",
+ "--output", destination,
+ "HEAD"
+ ]
+ }
+ }
+ }
+}
diff --git a/solr/docker/build.gradle b/solr/docker/build.gradle
index c2ed014..92cbb26 100644
--- a/solr/docker/build.gradle
+++ b/solr/docker/build.gradle
@@ -29,7 +29,7 @@ def dockerImageTag =
propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKER
def dockerImageName = propertyOrEnvOrDefault("solr.docker.imageName",
"SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}")
def baseDockerImage = propertyOrEnvOrDefault("solr.docker.baseImage",
"SOLR_DOCKER_BASE_IMAGE", 'openjdk:11-jre-slim')
-def releaseGpgFingerprint = propertyOrDefault('signing.gnupg.keyName','');
+def releaseGpgFingerprint =
propertyOrDefault('signing.gnupg.keyName',propertyOrDefault('signing.keyId',''));
// Build directory locations
def imageIdFile = "$buildDir/image-id"
@@ -41,6 +41,9 @@ configurations {
packaging {
canBeResolved = true
}
+ packagingOfficial {
+ canBeResolved = true
+ }
solrTgz {
canBeConsumed = false
canBeResolved = true
@@ -59,6 +62,7 @@ configurations {
}
ext {
+ dockerfilesDirPath = "${buildDir}/dockerfiles"
packagingDir = file("${buildDir}/packaging")
}
@@ -67,7 +71,11 @@ dependencies {
builtBy 'assemblePackaging'
}
- solrTgz project(path: ":solr:packaging", configuration: 'solrTgz')
+ packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official") {
+ builtBy 'createDockerfileOfficial'
+ }
+
+ solrTgz project(path: ":solr:packaging", configuration: "solrTgz")
solrTgzSignature project(path: ":solr:packaging", configuration:
'solrTgzSignature')
dockerImage files(imageIdFile) {
@@ -98,7 +106,7 @@ task assemblePackaging(type: Sync) {
from(projectDir, {
include "scripts/**"
})
- from(buildDir, {
+ from(dockerfilesDirPath, {
include 'Dockerfile.local'
})
into packagingDir
@@ -271,7 +279,7 @@ The final Dockerfiles are merely the snippet headers
combined with the snippet b
*/
[ dfLocalDetails, dfOfficialDetails ].each{ details ->
def fileName = "Dockerfile.${ -> details.name.toLowerCase(Locale.ROOT) }"
- def outFile = file("$buildDir/${fileName}")
+ def outFile = file("$dockerfilesDirPath/${fileName}")
def props = [
// Values defined here should be common (and consistent) across both
Dockerfiles
@@ -327,7 +335,7 @@ tasks.createDockerfileOfficial.dependsOn
configurations.solrTgz // to lazy compu
if (''.equals(releaseGpgFingerprint)) {
gradle.taskGraph.whenReady { graph ->
if ( graph.hasTask(createDockerfileOfficial) ) {
- throw new GradleException("No GPG keyName found, please see
help/gpgSigning.txt (GPG key is neccessary to create Dockerfile.official)")
+ throw new GradleException("No GPG keyName found, please see
help/publishing.txt (GPG key is neccessary to create Dockerfile.official)")
}
}
}
@@ -338,10 +346,10 @@ task testBuildDockerfileOfficial(type: Copy) {
dependsOn createDockerfileOfficial
dependsOn configurations.solrTgz
dependsOn configurations.solrTgzSignature
-
+
def mockHttpdHome = file("$smokeTestOfficial/mock-httpd-home");
- inputs.file("$buildDir/Dockerfile.official")
+ inputs.file("$dockerfilesDirPath/Dockerfile.official")
outputs.dir(mockHttpdHome)
outputs.file(imageIdFileOfficial)
@@ -383,7 +391,7 @@ task testBuildDockerfileOfficial(type: Copy) {
// *NOW* we can actually run our docker build command...
logger.lifecycle('Running docker build on Dockerfile.official...');
exec {
- standardInput =
file("${buildDir}/Dockerfile.official").newDataInputStream()
+ standardInput =
file("${dockerfilesDirPath}/Dockerfile.official").newDataInputStream()
commandLine 'docker', 'build',
'--add-host', "mock-solr-dl-server:${mockServerIp}",
'--no-cache', // force fresh downloads from our current network
diff --git a/solr/docker/gradle-help.txt b/solr/docker/gradle-help.txt
index 765fd44..83cc439 100644
--- a/solr/docker/gradle-help.txt
+++ b/solr/docker/gradle-help.txt
@@ -93,7 +93,7 @@ All users should build custom images using the instructions
above.
NOTE: All gradle commands for the Official Dockerfile below require the Solr
artifacts to be signed with a GPG Key.
For necessary inputs and properties, please refer to:
-gradlew helpGpgSigning
+gradlew helpPublishing
You can use the following command to build an official Solr Dockerfile.
The Dockerfile will be created at: solr/docker/build/Dockerfile.official
diff --git a/solr/docker/templates/Dockerfile.body.template
b/solr/docker/templates/Dockerfile.body.template
index 4aa7d3c..110a7a9 100644
--- a/solr/docker/templates/Dockerfile.body.template
+++ b/solr/docker/templates/Dockerfile.body.template
@@ -30,7 +30,7 @@
# TODO; arguably these permissions should have been set correctly previously
in the TAR
RUN set -ex; \
(cd /opt; ln -s solr-*/ solr); \
- rm -Rf /opt/solr/docs /opt/solr/docker/Dockerfile*
/opt/solr/dist/solr-solrj-*.jar /opt/solr/dist/solrj-lib
/opt/solr/dist/solr-core-*.jar; \
+ rm -Rf /opt/solr/docs /opt/solr/dist/solr-solrj-*.jar
/opt/solr/dist/solrj-lib /opt/solr/dist/solr-core-*.jar; \
find /opt/solr/ -type d -print0 | xargs -0 chmod 0755; \
find /opt/solr/ -type f -print0 | xargs -0 chmod 0644; \
chmod -R 0755 /opt/solr/docker/scripts /opt/solr/bin
/opt/solr/contrib/prometheus-exporter/bin/solr-exporter
/opt/solr/server/scripts/cloud-scripts
diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle
index 8f98c67..e2f86d1 100644
--- a/solr/packaging/build.gradle
+++ b/solr/packaging/build.gradle
@@ -21,7 +21,6 @@
plugins {
id 'base'
id 'distribution'
- id 'signing'
}
description = 'Solr distribution packaging'
@@ -81,6 +80,10 @@ dependencies {
docs project(path: ':solr:documentation', configuration: 'minimalSite')
docker project(path: ':solr:docker', configuration: 'packaging')
+
+ solrTgzSignature files("$buildDir/distributions/solr-${version}.tgz.asc") {
+ builtBy ":solr:distribution:signBinaryTgz"
+ }
}
distributions {
@@ -112,14 +115,6 @@ distributions {
include "README.md"
})
- // Should we include Lucene changes?
- /*
- from(project(":lucene").projectDir, {
- include "CHANGES.txt"
- rename { file -> 'LUCENE_CHANGES.txt' }
- })
- */
-
from(configurations.contrib, {
into "contrib"
})
@@ -164,48 +159,8 @@ task dev(type: Copy) {
into devDir
}
-assemble.dependsOn installDist
-
-
-// NOTE: we don't use the convinence DSL of the 'signing' extension to define
our 'Sign' tasks because
-// that (by default) adds our signature files to the 'archives' configuration
-- which is what
-// assemble & installDist try to copy/sync, so they wouldn't work w/o GPG
installed (which would be bad).
-//
-// We also want to hook in our own property check dependency since the default
error message from Sign task
-// refers to the wrong (internal only) property name ("signatory.keyId")
-signing {
- useGpgCmd() // so gpg-agent can be used
-}
-task failUnlessGpgKeyProperty {
- // placeholder that can be depended on by any task needing GPG key which
will 'fail fast' if it's not set.
- def propName = 'signing.gnupg.keyName'
-
- // This explicitly checks the taskGraph (instead of a simple 'doFirst') so
it can fail the user's gradle
- // invocation immediately before any unrelated build tasks may run in
parallel
- if ( ! project.hasProperty(propName) ) {
- gradle.taskGraph.whenReady { graph ->
- if ( graph.hasTask(failUnlessGpgKeyProperty) ) {
- // TODO: it would be really nice if taskGraph was an actual graph and
we could report what tasks in (transitive) depend on us
- throw new GradleException("'$propName' property must be set for GPG
signing, please see help/gpgSigning.txt")
- }
- }
- }
-}
-task signDistTar(type: Sign) {
- dependsOn failUnlessGpgKeyProperty
- sign configurations.solrTgz
-}
-dependencies {
- solrTgzSignature files(tasks.signDistTar.signatureFiles.singleFile) {
- builtBy signDistTar
- }
-}
-task signDistZip(type: Sign) {
- dependsOn failUnlessGpgKeyProperty
- sign configurations.solrZip
-}
-task signDist {
- group = 'Distribution'
- description = 'GPG Signs the main distributions'
- dependsOn signDistTar, signDistZip
+distTar {
+ compression = Compression.GZIP
}
+
+assemble.dependsOn installDist
\ No newline at end of file
