This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 32be6b5fb91 SOLR-17427: Fix url building in cli (#2678)
32be6b5fb91 is described below
commit 32be6b5fb910d6e7b125153dd33bfc385ef306cf
Author: Houston Putman <[email protected]>
AuthorDate: Fri Aug 30 10:40:37 2024 -0500
SOLR-17427: Fix url building in cli (#2678)
(cherry picked from commit e35207268b8b71559b9bdd8a164e9bc1f2018dfb)
---
.../core/src/java/org/apache/solr/cli/ApiTool.java | 2 +-
.../src/java/org/apache/solr/cli/PostTool.java | 82 ++++++++++++----------
.../java/org/apache/solr/cli/SimplePostTool.java | 35 ++-------
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 22 ++----
.../src/test/org/apache/solr/cli/ApiToolTest.java | 16 +++++
.../src/test/org/apache/solr/cli/PostToolTest.java | 34 +++++----
.../org/apache/solr/cli/SimplePostToolTest.java | 17 ++++-
7 files changed, 112 insertions(+), 96 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cli/ApiTool.java
b/solr/core/src/java/org/apache/solr/cli/ApiTool.java
index e6b6627cbfd..1c412062ec8 100644
--- a/solr/core/src/java/org/apache/solr/cli/ApiTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ApiTool.java
@@ -120,7 +120,7 @@ public class ApiTool extends ToolBase {
* @return Solr base url with port and root (from above example
http://localhost:8983/solr)
*/
public static String getSolrUrlFromUri(URI uri) {
- return uri.getScheme() + "://" + uri.getAuthority() + "/" +
uri.getPath().split("/")[1];
+ return uri.resolve("/" + uri.getPath().split("/")[1]).toString();
}
public static ModifiableSolrParams getSolrParamsFromUri(URI uri) {
diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java
b/solr/core/src/java/org/apache/solr/cli/PostTool.java
index 3077c8979db..08d1898f221 100644
--- a/solr/core/src/java/org/apache/solr/cli/PostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java
@@ -376,37 +376,32 @@ public class PostTool extends ToolBase {
private void doWebMode() {
reset();
int numPagesPosted;
- try {
- if (type != null) {
- throw new IllegalArgumentException(
- "Specifying content-type with \"--mode=web\" is not supported");
- }
+ if (type != null) {
+ throw new IllegalArgumentException(
+ "Specifying content-type with \"--mode=web\" is not supported");
+ }
- // Set Extracting handler as default
- solrUpdateUrl = appendUrlPath(solrUpdateUrl, "/extract");
+ // Set Extracting handler as default
+ solrUpdateUrl = appendUrlPath(solrUpdateUrl, "/extract");
- info("Posting web pages to Solr url " + solrUpdateUrl);
- auto = true;
- info(
- "Entering auto mode. Indexing pages with content-types corresponding
to file endings "
- + fileTypes);
- if (recursive > 0) {
- if (recursive > MAX_WEB_DEPTH) {
- recursive = MAX_WEB_DEPTH;
- warn("Too large recursion depth for web mode, limiting to " +
MAX_WEB_DEPTH + "...");
- }
- if (delay < DEFAULT_WEB_DELAY) {
- warn(
- "Never crawl an external web site faster than every 10 seconds,
your IP will probably be blocked");
- }
- info("Entering recursive mode, depth=" + recursive + ", delay=" +
delay + "s");
+ info("Posting web pages to Solr url " + solrUpdateUrl);
+ auto = true;
+ info(
+ "Entering auto mode. Indexing pages with content-types corresponding
to file endings "
+ + fileTypes);
+ if (recursive > 0) {
+ if (recursive > MAX_WEB_DEPTH) {
+ recursive = MAX_WEB_DEPTH;
+ warn("Too large recursion depth for web mode, limiting to " +
MAX_WEB_DEPTH + "...");
}
- numPagesPosted = postWebPages(args, 0, out);
- info(numPagesPosted + " web pages indexed.");
-
- } catch (URISyntaxException e) {
- warn("Wrong URL trying to append /extract to " + solrUpdateUrl);
+ if (delay < DEFAULT_WEB_DELAY) {
+ warn(
+ "Never crawl an external web site faster than every 10 seconds,
your IP will probably be blocked");
+ }
+ info("Entering recursive mode, depth=" + recursive + ", delay=" + delay
+ "s");
}
+ numPagesPosted = postWebPages(args, 0, out);
+ info(numPagesPosted + " web pages indexed.");
}
private void doStdinMode() {
@@ -530,7 +525,7 @@ public class PostTool extends ToolBase {
postFile(srcFile, out, type);
Thread.sleep(delay * 1000L);
filesPosted++;
- } catch (InterruptedException | URISyntaxException e) {
+ } catch (InterruptedException | MalformedURLException |
URISyntaxException e) {
throw new RuntimeException(e);
}
}
@@ -696,13 +691,14 @@ public class PostTool extends ToolBase {
* @param link the absolute or relative link
* @return the string version of the full URL
*/
- protected String computeFullUrl(URL baseUrl, String link) {
+ protected static String computeFullUrl(URL baseUrl, String link)
+ throws MalformedURLException, URISyntaxException {
if (link == null || link.length() == 0) {
return null;
}
if (!link.startsWith("http")) {
if (link.startsWith("/")) {
- link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + link;
+ link = baseUrl.toURI().resolve(link).toString();
} else {
if (link.contains(":")) {
return null; // Skip non-relative URLs
@@ -712,10 +708,12 @@ public class PostTool extends ToolBase {
int sep = path.lastIndexOf('/');
String file = path.substring(sep + 1);
if (file.contains(".") || file.contains("?")) {
- path = path.substring(0, sep);
+ path = path.substring(0, sep + 1);
+ } else {
+ path = path + "/";
}
}
- link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path +
"/" + link;
+ link = baseUrl.toURI().resolve(path + link).toString();
}
}
link = normalizeUrlEnding(link);
@@ -815,7 +813,8 @@ public class PostTool extends ToolBase {
}
/** Opens the file and posts its contents to the solrUrl, writes to response
to output. */
- public void postFile(File file, OutputStream output, String type) throws
URISyntaxException {
+ public void postFile(File file, OutputStream output, String type)
+ throws MalformedURLException, URISyntaxException {
InputStream is = null;
URI uri = solrUpdateUrl;
@@ -893,9 +892,20 @@ public class PostTool extends ToolBase {
* @param append the path to append
* @return the final URL version
*/
- protected static URI appendUrlPath(URI uri, String append) throws
URISyntaxException {
- var newPath = uri.getPath() + append;
- return new URI(uri.getScheme(), uri.getAuthority(), newPath,
uri.getQuery(), uri.getFragment());
+ protected static URI appendUrlPath(URI uri, String append) {
+ if (append == null || append.isEmpty()) {
+ return uri;
+ }
+ if (append.startsWith("/")) {
+ append = append.substring(1);
+ }
+ if (uri.getQuery() != null && !uri.getQuery().isEmpty()) {
+ append += "?" + uri.getQuery();
+ }
+ if (!uri.getPath().endsWith("/")) {
+ append = uri.getPath() + "/" + append;
+ }
+ return uri.resolve(append);
}
/**
diff --git a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
index f24fc5dcc9c..fa5cf60db66 100644
--- a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
@@ -766,35 +766,9 @@ public class SimplePostTool {
* @param link the absolute or relative link
* @return the string version of the full URL
*/
- protected String computeFullUrl(URL baseUrl, String link) {
- if (link == null || link.length() == 0) {
- return null;
- }
- if (!link.startsWith("http")) {
- if (link.startsWith("/")) {
- link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + link;
- } else {
- if (link.contains(":")) {
- return null; // Skip non-relative URLs
- }
- String path = baseUrl.getPath();
- if (!path.endsWith("/")) {
- int sep = path.lastIndexOf('/');
- String file = path.substring(sep + 1);
- if (file.contains(".") || file.contains("?")) {
- path = path.substring(0, sep);
- }
- }
- link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path +
"/" + link;
- }
- }
- link = normalizeUrlEnding(link);
- String l = link.toLowerCase(Locale.ROOT);
- // Simple brute force skip images
- if (l.endsWith(".jpg") || l.endsWith(".jpeg") || l.endsWith(".png") ||
l.endsWith(".gif")) {
- return null; // Skip images
- }
- return link;
+ protected String computeFullUrl(URL baseUrl, String link)
+ throws MalformedURLException, URISyntaxException {
+ return PostTool.computeFullUrl(baseUrl, link);
}
/**
@@ -947,8 +921,7 @@ public class SimplePostTool {
* @return the final URI version
*/
protected static URI appendUrlPath(URI uri, String append) throws
URISyntaxException {
- var newPath = uri.getPath() + append;
- return new URI(uri.getScheme(), uri.getAuthority(), newPath,
uri.getQuery(), uri.getFragment());
+ return PostTool.appendUrlPath(uri, append);
}
/**
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index be597c30363..e3bd54071dc 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -622,21 +622,13 @@ public class SolrCLI implements CLIO {
// Only consider URI path component when normalizing hostContext
if (urlPath.contains(hostContext)) {
String newSolrUrl =
- String.format(
- Locale.ROOT,
- "%s://%s:%s%s",
- uri.getScheme(),
- uri.getHost(),
- uri.getPort(),
- urlPath.substring(0, urlPath.indexOf(hostContext)));
- if (logUrlFormatWarning) {
- CLIO.out(
- "WARNING: URLs provided to this tool needn't include Solr's
context-root (e.g. \"/solr\"). Such URLs are deprecated and support for them
will be removed in a future release. Correcting from ["
- + solrUrl
- + "] to ["
- + newSolrUrl
- + "].");
- }
+ uri.resolve(urlPath.substring(0, urlPath.indexOf(hostContext)) +
"/").toString();
+ CLIO.out(
+ "WARNING: URLs provided to this tool needn't include Solr's
context-root (e.g. \"/solr\"). Such URLs are deprecated and support for them
will be removed in a future release. Correcting from ["
+ + solrUrl
+ + "] to ["
+ + newSolrUrl
+ + "].");
solrUrl = newSolrUrl;
}
if (solrUrl.endsWith("/")) {
diff --git a/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java
b/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java
index d817845854a..b013fec9de1 100644
--- a/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java
@@ -102,4 +102,20 @@ public class ApiToolTest extends SolrCloudTestCase {
String.format(Locale.ROOT, "Could not find string %s in response:
\n%s", find, json),
json.contains(find));
}
+
+ @Test
+ public void testSolrUrlParsing() throws Exception {
+ assertEquals(
+ "https://test-host.solr:8983/solr";,
+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr:8983/solr";)));
+ assertEquals(
+ "https://test-host.solr:8983/solr";,
+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr:8983/solr/test/api";)));
+ assertEquals(
+ "https://test-host.solr/solr";,
+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr/solr/test/api";)));
+ assertEquals(
+ "http://test-host.solr/solr";,
+
ApiTool.getSolrUrlFromUri(URI.create("http://test-host.solr/solr/test/api";)));
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
index 0e7485d2267..d32d99da76d 100644
--- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
@@ -194,30 +194,25 @@ public class PostToolTest extends SolrCloudTestCase {
}
@Test
- public void testComputeFullUrl() throws IOException {
-
- PostTool webPostTool = new PostTool();
-
+ public void testComputeFullUrl() throws IOException, URISyntaxException {
assertEquals(
"http://[ff01::114]/index.html";,
- webPostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"/index.html"));
+ PostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"/index.html"));
assertEquals(
"http://[ff01::114]/index.html";,
- webPostTool.computeFullUrl(
- URI.create("http://[ff01::114]/foo/bar/";).toURL(), "/index.html"));
+
PostTool.computeFullUrl(URI.create("http://[ff01::114]/foo/bar/";).toURL(),
"/index.html"));
assertEquals(
"http://[ff01::114]/fil.html";,
- webPostTool.computeFullUrl(
+ PostTool.computeFullUrl(
URI.create("http://[ff01::114]/foo.htm?baz#hello";).toURL(),
"fil.html"));
// TODO: How to know what is the base if URL path ends with "foo"??
// assertEquals("http://[ff01::114]/fil.html";, t_web.computeFullUrl(new
// URL("http://[ff01::114]/foo?baz#hello";), "fil.html"));
-
assertNull(webPostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"fil.jpg"));
+
assertNull(PostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"fil.jpg"));
assertNull(
- webPostTool.computeFullUrl(
- URI.create("http://[ff01::114]/";).toURL(),
"mailto:[email protected]";));
+ PostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"mailto:[email protected]";));
assertNull(
- webPostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"ftp://server/file";));
+ PostTool.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"ftp://server/file";));
}
@Test
@@ -248,6 +243,21 @@ public class PostToolTest extends SolrCloudTestCase {
assertEquals(
URI.create("http://[ff01::114]/a?foo=bar";),
PostTool.appendUrlPath(URI.create("http://[ff01::114]?foo=bar";),
"/a"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a?foo=bar";),
+ PostTool.appendUrlPath(URI.create("http://[ff01::114]/?foo=bar";),
"/a"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+ PostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar";),
"/b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+ PostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar";),
"/b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+ PostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar";),
"b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+ PostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar";),
"b"));
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/cli/SimplePostToolTest.java
b/solr/core/src/test/org/apache/solr/cli/SimplePostToolTest.java
index 64e6e1e5dc1..d8974c0fe68 100644
--- a/solr/core/src/test/org/apache/solr/cli/SimplePostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/SimplePostToolTest.java
@@ -106,7 +106,7 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
}
@Test
- public void testComputeFullUrl() throws MalformedURLException {
+ public void testComputeFullUrl() throws MalformedURLException,
URISyntaxException {
assertEquals(
"http://[ff01::114]/index.html";,
t_web.computeFullUrl(URI.create("http://[ff01::114]/";).toURL(),
"/index.html"));
@@ -160,6 +160,21 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
assertEquals(
URI.create("http://[ff01::114]/a?foo=bar";),
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]?foo=bar";),
"/a"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a?foo=bar";),
+
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]/?foo=bar";), "/a"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar";), "/b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar";),
"/b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar";), "b"));
+ assertEquals(
+ URI.create("http://[ff01::114]/a/b?foo=bar";),
+
SimplePostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar";), "b"));
}
@Test
