This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 3ed47d7 SOLR-15405 AwaitsFix enough HDFS tests to get 3 consecutive
runs with no hdfs failures. (#127)
3ed47d7 is described below
commit 3ed47d7cdfeaa7cc4ad0b2aa758de396869de45a
Author: Gus Heck <[email protected]>
AuthorDate: Thu May 13 21:54:49 2021 -0400
SOLR-15405 AwaitsFix enough HDFS tests to get 3 consecutive runs with no
hdfs failures. (#127)
---
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 2 ++
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 2 ++
.../org/apache/solr/search/TestRecoveryHdfs.java | 38 ++++++++++++----------
.../org/apache/solr/update/TestHdfsUpdateLog.java | 18 +++++-----
4 files changed, 34 insertions(+), 26 deletions(-)
diff --git
a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index d88b1d1..9386b91 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -19,6 +19,7 @@ package org.apache.solr.cloud.hdfs;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.SolrQuery;
@@ -37,6 +38,7 @@ import org.junit.BeforeClass;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
[email protected](bugUrl = "SOLR-15405")
public class HDFSCollectionsAPITest extends SolrCloudTestCase {
private static MiniDFSCluster dfsCluster;
diff --git
a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index 7285d34..3b08bab 100644
--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.index.BaseTestCheckIndex;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -49,6 +50,7 @@ import
com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@SolrTestCaseJ4.SuppressSSL
[email protected](bugUrl = "SOLR-15405")
// commented out on: 24-Dec-2018
@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028";)
// 12-Jun-2018
public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
private static MiniDFSCluster dfsCluster;
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index eb4592c8..17a1d01 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -68,6 +69,7 @@ import static
org.apache.solr.update.processor.DistributingUpdateProcessorFactor
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
// TODO: longer term this should be combined with TestRecovery somehow ??
[email protected](bugUrl = "SOLR-15405")
public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// means that we've seen the leader and have version info (i.e. we are a
non-leader replica)
private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
@@ -78,17 +80,17 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
private static String hdfsUri;
private static FileSystem fs;
-
+
@After
public void afterTest() {
TestInjection.reset(); // do after every test, don't wait for AfterClass
}
-
+
@BeforeClass
public static void beforeClass() throws Exception {
dfsCluster =
HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
hdfsUri = HdfsTestUtil.getURI(dfsCluster);
-
+
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
@@ -98,10 +100,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
-
+
initCore("solrconfig-tlog.xml","schema15.xml");
}
-
+
@AfterClass
public static void afterClass() throws Exception {
IOUtils.closeQuietly(fs);
@@ -123,10 +125,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
@Test
public void testReplicationFactor() throws Exception {
- clearIndex();
-
+ clearIndex();
+
HdfsUpdateLog ulog = (HdfsUpdateLog)
h.getCore().getUpdateHandler().getUpdateLog();
-
+
assertU(commit());
addAndGetVersion(sdoc("id", "REP1"), null);
assertU(commit());
@@ -140,10 +142,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
break;
}
}
-
+
assertTrue("Expected to find tlogs with a replication factor of 2",
foundRep2);
}
-
+
@Test
public void testLogReplay() throws Exception {
try {
@@ -650,7 +652,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
req().close();
}
}
-
+
private void addDocs(int nDocs, int start, LinkedList<Long> versions) throws
Exception {
for (int i=0; i<nDocs; i++) {
versions.addFirst( addAndGetVersion( sdoc("id",Integer.toString(start +
nDocs)) , null) );
@@ -679,7 +681,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertU(commit());
String logDir =
h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
-
+
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
@@ -788,16 +790,16 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertU(adoc("id","F1"));
assertU(adoc("id","F2"));
assertU(adoc("id","F3"));
-
+
h.close();
-
-
+
+
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
FSDataOutputStream dos = fs.append(new Path(logDir,
files[files.length-1]));
-
+
dos.writeLong(0xffffffffffffffffL);
dos.writeChars("This should be appended to a good log file, representing
a bad partially written record.");
dos.close();
@@ -837,7 +839,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
TestInjection.skipIndexWriterCommitOnClose = true;
String logDir =
h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
-
+
clearIndex();
assertU(commit());
@@ -970,7 +972,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
System.arraycopy(to, 0, data, idx, to.length);
}
}
-
+
private static int indexOf(byte[] target, byte[] data, int start) {
outer: for (int i=start; i<data.length - target.length; i++) {
for (int j=0; j<target.length; j++) {
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index 2ac5051..2e65db5 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -23,6 +23,7 @@ import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -41,16 +42,17 @@ import
com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
[email protected](bugUrl = "SOLR-15405")
public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
private static String hdfsUri;
private static FileSystem fs;
-
+
@BeforeClass
public static void beforeClass() throws Exception {
dfsCluster =
HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
hdfsUri = HdfsTestUtil.getURI(dfsCluster);
-
+
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
@@ -60,10 +62,10 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
}
System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
-
+
initCore("solrconfig-tlog.xml","schema15.xml");
}
-
+
@AfterClass
public static void afterClass() throws Exception {
IOUtils.closeQuietly(fs);
@@ -90,14 +92,14 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
((DirectUpdateHandler2)
uhandler).getCommitTracker().setTimeUpperBound(100);
((DirectUpdateHandler2)
uhandler).getCommitTracker().setOpenSearcher(false);
final UpdateLog ulog = uhandler.getUpdateLog();
-
+
clearIndex();
assertU(commit());
-
+
// we hammer on init in a background thread to make
// sure we don't run into any filesystem already closed
// problems (SOLR-7113)
-
+
Thread thread = new Thread() {
public void run() {
int cnt = 0;
@@ -114,7 +116,7 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
}
}
};
-
+
Thread thread2 = new Thread() {
public void run() {
int cnt = 0;
