This is an automated email from the ASF dual-hosted git repository.
lhaiesp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
The following commit(s) were added to refs/heads/master by this push:
new 9bd3d61 SAMZA-2486: Update TestFileReaderSystemAdmin to use test
resource files (#1315)
9bd3d61 is described below
commit 9bd3d616b3699f344e92ccaadb4194f3b4179132
Author: Ke Wu <[email protected]>
AuthorDate: Mon Mar 16 16:57:48 2020 -0700
SAMZA-2486: Update TestFileReaderSystemAdmin to use test resource files
(#1315)
Symptom:
samza-core tests are randomly failing.
Cause:
TestFileReaderSystemAdmin and TestFileReaderSystemConsumer are creating and
deleting the same set of files, when runnig in parallel, FileNotFoundException
could happen depending on the timing.
Changes:
1. Update TestFileReaderSystemAdmin to use test resource files.
2. Update assert conditions as we removed tailing spaces and newlines in
the files.
3. Keep TestFileReaderSystemConsumer as it is because it is also appending
data to test files, thus cannot share to use the same test resource file.
Tests:
./gradlew test
API Changes:
None
Upgrade Instructions:
None
Usage Instructions:
None
Co-authored-by: Ke Wu <[email protected]>
---
build.gradle | 1 +
samza-core/src/test/resources/empty.txt | 0
samza-core/src/test/resources/moreEnter.txt | 4 ++
samza-core/src/test/resources/noEnter.txt | 1 +
samza-core/src/test/resources/oneEnter.txt | 2 +
samza-core/src/test/resources/twoEnter.txt | 3 +
.../filereader/TestFileReaderSystemAdmin.scala | 71 ++++++++--------------
7 files changed, 38 insertions(+), 44 deletions(-)
diff --git a/build.gradle b/build.gradle
index 16402ec..038c606 100644
--- a/build.gradle
+++ b/build.gradle
@@ -90,6 +90,7 @@ rat {
'README.md',
'RELEASE.md',
'samza-core/src/test/resources/classloader/samza-framework-api-classes.txt',
+ 'samza-core/src/test/resources/*.txt',
'samza-test/src/main/resources/**',
'samza-hdfs/src/main/resources/**',
'samza-hdfs/src/test/resources/**',
diff --git a/samza-core/src/test/resources/empty.txt
b/samza-core/src/test/resources/empty.txt
new file mode 100644
index 0000000..e69de29
diff --git a/samza-core/src/test/resources/moreEnter.txt
b/samza-core/src/test/resources/moreEnter.txt
new file mode 100644
index 0000000..a275be3
--- /dev/null
+++ b/samza-core/src/test/resources/moreEnter.txt
@@ -0,0 +1,4 @@
+first line
+second line
+third line
+other lines
diff --git a/samza-core/src/test/resources/noEnter.txt
b/samza-core/src/test/resources/noEnter.txt
new file mode 100644
index 0000000..6cc3bfa
--- /dev/null
+++ b/samza-core/src/test/resources/noEnter.txt
@@ -0,0 +1 @@
+first line
\ No newline at end of file
diff --git a/samza-core/src/test/resources/oneEnter.txt
b/samza-core/src/test/resources/oneEnter.txt
new file mode 100644
index 0000000..9c40573
--- /dev/null
+++ b/samza-core/src/test/resources/oneEnter.txt
@@ -0,0 +1,2 @@
+first line
+second line
\ No newline at end of file
diff --git a/samza-core/src/test/resources/twoEnter.txt
b/samza-core/src/test/resources/twoEnter.txt
new file mode 100644
index 0000000..c1d496e
--- /dev/null
+++ b/samza-core/src/test/resources/twoEnter.txt
@@ -0,0 +1,3 @@
+first line
+second line
+other lines
\ No newline at end of file
diff --git
a/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
b/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
index 8cc593a..96098c2 100644
---
a/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
+++
b/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
@@ -19,60 +19,38 @@
package org.apache.samza.system.filereader
-import java.io.PrintWriter
-import java.io.File
-
import org.apache.samza.system.SystemStreamPartition
import org.apache.samza.Partition
import
org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata
import org.junit.Assert._
import org.junit.Test
-import org.junit.Before
-import org.junit.After
import org.scalatest.junit.AssertionsForJUnit
-import scala.collection.mutable.HashMap
import scala.collection.JavaConverters._
+import scala.collection.mutable
class TestFileReaderSystemAdmin extends AssertionsForJUnit {
- val files = List("empty.txt", "noEnter.txt", "oneEnter.txt", "twoEnter.txt",
"moreEnter.txt")
-
- @Before
- def createFiles {
- files.foreach(file => {
- val writer = new PrintWriter(new File(file))
- file match {
- case "empty.txt" =>
- case "noEnter.txt" => writer.write("first line")
- case "oneEnter.txt" => writer.write("first line \nsecond line")
- case "twoEnter.txt" => writer.write("first line \nsecond line \nother
lines")
- case "moreEnter.txt" => writer.write("first line \nsecond line \nthird
line \nother lines \n")
- }
- writer.close
- })
- }
-
- @After
- def deleteFiles {
- files.foreach(file => (new File(file)).delete)
- }
+ val files = List(
+ getClass.getResource("/empty.txt").getPath,
+ getClass.getResource("/noEnter.txt").getPath,
+ getClass.getResource("/oneEnter.txt").getPath,
+ getClass.getResource("/twoEnter.txt").getPath,
+ getClass.getResource("/moreEnter.txt").getPath)
@Test
def testGetOffsetsAfter {
val fileReaderSystemAdmin = new FileReaderSystemAdmin
- val ssp1 = new SystemStreamPartition("file-reader", files(0), new
Partition(0))
- val ssp2 = new SystemStreamPartition("file-reader", files(1), new
Partition(0))
val ssp3 = new SystemStreamPartition("file-reader", files(2), new
Partition(0))
val ssp4 = new SystemStreamPartition("file-reader", files(3), new
Partition(0))
val ssp5 = new SystemStreamPartition("file-reader", files(4), new
Partition(0))
val offsets: java.util.Map[SystemStreamPartition, String] =
- HashMap(ssp3 -> "0", ssp4 -> "12", ssp5 -> "25").asJava
+ mutable.HashMap(ssp3 -> "0", ssp4 -> "12", ssp5 -> "25").asJava
val afterOffsets = fileReaderSystemAdmin.getOffsetsAfter(offsets)
- assertEquals("12", afterOffsets.get(ssp3))
- assertEquals("25", afterOffsets.get(ssp4))
- assertEquals("37", afterOffsets.get(ssp5))
+ assertEquals("11", afterOffsets.get(ssp3))
+ assertEquals("23", afterOffsets.get(ssp4))
+ assertEquals("34", afterOffsets.get(ssp5))
}
@Test
@@ -81,20 +59,25 @@ class TestFileReaderSystemAdmin extends AssertionsForJUnit {
val allMetadata =
fileReaderSystemAdmin.getSystemStreamMetadata(setAsJavaSetConverter(files.toSet).asJava)
val expectedEmpty = new SystemStreamPartitionMetadata(null, null, "0")
val expectedNoEntry = new SystemStreamPartitionMetadata("0", "0", "0")
- val expectedOneEntry = new SystemStreamPartitionMetadata("0", "0", "12")
- val expectedTwoEntry = new SystemStreamPartitionMetadata("0", "12", "25")
- val expectedMoreEntry = new SystemStreamPartitionMetadata("0", "37", "50")
+ val expectedOneEntry = new SystemStreamPartitionMetadata("0", "0", "11")
+ val expectedTwoEntry = new SystemStreamPartitionMetadata("0", "11", "23")
+ val expectedMoreEntry = new SystemStreamPartitionMetadata("0", "34", "46")
allMetadata.asScala.foreach { entry =>
{
- val result = (entry._2).getSystemStreamPartitionMetadata().get(new
Partition(0))
- entry._1 match {
- case "empty.txt" => assertEquals(expectedEmpty, result)
- case "noEnter.txt" => assertEquals(expectedNoEntry, result)
- case "oneEnter.txt" => assertEquals(expectedOneEntry, result)
- case "twoEnter.txt" => assertEquals(expectedTwoEntry, result)
- case "moreEnter.txt" => assertEquals(expectedMoreEntry, result)
- }
+ val result = entry._2.getSystemStreamPartitionMetadata.get(new
Partition(0))
+ if (entry._1.endsWith("empty.txt")) {
+ assertEquals(expectedEmpty, result)
+ } else if (entry._1.endsWith("noEnter.txt")) {
+ assertEquals(expectedNoEntry, result)
+ } else if (entry._1.endsWith("oneEnter.txt")) {
+ assertEquals(expectedOneEntry, result)
+ } else if (entry._1.endsWith("twoEnter.txt")) {
+ assertEquals(expectedTwoEntry, result)
+ } else if (entry._1.endsWith("moreEnter.txt")) {
+ assertEquals(expectedMoreEntry, result)
+ } else
+ fail()
}
}
}
