Hi Vino,
I am using a custom FileInputFormat, but the mentioned problem only comes
when I try a custom FilePathFilter.
My whole file for that custom FilePathFilter is quoted below.
Regarding enabling DEBUG, which classes/packages should I turn DEBUG on? as
I am afraid that turning DEBUG on at the global level would be too heavy.
Thanks and regards,
Averell
==================
import java.util.Date
import org.apache.flink.api.common.io.FilePathFilter
import org.apache.flink.core.fs.Path
import org.slf4j.LoggerFactory
object SdcFilePathFilter {
private val TIME_FORMAT = new java.text.SimpleDateFormat("yyyyMMdd
hhmm")
private val LOG = LoggerFactory.getLogger(classOf[SdcFilePathFilter])
}
class SdcFilePathFilter(lookBackPeriod: Long, homePath: Path) extends
FilePathFilter {
private val homeDepth = homePath.depth()
override def filterPath(filePath: Path): Boolean = {
filePath == null ||
filePath.getName.startsWith(".") ||
filePath.getName.startsWith("_") ||
filePath.getName.contains(FilePathFilter.HADOOP_COPYING) ||
!(filePath.getName.endsWith(".tar.gz") ||
filePath.getName.matches("""^\d{8}$""") ||
(filePath.getName.matches("""^\d{4}$""") &&
SdcFilePathFilter.TIME_FORMAT.parse(s"${filePath.getParent.getName}
${filePath.getName}").getTime <
new Date().getTime - lookBackPeriod))
}
}
--
Sent from: http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/
