Hello,Every Hacker:
I am a junior software engineer. I tried to rewrite Stream as a sink and
found that when I read a large update of the database, I throw an exception
OutOfMemory. Can anyone give me some advice? Why am I wrong, thanks :-)
def run(url: String, user: String, password: String, replUser: String,
replPass: String): Unit = {
implicit val system: ActorSystem = ActorSystem("log-analyzer")
implicit val materializer: ActorMaterializer = ActorMaterializer()
implicit val ec: ExecutionContextExecutor = system.dispatcher
class DeadLetterListener extends Actor {
def receive: PartialFunction[Any, Unit] = {
case dl: DeadLetter ⇒
logger.error("dead letter: {}", dl)
}
}
val listener = system.actorOf(Props[DeadLetterListener])
system.eventStream.subscribe(listener, classOf[DeadLetter])
Class.forName("org.postgresql.Driver")
// check last confirmed lsn
val sqlConnection: Connection = this.getSqlConnection(url, user, password)
val lastLSNOpt: Option[LogSequenceNumber] =
getLastConfirmedLSN(SLOT_NAME).run(sqlConnection)
var stream: PGReplicationStream = null
var replConnection: PGConnection = this.getReplicationConnection(url,
replUser, replPass)
if (lastLSNOpt.isEmpty) {
logger.info(s"未发现slot信息，初始化slot: $SLOT_NAME")
createSlot(replConnection)
} else {
logger.info(s"从 ${lastLSNOpt.get} 开始")
}
// start stream flow process
// val counter = new AtomicInteger(0)
var lastTs = System.currentTimeMillis()
var lastReceiveLSN = LogSequenceNumber.INVALID_LSN
while (true) {
try {
if (replConnection == null ||
replConnection.asInstanceOf[Connection].isClosed) {
replConnection = this.getReplicationConnection(url, replUser, replPass)
}
stream = replConnection
.getReplicationAPI
.replicationStream
.logical
.withSlotName(SLOT_NAME)
.withSlotOption("include-xids", false)
// .withSlotOption("skip-empty-xacts", true)
test_decoding下的设定，wal2json不适用
.withStatusInterval(STATUS_INTERVAL.toSeconds.asInstanceOf[Int],
TimeUnit.SECONDS)
.withStartPosition(lastReceiveLSN)
.start() // start replication stream
object stream2iter extends Iterator[(LogSequenceNumber, String)] {
override def next: (LogSequenceNumber, String) = {
try {
//outofmemory when big update in database
val msg = stream.read
//println(msg)
val offset = msg.arrayOffset
val src = msg.array
val len = src.length - offset
val log = new String(src, offset, len)
logger.debug(log)
val tmpLSN = stream.getLastReceiveLSN
(tmpLSN, log)
} catch {
case e: Throwable =>
println("localhost: " + e)
(LogSequenceNumber.INVALID_LSN, "")
}
}
override def hasNext: Boolean = !stream.isClosed
}
val srcStream: Source[(LogSequenceNumber, String), NotUsed] =
Source.fromIterator(() => stream2iter)
val sink: Sink[LogRecord, Future[Done]] = Sink.foreach(log => {
logger.debug(s"LSN: ${log.lsn}, target: ${log.target.fullname}")
if (log.lsn != LogSequenceNumber.INVALID_LSN)
lastReceiveLSN = log.lsn
stream.setAppliedLSN(lastReceiveLSN)
stream.setFlushedLSN(lastReceiveLSN)
})
val totalG: RunnableGraph[NotUsed] = srcStream.via(graph(system)).to(sink)
val res = totalG.run()
} catch {
// 断连常态化
case e: PSQLException =>
logger.warn("Connection closed, retrying...", e)
val conn = replConnection.asInstanceOf[Connection]
if (!conn.isClosed) {
conn.close()
}
TimeUnit.SECONDS.sleep(1)
restarts.inc()
replConnection = null
case e: Throwable =>
logger.error("Connection can not be established!", e)
shutdown(system, e)
}
}
}
--
*****************************************************************************************************
** New discussion forum: https://discuss.akka.io/ replacing akka-user
google-group soon.
** This group will soon be put into read-only mode, and replaced by
discuss.akka.io
** More details: https://akka.io/blog/news/2018/03/13/discuss.akka.io-announced
*****************************************************************************************************
>>>>>>>>>>
>>>>>>>>>> Read the docs: http://akka.io/docs/
>>>>>>>>>> Check the FAQ:
>>>>>>>>>> http://doc.akka.io/docs/akka/current/additional/faq.html
>>>>>>>>>> Search the archives: https://groups.google.com/group/akka-user
---
You received this message because you are subscribed to the Google Groups "Akka
User List" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at https://groups.google.com/group/akka-user.
For more options, visit https://groups.google.com/d/optout.
