Github user frreiss commented on a diff in the pull request:
https://github.com/apache/spark/pull/14553#discussion_r83524491
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/socket.scala
---
@@ -92,21 +105,64 @@ class TextSocketSource(host: String, port: Int,
includeTimestamp: Boolean, sqlCo
override def schema: StructType = if (includeTimestamp)
TextSocketSource.SCHEMA_TIMESTAMP
else TextSocketSource.SCHEMA_REGULAR
- /** Returns the maximum available offset for this source. */
+ override def lastCommittedOffset: Option[Offset] = synchronized {
+ if (lastOffsetCommitted.offset == -1) {
+ None
+ } else {
+ Some(lastOffsetCommitted)
+ }
+ }
+
override def getOffset: Option[Offset] = synchronized {
- if (lines.isEmpty) None else Some(LongOffset(lines.size - 1))
+ if (currentOffset.offset == -1) {
+ None
+ } else {
+ Some(currentOffset)
+ }
}
/** Returns the data that is between the offsets (`start`, `end`]. */
override def getBatch(start: Option[Offset], end: Offset): DataFrame =
synchronized {
- val startIdx = start.map(_.asInstanceOf[LongOffset].offset.toInt +
1).getOrElse(0)
- val endIdx = end.asInstanceOf[LongOffset].offset.toInt + 1
- val data = synchronized { lines.slice(startIdx, endIdx) }
+ val startOrdinal =
+
start.map(_.asInstanceOf[LongOffset]).getOrElse(LongOffset(-1)).offset.toInt + 1
+ val endOrdinal = end.asInstanceOf[LongOffset].offset.toInt + 1
+
+ // Internal buffer only holds the batches after lastOffsetCommitted
+ val rawList = synchronized {
+ val sliceStart = startOrdinal - lastOffsetCommitted.offset.toInt - 1
+ val sliceEnd = endOrdinal - lastOffsetCommitted.offset.toInt - 1
+ batches.slice(sliceStart, sliceEnd)
+ }
+
import sqlContext.implicits._
+ val rawBatch = sqlContext.createDataset(rawList)
+
+
+
--- End diff --
Fixed in my local copy.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
