Github user QiangCai commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1470#discussion_r149283558
--- Diff:
streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
---
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.streaming
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.scheduler.{SparkListener,
SparkListenerApplicationEnd}
+import org.apache.spark.sql.SparkSession
+import
org.apache.spark.sql.execution.streaming.{CarbonAppendableStreamSink, Sink}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.dictionary.server.DictionaryServer
+import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel
+import org.apache.carbondata.spark.util.DataLoadingUtil
+import org.apache.carbondata.streaming.segment.StreamSegmentManager
+
+/**
+ * Stream sink factory
+ */
+object StreamSinkFactory {
+
+ private val LOGGER =
LogServiceFactory.getLogService(StreamSinkFactory.getClass.getCanonicalName)
+
+ def createStreamTableSink(
+ sparkSession: SparkSession,
+ carbonTable: CarbonTable,
+ parameters: Map[String, String]): Sink = {
+ validateParameters(parameters)
+
+ // prepare the stream segment
+ val segmentId = getStreamSegmentId(carbonTable)
+ // build load model
+ val carbonLoadModel = buildCarbonLoadModelForStream(
+ sparkSession,
+ carbonTable,
+ parameters,
+ segmentId)
+ // start server if necessary
+ val server = startDictionaryServer(
--- End diff --
this method will add SparkListener to sc, it will trigger shutdown on
application end.
---
