corrected data insertion

Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/f722127c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/f722127c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/f722127c

Branch: refs/heads/master
Commit: f722127ca410a88355cc3f6d952739845c7499ce
Parents: a166efc
Author: tpltnt <tpltnt.git...@dropcut.net>
Authored: Mon Feb 5 19:05:13 2018 +0100
Committer: tpltnt <tpltnt.git...@dropcut.net>
Committed: Mon Feb 5 19:05:13 2018 +0100

----------------------------------------------------------------------
 spot-ingest/pipelines/proxy/bluecoat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/f722127c/spot-ingest/pipelines/proxy/bluecoat.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/bluecoat.py 
b/spot-ingest/pipelines/proxy/bluecoat.py
index 01b9922..d476733 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -168,7 +168,7 @@ def save_data(rdd, sqc, db, db_table, topic):
         sqc.setConf("hive.exec.dynamic.partition", "true")
         sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
         hive_table = "{0}.{1}".format(db, db_table)
-        df.write.format("parquet").mode("append").insertInto(hive_table)
+        df.write.format("parquet").mode("append").partitionBy('y', 'm', 'd', 
'h').insertInto(hive_table)
 
     else:
         print("------------------------LISTENING KAFKA 
TOPIC:{0}------------------------".format(topic))

Reply via email to