corrected data insertion
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/f722127c Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/f722127c Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/f722127c Branch: refs/heads/SPOT-181_ODM Commit: f722127ca410a88355cc3f6d952739845c7499ce Parents: a166efc Author: tpltnt <tpltnt.git...@dropcut.net> Authored: Mon Feb 5 19:05:13 2018 +0100 Committer: tpltnt <tpltnt.git...@dropcut.net> Committed: Mon Feb 5 19:05:13 2018 +0100 ---------------------------------------------------------------------- spot-ingest/pipelines/proxy/bluecoat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/f722127c/spot-ingest/pipelines/proxy/bluecoat.py ---------------------------------------------------------------------- diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py index 01b9922..d476733 100644 --- a/spot-ingest/pipelines/proxy/bluecoat.py +++ b/spot-ingest/pipelines/proxy/bluecoat.py @@ -168,7 +168,7 @@ def save_data(rdd, sqc, db, db_table, topic): sqc.setConf("hive.exec.dynamic.partition", "true") sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict") hive_table = "{0}.{1}".format(db, db_table) - df.write.format("parquet").mode("append").insertInto(hive_table) + df.write.format("parquet").mode("append").partitionBy('y', 'm', 'd', 'h').insertInto(hive_table) else: print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))
