This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-2-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d1834043864f58bbf3efea543271743dc33c4e14 Author: Daniel Robert <[email protected]> AuthorDate: Thu Oct 14 18:22:10 2021 -0400 Update CSV ingest code for tutorial (#18960) (cherry picked from commit fb00240db972ed2adb60357e3e9116830b3fad5b) --- docs/apache-airflow/pipeline_example.csv | 190 +++++++++++++++---------------- docs/apache-airflow/tutorial.rst | 54 ++++----- 2 files changed, 120 insertions(+), 124 deletions(-) diff --git a/docs/apache-airflow/pipeline_example.csv b/docs/apache-airflow/pipeline_example.csv index a1b9f2d..7e600d3 100644 --- a/docs/apache-airflow/pipeline_example.csv +++ b/docs/apache-airflow/pipeline_example.csv @@ -1,96 +1,96 @@ Serial Number,Company Name,Employee Markme,Description,Leave -9.78819E+12,TALES OF SHIVA,Mark,mark,0 -9.7801E+12,1Q84 THE COMPLETE TRILOGY,HARUKI MURAKAMI,Mark,0 -9.7802E+12,MY KUMAN,Mark,Mark,0 -9.78001E+12,THE GOD OF SMAAL THINGS,ARUNDHATI ROY,4TH HARPER COLLINS,2 -9.78055E+12,THE BLACK CIRCLE,Mark,4TH HARPER COLLINS,0 -9.78813E+12,THE THREE LAWS OF PERFORMANCE,Mark,4TH HARPER COLLINS,0 -9.78938E+12,CHAMarkKYA MANTRA,Mark,4TH HARPER COLLINS,0 -9.78818E+12,59.FLAGS,Mark,4TH HARPER COLLINS,0 -9.78074E+12,THE POWER OF POSITIVE THINKING FROM,Mark,A & A PUBLISHER,0 -9.78938E+12,YOU CAN IF YO THINK YO CAN,PEALE,A & A PUBLISHER,0 -9.78818E+12,DONGRI SE DUBAI TAK (MPH),Mark,A & A PUBLISHER,0 -9.78819E+12,MarkLANDA ADYTAN KOSH,Mark,AADISH BOOK DEPOT,0 -9.78819E+12,MarkLANDA VISHAL SHABD SAGAR,-,AADISH BOOK DEPOT,1 -8187776021,MarkLANDA CONCISE DICT(ENG TO HINDI),Mark,AADISH BOOK DEPOT,0 -9.78938E+12,LIEUTEMarkMarkT GENERAL BHAGAT: A SAGA OF BRAVERY AND LEADERSHIP,Mark,AAM COMICS,2 -9.78938E+12,LN. MarkIK SUNDER SINGH,N.A,AAN COMICS,0 -9.78938E+12,I AM KRISHMark,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 -9.78938E+12,DON'T TEACH ME TOLERANCE INDIA,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 -9.78938E+12,MUJHE SAHISHNUTA MAT SIKHAO BHARAT,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 -9.78938E+12,SECRETS OF DESTINY,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 -9.78938E+12,BHAGYA KE RAHASYA (HINDI) SECRET OF DESTINY,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 -9.78819E+12,MEIN MANN HOON,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 -9.78938E+12,I AM THE MIND,DEEP TRIVEDI,AATMARAM & SONS,0 -9.78035E+12,THE ART OF CHOOSING,SHEEMark IYENGAR,ABACUS,0 -9.78035E+12,IN SPITE OF THE GODS,EDWARD LUCE,ABACUS,1 -9.78819E+12,QUESTIONS & ANWERS ABOUT THE GREAT BIBLE,Mark,ABC PUBLISHERS DISTRIBUTORS,4 -9.78938E+12,NIBANDH EVAM KAHANI LEKHAN { HINDI },Mark,ABHI BOOKS,1 -9.78933E+12,INDIAN ECONOMY SINCE INDEPENDENCE 27TH /E,UMA KAPILA,ACADEMIC FOUNDATION,1 -9.78817E+12,ECONOMIC DEVELOPMENT AND POLICY IN INDIA,UMA KAPILA,ACADEMIC FOUNDATION,1 -9.78933E+12,INDIAN ECONOMY PERFORMANCE 18TH/E 2017-2018,UMA KAPILA,ACADEMIC FOUNDATION,2 -9.78933E+12,INDIAN ECONOMIC DEVELOPMENTSINCE 1947 (NO RETURMarkBLE),UMA KAPILA,ACADEMIC FOUNDATION,1 -9.78938E+12,PRELIMS SPECIAL READING COMPREHENSION PAPER II CSAT,MarkGENDRA PRATAP,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78938E+12,THE CONSTITUTION OF INDIA 2ND / E,AR KHAN,ACCESS PUBLISHING INDIA PVT.LTD,10 -9.78939E+12,"INDIAN HERITAGE ,ART & CULTURE",MADHUKAR,ACCESS PUBLISHING INDIA PVT.LTD,10 -9.78938E+12,BHARAT KA SAMVIDHAN,AR KHAN,ACCESS PUBLISHING INDIA PVT.LTD,4 -9.78938E+12,"ETHICS, INTEGRITY & APTITUDE ( 3RD/E)","P N ROY ,G SUBBA RAO",ACCESS PUBLISHING INDIA PVT.LTD,10 -9.78938E+12,GENERAL STUDIES PAPER -- I (2016),Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78938E+12,GENERAL STUDIES PAPER - II (2016),Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78938E+12,INDIAN AND WORLD GEOGRAPHY 2E,D R KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,10 -9.78938E+12,VASTUNISTHA PRASHN SANGRAHA: BHARAT KA ITIHAS,MEEMarkKSHI KANT,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78938E+12,"PHYSICAL, HUMAN AND ECONOMIC GEOGRAPHY",D R KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,4 -9.78938E+12,WORLD GEOGRAPHY,DR KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,5 -9.78938E+12,INDIA: MAP ENTRIES IN GEOGRAPHY,MAJID HUSAIN,ACCESS PUBLISHING INDIA PVT.LTD,5 -9.78938E+12,GOOD GOVERMarkNCE IN INDIA 2/ED.,G SUBBA RAO,ACCESS PUBLISHING INDIA PVT.LTD,1 -9.78938E+12,KAMYABI KE SUTRA-CIVIL SEWA PARIKSHA AAP KI MUTTHI MEIN,ASHOK KUMAR,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78938E+12,GENERAL SCIENCE PRELIRY EXAM,Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 -9.78174E+12,SUCCESS AND DYSLEXIA,SUCCESS AND DYSLEXIA,ACER PRESS,0 -9.78174E+12,AN EXTRAORDIMarkRY SCHOOL,SARA JAMES,ACER PRESS,0 -9.78174E+12,POWERFUL PRACTICES FOR READING IMPROVEMENT,GLASSWELL,ACER PRESS,0 -9.78174E+12,EARLY CHILDHOOD PLAY MATTERS,SHOMark BASS,ACER PRESS,0 -9.78174E+12,LEADING LEARNING AND TEACHING,STEPHEN DINHAM,ACER PRESS,0 -9.78174E+12,READING AND LEARNING DIFFICULTIES,PETER WESTWOOD,ACER PRESS,0 -9.78174E+12,NUMERACY AND LEARNING DIFFICULTIES,PETER WOODLAND],ACER PRESS,0 -9.78174E+12,TEACHING AND LEARNING DIFFICULTIES,PETER WOODLAND,ACER PRESS,0 -9.78174E+12,USING DATA TO IMPROVE LEARNING,ANTHONY SHADDOCK,ACER PRESS,0 -9.78174E+12,PATHWAYS TO SCHOOL SYSTEM IMPROVEMENT,MICHAEL GAFFNEY,ACER PRESS,0 -9.78174E+12,FOR THOSE WHO TEACH,PHIL RIDDEN,ACER PRESS,0 -9.78174E+12,KEYS TO SCHOOL LEADERSHIP,PHIL RIDDEN & JOHN DE NOBILE,ACER PRESS,0 -9.78174E+12,DIVERSE LITERACIES IN EARLY CHILDHOOD,LEONIE ARTHUR,ACER PRESS,0 -9.78174E+12,CREATIVE ARTS IN THE LIVESOF YOUNG CHILDREN,ROBYN EWING,ACER PRESS,0 -9.78174E+12,SOCIAL AND EMOTIOMarkL DEVELOPMENT,ROS LEYDEN AND ERIN SHALE,ACER PRESS,0 -9.78174E+12,DISCUSSIONS IN SCIENCE,TIM SPROD,ACER PRESS,0 -9.78174E+12,YOUNG CHILDREN LEARNING MATHEMATICS,ROBERT HUNTING,ACER PRESS,0 -9.78174E+12,COACHING CHILDREN,KELLY SUMICH,ACER PRESS,1 -9.78174E+12,TEACHING PHYSICAL EDUCATIOMarkL IN PRIMARY SCHOOL,JANET L CURRIE,ACER PRESS,0 -9.78174E+12,ASSESSMENT AND REPORTING,PHIL RIDDEN AND SANDY,ACER PRESS,0 -9.78174E+12,COLLABORATION IN LEARNING,MAL LEE AND LORRAE WARD,ACER PRESS,0 -9.78086E+12,RE-IMAGINING EDUCATIMarkL LEADERSHIP,BRIAN J.CALDWELL,ACER PRESS,0 -9.78086E+12,TOWARDS A MOVING SCHOOL,FLEMING & KLEINHENZ,ACER PRESS,0 -9.78086E+12,DESINGNING A THINKING A CURRICULAM,SUSAN WILKS,ACER PRESS,0 -9.78086E+12,LEADING A DIGITAL SCHOOL,MAL LEE AND MICHEAL GAFFNEY,ACER PRESS,0 -9.78086E+12,NUMERACY,WESTWOOD,ACER PRESS,0 -9.78086E+12,TEACHING ORAL LANGUAGE,JOHN MUNRO,ACER PRESS,0 -9.78086E+12,SPELLING,WESTWOOD,ACER PRESS,0 -9.78819E+12,STORIES OF SHIVA,Mark,ACK,0 -9.78819E+12,JAMSET JI TATA: THE MAN WHO SAW TOMORROW,,ACK,0 -9.78818E+12,HEROES FROM THE MAHABHARTA { 5-IN-1 },Mark,ACK,0 -9.78818E+12,SURYA,,ACK,0 -9.78818E+12,TALES OF THE MOTHER GODDESS,-,ACK,0 -9.78818E+12,ADVENTURES OF KRISHMark,Mark,ACK,0 -9.78818E+12,MAHATMA GANDHI,Mark,ACK,1 -9.78818E+12,TALES FROM THE PANCHATANTRA 3-IN-1,-,ACK,0 -9.78818E+12,YET MORE TALES FROM THE JATAKAS { 3-IN-1 },AMarkNT PAI,ACK,0 -9.78818E+12,LEGENDARY RULERS OF INDIA,-,ACK,0 -9.78818E+12,GREAT INDIAN CLASSIC,Mark,ACK,0 -9.78818E+12,TULSIDAS ' RAMAYAMark,Mark,ACK,0 -9.78818E+12,TALES OF HANUMAN,-,ACK,0 -9.78818E+12,VALMIKI'S RAMAYAMark,A C K,ACK,1 -9.78818E+12,THE BEST OF INIDAN WIT AND WISDOM,Mark,ACK,0 -9.78818E+12,MORE TALES FROM THE PANCHTANTRA,AMarkNT PAL,ACK,0 -9.78818E+12,THE GREAT MUGHALS {5-IN-1},AMarkNT.,ACK,0 -9.78818E+12,FAMOUS SCIENTISTS,Mark,ACK,0 -9.78818E+12,KOMarkRK,Mark,ACK,0 -9.78818E+12,THE MUGHAL COURT,REEMark,ACK,0 -9.78818E+12,MORE STORIES FROM THE JATAKAS,Mark,ACK,0 +2,TALES OF SHIVA,Mark,mark,0 +3,1Q84 THE COMPLETE TRILOGY,HARUKI MURAKAMI,Mark,0 +4,MY KUMAN,Mark,Mark,0 +5,THE GOD OF SMAAL THINGS,ARUNDHATI ROY,4TH HARPER COLLINS,2 +6,THE BLACK CIRCLE,Mark,4TH HARPER COLLINS,0 +7,THE THREE LAWS OF PERFORMANCE,Mark,4TH HARPER COLLINS,0 +8,CHAMarkKYA MANTRA,Mark,4TH HARPER COLLINS,0 +9,59.FLAGS,Mark,4TH HARPER COLLINS,0 +10,THE POWER OF POSITIVE THINKING FROM,Mark,A & A PUBLISHER,0 +11,YOU CAN IF YO THINK YO CAN,PEALE,A & A PUBLISHER,0 +12,DONGRI SE DUBAI TAK (MPH),Mark,A & A PUBLISHER,0 +13,MarkLANDA ADYTAN KOSH,Mark,AADISH BOOK DEPOT,0 +14,MarkLANDA VISHAL SHABD SAGAR,-,AADISH BOOK DEPOT,1 +15,MarkLANDA CONCISE DICT(ENG TO HINDI),Mark,AADISH BOOK DEPOT,0 +16,LIEUTEMarkMarkT GENERAL BHAGAT: A SAGA OF BRAVERY AND LEADERSHIP,Mark,AAM COMICS,2 +17,LN. MarkIK SUNDER SINGH,N.A,AAN COMICS,0 +18,I AM KRISHMark,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 +19,DON'T TEACH ME TOLERANCE INDIA,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 +20,MUJHE SAHISHNUTA MAT SIKHAO BHARAT,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 +21,SECRETS OF DESTINY,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 +22,BHAGYA KE RAHASYA (HINDI) SECRET OF DESTINY,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,1 +23,MEIN MANN HOON,DEEP TRIVEDI,AATMAN INNOVATIONS PVT LTD,0 +24,I AM THE MIND,DEEP TRIVEDI,AATMARAM & SONS,0 +25,THE ART OF CHOOSING,SHEEMark IYENGAR,ABACUS,0 +26,IN SPITE OF THE GODS,EDWARD LUCE,ABACUS,1 +27,QUESTIONS & ANWERS ABOUT THE GREAT BIBLE,Mark,ABC PUBLISHERS DISTRIBUTORS,4 +28,NIBANDH EVAM KAHANI LEKHAN { HINDI },Mark,ABHI BOOKS,1 +29,INDIAN ECONOMY SINCE INDEPENDENCE 27TH /E,UMA KAPILA,ACADEMIC FOUNDATION,1 +30,ECONOMIC DEVELOPMENT AND POLICY IN INDIA,UMA KAPILA,ACADEMIC FOUNDATION,1 +31,INDIAN ECONOMY PERFORMANCE 18TH/E 2017-2018,UMA KAPILA,ACADEMIC FOUNDATION,2 +32,INDIAN ECONOMIC DEVELOPMENTSINCE 1947 (NO RETURMarkBLE),UMA KAPILA,ACADEMIC FOUNDATION,1 +33,PRELIMS SPECIAL READING COMPREHENSION PAPER II CSAT,MarkGENDRA PRATAP,ACCESS PUBLISHING INDIA PVT.LTD,0 +34,THE CONSTITUTION OF INDIA 2ND / E,AR KHAN,ACCESS PUBLISHING INDIA PVT.LTD,10 +35,"INDIAN HERITAGE ,ART & CULTURE",MADHUKAR,ACCESS PUBLISHING INDIA PVT.LTD,10 +36,BHARAT KA SAMVIDHAN,AR KHAN,ACCESS PUBLISHING INDIA PVT.LTD,4 +37,"ETHICS, INTEGRITY & APTITUDE ( 3RD/E)","P N ROY ,G SUBBA RAO",ACCESS PUBLISHING INDIA PVT.LTD,10 +38,GENERAL STUDIES PAPER -- I (2016),Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 +39,GENERAL STUDIES PAPER - II (2016),Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 +40,INDIAN AND WORLD GEOGRAPHY 2E,D R KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,10 +41,VASTUNISTHA PRASHN SANGRAHA: BHARAT KA ITIHAS,MEEMarkKSHI KANT,ACCESS PUBLISHING INDIA PVT.LTD,0 +42,"PHYSICAL, HUMAN AND ECONOMIC GEOGRAPHY",D R KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,4 +43,WORLD GEOGRAPHY,DR KHULLAR,ACCESS PUBLISHING INDIA PVT.LTD,5 +44,INDIA: MAP ENTRIES IN GEOGRAPHY,MAJID HUSAIN,ACCESS PUBLISHING INDIA PVT.LTD,5 +45,GOOD GOVERMarkNCE IN INDIA 2/ED.,G SUBBA RAO,ACCESS PUBLISHING INDIA PVT.LTD,1 +46,KAMYABI KE SUTRA-CIVIL SEWA PARIKSHA AAP KI MUTTHI MEIN,ASHOK KUMAR,ACCESS PUBLISHING INDIA PVT.LTD,0 +47,GENERAL SCIENCE PRELIRY EXAM,Mark,ACCESS PUBLISHING INDIA PVT.LTD,0 +48,SUCCESS AND DYSLEXIA,SUCCESS AND DYSLEXIA,ACER PRESS,0 +49,AN EXTRAORDIMarkRY SCHOOL,SARA JAMES,ACER PRESS,0 +50,POWERFUL PRACTICES FOR READING IMPROVEMENT,GLASSWELL,ACER PRESS,0 +51,EARLY CHILDHOOD PLAY MATTERS,SHOMark BASS,ACER PRESS,0 +52,LEADING LEARNING AND TEACHING,STEPHEN DINHAM,ACER PRESS,0 +53,READING AND LEARNING DIFFICULTIES,PETER WESTWOOD,ACER PRESS,0 +54,NUMERACY AND LEARNING DIFFICULTIES,PETER WOODLAND],ACER PRESS,0 +55,TEACHING AND LEARNING DIFFICULTIES,PETER WOODLAND,ACER PRESS,0 +56,USING DATA TO IMPROVE LEARNING,ANTHONY SHADDOCK,ACER PRESS,0 +57,PATHWAYS TO SCHOOL SYSTEM IMPROVEMENT,MICHAEL GAFFNEY,ACER PRESS,0 +58,FOR THOSE WHO TEACH,PHIL RIDDEN,ACER PRESS,0 +59,KEYS TO SCHOOL LEADERSHIP,PHIL RIDDEN & JOHN DE NOBILE,ACER PRESS,0 +60,DIVERSE LITERACIES IN EARLY CHILDHOOD,LEONIE ARTHUR,ACER PRESS,0 +61,CREATIVE ARTS IN THE LIVESOF YOUNG CHILDREN,ROBYN EWING,ACER PRESS,0 +62,SOCIAL AND EMOTIOMarkL DEVELOPMENT,ROS LEYDEN AND ERIN SHALE,ACER PRESS,0 +63,DISCUSSIONS IN SCIENCE,TIM SPROD,ACER PRESS,0 +64,YOUNG CHILDREN LEARNING MATHEMATICS,ROBERT HUNTING,ACER PRESS,0 +65,COACHING CHILDREN,KELLY SUMICH,ACER PRESS,1 +66,TEACHING PHYSICAL EDUCATIOMarkL IN PRIMARY SCHOOL,JANET L CURRIE,ACER PRESS,0 +67,ASSESSMENT AND REPORTING,PHIL RIDDEN AND SANDY,ACER PRESS,0 +68,COLLABORATION IN LEARNING,MAL LEE AND LORRAE WARD,ACER PRESS,0 +69,RE-IMAGINING EDUCATIMarkL LEADERSHIP,BRIAN J.CALDWELL,ACER PRESS,0 +70,TOWARDS A MOVING SCHOOL,FLEMING & KLEINHENZ,ACER PRESS,0 +71,DESIGNING A THINKING A CURRICULAM,SUSAN WILKS,ACER PRESS,0 +72,LEADING A DIGITAL SCHOOL,MAL LEE AND MICHEAL GAFFNEY,ACER PRESS,0 +73,NUMERACY,WESTWOOD,ACER PRESS,0 +74,TEACHING ORAL LANGUAGE,JOHN MUNRO,ACER PRESS,0 +75,SPELLING,WESTWOOD,ACER PRESS,0 +76,STORIES OF SHIVA,Mark,ACK,0 +77,JAMSET JI TATA: THE MAN WHO SAW TOMORROW,,ACK,0 +78,HEROES FROM THE MAHABHARTA { 5-IN-1 },Mark,ACK,0 +79,SURYA,,ACK,0 +80,TALES OF THE MOTHER GODDESS,-,ACK,0 +81,ADVENTURES OF KRISHMark,Mark,ACK,0 +82,MAHATMA GANDHI,Mark,ACK,1 +83,TALES FROM THE PANCHATANTRA 3-IN-1,-,ACK,0 +84,YET MORE TALES FROM THE JATAKAS { 3-IN-1 },AMarkNT PAI,ACK,0 +85,LEGENDARY RULERS OF INDIA,-,ACK,0 +86,GREAT INDIAN CLASSIC,Mark,ACK,0 +87,TULSIDAS ' RAMAYAMark,Mark,ACK,0 +88,TALES OF HANUMAN,-,ACK,0 +89,VALMIKI'S RAMAYAMark,A C K,ACK,1 +90,THE BEST OF INDIAN WIT AND WISDOM,Mark,ACK,0 +91,MORE TALES FROM THE PANCHTANTRA,AMarkNT PAL,ACK,0 +92,THE GREAT MUGHALS {5-IN-1},AMarkNT.,ACK,0 +93,FAMOUS SCIENTISTS,Mark,ACK,0 +94,KOMarkRK,Mark,ACK,0 +95,THE MUGHAL COURT,REEMark,ACK,0 +96,MORE STORIES FROM THE JATAKAS,Mark,ACK,0 diff --git a/docs/apache-airflow/tutorial.rst b/docs/apache-airflow/tutorial.rst index df87f96..d9587bc 100644 --- a/docs/apache-airflow/tutorial.rst +++ b/docs/apache-airflow/tutorial.rst @@ -395,7 +395,7 @@ Create a Employee table in postgres using this create table "Employees_temp" ( "Serial Number" numeric not null - constraint employees_pk + constraint employees_temp_pk primary key, "Company Name" text, "Employee Markme" text, @@ -410,29 +410,23 @@ Let's break this down into 2 steps: get data & merge data: @task def get_data(): + data_path = "/usr/local/airflow/dags/files/employees.csv" + url = "https://raw.githubusercontent.com/apache/airflow/main/docs/apache-airflow/pipeline_example.csv"; response = requests.request("GET", url) - with open("/usr/local/airflow/dags/files/employees.csv", "w") as file: + with open(data_path, "w") as file: for row in response.text.split("\n"): - file.write(row) + if row: + file.write(row + "\n") postgres_hook = PostgresHook(postgres_conn_id="LOCAL") conn = postgres_hook.get_conn() cur = conn.cursor() - with open("/usr/local/airflow/dags/files/employees.csv", "r") as file: - cur.copy_from( - f, - "Employees_temp", - columns=[ - "Serial Number", - "Company Name", - "Employee Markme", - "Description", - "Leave", - ], - sep=",", + with open(data_path, "r") as file: + cur.copy_expert( + "COPY \"Employees_temp\" FROM stdin WITH CSV HEADER DELIMITER AS ','", file ) conn.commit() @@ -468,6 +462,12 @@ Lets look at our DAG: .. code-block:: python + from airflow.decorators import dag, task + from airflow.hooks.postgres_hook import PostgresHook + from datetime import datetime, timedelta + import requests + + @dag( schedule_interval="0 0 * * *", start_date=datetime(2021, 1, 1), @@ -477,29 +477,25 @@ Lets look at our DAG: def Etl(): @task def get_data(): + # NOTE: configure this as appropriate for your airflow environment + data_path = "/usr/local/airflow/dags/files/employees.csv" + url = "https://raw.githubusercontent.com/apache/airflow/main/docs/apache-airflow/pipeline_example.csv"; response = requests.request("GET", url) - with open("/usr/local/airflow/dags/files/employees.csv", "w") as file: + with open(data_path, "w") as file: for row in response.text.split("\n"): - file.write(row) + if row: + file.write(row + "\n") postgres_hook = PostgresHook(postgres_conn_id="LOCAL") conn = postgres_hook.get_conn() cur = conn.cursor() - with open("/usr/local/airflow/dags/files/employees.csv", "r") as file: - cur.copy_from( + with open(data_path, "r") as file: + cur.copy_expert( + "COPY \"Employees_temp\" FROM stdin WITH CSV HEADER DELIMITER AS ','", file, - "Employees_temp", - columns=[ - "Serial Number", - "Company Name", - "Employee Markme", - "Description", - "Leave", - ], - sep=",", ) conn.commit() @@ -530,7 +526,7 @@ Lets look at our DAG: dag = Etl() This dag runs daily at 00:00. -Add this python file to airflow/dags folder and go back to the main folder and run +Add this python file to airflow/dags folder (e.g. ``dags/etl.py``) and go back to the main folder and run .. code-block:: bash
