This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 6583915b6c [Parquet Input/Output] Add comprehensive support for
Parquet Logical Types, fixes #5947 (#6186)
6583915b6c is described below
commit 6583915b6cd22aa52d45eb93860b4c4a2fa8b9a5
Author: jbs-atolcd <[email protected]>
AuthorDate: Mon Dec 15 16:50:00 2025 +0100
[Parquet Input/Output] Add comprehensive support for Parquet Logical
Types, fixes #5947 (#6186)
This commit introduces significant improvements to the Parquet Input and
Output transforms by implementing comprehensive support for Parquet's Logical
Types.
Previously, the transforms relied primarily on primitive types, leading to
conversions issues and errors with data when handling complex types, such as
Timestamps..
Key Changes & Features:
1. Parquet Input:
* Logical Type Mapping: Refactors the field discovery to use
`LogicalTypeAnnotation` (instead of only primitive type), enabling correct
mapping for semantic types.
* Timestamp/Date Precision: Implements a conversion mechanism to map
Parquet's timestamps units (MILLIS, MICROS..) to Hop's `TYPE_TIMESTAMP` and
`TYPE_DATE`, preserving precision and handling UTC adjustments.
* JSON Support: Adds explicit support for the JSON Logical Type, converting
the Parquet binary/string data into Hop's `TYPE_JSON` object.
* Decimal Handling: Uses precision and scale from
`DecimalLogicalTypeAnnotation` to correctly convert binary/long Parquet
decimals into Hop's `TYPE_BIGNUMBER`.
2. Parquet Output:
* Date/Timestamp Consistency: Ensures that Hop's `TYPE_DATE` and
`TYPE_TIMESTAMP` are consistently converted to a `LONG` representation with the
Parquet `timestampMillis` logical annotation, which is the most compatible
format.
* Schema Mapping: Maps Hop's `TYPE_JSON` and `TYPE_UUID` to Parquet
`STRING` types in the schema definition.
Testing and Validation:
* Test Data Enrichment: The test dataset (`golden-parquet-input.json`) was
extended to include new fields: `isActive` (Boolean), `registrationTimestamp`
(Timestamp), and `metadataJson` (JSON), ensuring the new types are covered
end-to-end.
* Unit Test Update: The unit test configuration (`0029-parquet-input
UNIT.json`) was updated to map and validate the new fields, confirming the
correct functionality of the transform.
This resolves a major limitation regarding data fidelity when dealing with
common modern Parquet schemas.
---
.../java/org/apache/hop/core/row/IValueMeta.java | 3 +
.../transforms/0029-parquet-input.hpl | 40 ++--
.../transforms/0029-parquet-output.hpl | 48 ++++-
.../transforms/datasets/golden-parquet-input.csv | 202 ++++++++++-----------
.../transforms/files/customers-100.txt | 202 ++++++++++-----------
.../metadata/dataset/golden-parquet-input.json | 64 +++++--
.../unit-test/0029-parquet-input UNIT.json | 26 ++-
.../parquet/transforms/input/ParquetInputMeta.java | 52 ++++--
.../transforms/input/ParquetRowConverter.java | 5 +-
.../transforms/input/ParquetValueConverter.java | 102 ++++++++++-
.../parquet/transforms/output/ParquetOutput.java | 49 ++++-
11 files changed, 528 insertions(+), 265 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
b/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
index 072cb48d52..cde4687d83 100644
--- a/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
+++ b/core/src/main/java/org/apache/hop/core/row/IValueMeta.java
@@ -199,6 +199,9 @@ public interface IValueMeta extends Cloneable {
/** Value type indicating that the value contains an Avro Record */
int TYPE_AVRO = 20;
+ /** Value type indicating that the value contains an UUID */
+ int TYPE_UUID = 32;
+
/** The Constant typeCodes. */
String[] typeCodes =
new String[] {
diff --git a/integration-tests/transforms/0029-parquet-input.hpl
b/integration-tests/transforms/0029-parquet-input.hpl
index 7d288daa11..260b18f970 100644
--- a/integration-tests/transforms/0029-parquet-input.hpl
+++ b/integration-tests/transforms/0029-parquet-input.hpl
@@ -67,26 +67,21 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <filter>
- <filterfiletype>all_files</filterfiletype>
- </filter>
<doNotFailIfNoFile>N</doNotFailIfNoFile>
- <rownum>N</rownum>
- <isaddresult>Y</isaddresult>
- <filefield>N</filefield>
- <rownum_field/>
- <filename_Field/>
- <wildcard_Field/>
- <exclude_wildcard_Field/>
<dynamic_include_subfolders>N</dynamic_include_subfolders>
- <limit>0</limit>
<file>
- <name>${java.io.tmpdir}/it/parquet/customers.parquet.snappy</name>
- <filemask/>
- <exclude_filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/it/parquet/customers.parquet.snappy</name>
</file>
+ <filefield>N</filefield>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <isaddresult>Y</isaddresult>
+ <limit>0</limit>
+ <raiseAnExceptionIfNoFile>N</raiseAnExceptionIfNoFile>
+ <rownum>N</rownum>
<attributes/>
<GUI>
<xloc>160</xloc>
@@ -156,6 +151,21 @@ limitations under the License.
<target_field>state</target_field>
<target_type>String</target_type>
</field>
+ <field>
+ <source_field>isActive</source_field>
+ <target_field>isActive</target_field>
+ <target_type>Boolean</target_type>
+ </field>
+ <field>
+ <source_field>registrationTimestamp</source_field>
+ <target_field>registrationTimestamp</target_field>
+ <target_type>Timestamp</target_type>
+ </field>
+ <field>
+ <source_field>metadataJson</source_field>
+ <target_field>metadataJson</target_field>
+ <target_type>JSON</target_type>
+ </field>
</fields>
<filename_field>filename</filename_field>
<attributes/>
@@ -194,7 +204,9 @@ limitations under the License.
</partitioning>
<fields>
<field>
+ <length>-2</length>
<name>filename</name>
+ <precision>-2</precision>
</field>
<select_unspecified>N</select_unspecified>
</fields>
diff --git a/integration-tests/transforms/0029-parquet-output.hpl
b/integration-tests/transforms/0029-parquet-output.hpl
index 4fabba275a..a99fe28f8b 100644
--- a/integration-tests/transforms/0029-parquet-output.hpl
+++ b/integration-tests/transforms/0029-parquet-output.hpl
@@ -101,8 +101,19 @@ limitations under the License.
<source_field>state</source_field>
<target_field>state</target_field>
</field>
+ <field>
+ <source_field>isActive</source_field>
+ <target_field>isActive</target_field>
+ </field>
+ <field>
+ <source_field>registrationTimestamp</source_field>
+ <target_field>registrationTimestamp</target_field>
+ </field>
+ <field>
+ <source_field>metadataJson</source_field>
+ <target_field>metadataJson</target_field>
+ </field>
</fields>
- <filename_split_size>1000000</filename_split_size>
<filename_base>${java.io.tmpdir}/it/parquet/customers</filename_base>
<filename_create_parent_folders>Y</filename_create_parent_folders>
<filename_datetime_format>yyyyMMdd-HHmmss</filename_datetime_format>
@@ -112,6 +123,7 @@ limitations under the License.
<filename_include_datetime>N</filename_include_datetime>
<filename_include_split>N</filename_include_split>
<filename_include_time>N</filename_include_time>
+ <filename_split_size>1000000</filename_split_size>
<row_group_size>20000</row_group_size>
<version>1.0</version>
<attributes/>
@@ -139,6 +151,7 @@ limitations under the License.
<enclosure>"</enclosure>
<header>Y</header>
<buffer_size>50000</buffer_size>
+ <schemaDefinition/>
<lazy_conversion>Y</lazy_conversion>
<add_filename_result>N</add_filename_result>
<parallel>N</parallel>
@@ -255,6 +268,39 @@ limitations under the License.
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
+ <field>
+ <name>isActive</name>
+ <type>Boolean</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>registrationTimestamp</name>
+ <type>Timestamp</type>
+ <format>yyyy-MM-dd HH:mm:ss.SSS</format>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>metadataJson</name>
+ <type>JSON</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>,</decimal>
+ <group> </group>
+ <length>64</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
</fields>
<attributes/>
<GUI>
diff --git a/integration-tests/transforms/datasets/golden-parquet-input.csv
b/integration-tests/transforms/datasets/golden-parquet-input.csv
index cb21054f8a..539861280a 100644
--- a/integration-tests/transforms/datasets/golden-parquet-input.csv
+++ b/integration-tests/transforms/datasets/golden-parquet-input.csv
@@ -1,101 +1,101 @@
-id,name,firstname,zip,city,birthdate,street,housenr,stateCode,state
-1,jwcdf-name,fsj-firstname,13520,oem-city,1954/02/07,amrb-street,145,AK,ALASKA
-2,flhxu-name,tum-firstname,17520,buo-city,1966/04/24,wfyz-street,96,GA,GEORGIA
-3,xthfg-name,gfe-firstname,12560,vtz-city,1990/01/11,doxx-street,46,NJ,NEW
JERSEY
-4,ulzrz-name,bnl-firstname,11620,prz-city,1966/08/02,bxqn-street,104,NY,NEW
YORK
-5,oxhyr-name,onx-firstname,15180,bpn-city,1970/11/14,pksn-street,133,IN,INDIANA
-6,fiqjz-name,sce-firstname,16020,fnn-city,1954/09/24,wbhg-street,35,MD,MARYLAND
-7,tkiat-name,xti-firstname,12720,stt-city,1966/08/11,tvnf-street,21,PA,PENNSYLVANIA
-8,kljcz-name,uqd-firstname,13340,ntt-city,1987/01/15,jyje-street,10,PW,PALAU
-9,pgunz-name,hcm-firstname,16680,gxh-city,1970/11/08,shbe-street,184,NC,NORTH
CAROLINA
-10,oyjha-name,uhj-firstname,18880,uyg-city,1966/04/10,bjgw-street,176,AR,ARKANSAS
-11,igxbd-name,uph-firstname,13480,ndh-city,1962/12/03,jdcd-street,151,NH,NEW
HAMPSHIRE
-12,vnaov-name,wha-firstname,13120,egm-city,1954/03/28,hpep-street,20,CA,CALIFORNIA
-13,dauuz-name,hwg-firstname,13740,khn-city,1958/05/15,etqx-street,5,OK,OKLAHOMA
-14,gkuuo-name,kkb-firstname,13560,xdt-city,1962/04/07,sdoj-street,35,MT,MONTANA
-15,wdhze-name,jjk-firstname,16900,due-city,1970/07/17,pmmu-street,174,AS,AMERICAN
SAMOA
-16,ncayz-name,ynb-firstname,15720,lxj-city,1974/04/27,mdtb-street,109,MA,MASSACHUSETTS
-17,rdjin-name,hhu-firstname,14480,lpc-city,1958/11/16,wxik-street,145,KY,KENTUCKY
-18,nxzij-name,bdl-firstname,10740,avx-city,1958/02/20,nybz-street,138,WI,WISCONSIN
-19,xgrzc-name,dxw-firstname,18900,vpq-city,1990/11/16,wzjh-street,58,ME,MAINE
-20,ehgrn-name,vbe-firstname,17500,cik-city,1978/05/21,ucnw-street,135,MD,MARYLAND
-21,gctjx-name,upx-firstname,11960,yqr-city,1958/03/03,rlko-street,141,TN,TENNESSEE
-22,ptzmg-name,hva-firstname,15740,gux-city,1978/05/04,pugy-street,122,VI,VIRGIN
ISLANDS
-23,eyeti-name,gnw-firstname,17420,eko-city,1962/10/26,ylph-street,61,NC,NORTH
CAROLINA
-24,wccwo-name,zpj-firstname,16600,uim-city,1962/09/29,ygih-street,26,WA,WASHINGTON
-25,bwkoe-name,ayl-firstname,18660,rtw-city,1978/07/16,mzww-street,179,CA,CALIFORNIA
-26,rezku-name,zio-firstname,19080,nvt-city,1982/07/14,wwkd-street,91,CA,CALIFORNIA
-27,mjlsk-name,ecx-firstname,10800,yxu-city,1950/12/11,vttb-street,195,MO,MISSOURI
-28,wdjsi-name,aoq-firstname,13660,smo-city,1954/02/01,kako-street,7,NV,NEVADA
-29,mwfnd-name,nyb-firstname,19760,bbu-city,1986/09/23,apdi-street,91,MS,MISSISSIPPI
-30,vtuoz-name,jhh-firstname,17620,vad-city,1982/05/05,kzup-street,79,GA,GEORGIA
-31,rhhxk-name,ndr-firstname,16760,fub-city,1978/11/12,regd-street,55,OK,OKLAHOMA
-32,lpstk-name,mqz-firstname,18940,tnr-city,1982/09/16,cdhf-street,4,SD,SOUTH
DAKOTA
-33,ldhyr-name,yts-firstname,12000,auk-city,1986/11/14,abph-street,147,IN,INDIANA
-34,cjdml-name,iti-firstname,16900,wkq-city,1970/06/05,npow-street,96,NH,NEW
HAMPSHIRE
-35,cpenz-name,sbi-firstname,16380,ssl-city,1962/08/19,kilz-street,44,MS,MISSISSIPPI
-36,rxtbg-name,anr-firstname,14720,bqc-city,1958/08/10,pudg-street,140,NV,NEVADA
-37,udblf-name,raa-firstname,11500,wli-city,1978/12/13,xomd-street,41,PW,PALAU
-38,vvyce-name,gep-firstname,13740,gtd-city,1982/05/23,kwbv-street,123,undefined,undefined
-39,kwfnz-name,ucu-firstname,10580,sns-city,1978/08/18,nnun-street,20,OK,OKLAHOMA
-40,zxydx-name,tml-firstname,14680,jda-city,1974/05/29,wfjn-street,157,DC,DISTRICT
OF COLUMBIA
-41,bfscx-name,jnl-firstname,16920,yyg-city,1970/11/30,cgfh-street,178,CO,COLORADO
-42,qitur-name,yra-firstname,15560,ijp-city,1978/01/30,fonc-street,155,AK,ALASKA
-43,msixi-name,ynb-firstname,12720,ksl-city,1958/07/17,zpjw-street,46,VI,VIRGIN
ISLANDS
-44,wzkjq-name,rgh-firstname,19000,hkm-city,1974/08/12,yixf-street,134,CA,CALIFORNIA
-45,dqfmf-name,yxr-firstname,13840,vie-city,1962/10/23,stvx-street,39,TX,TEXAS
-46,biluz-name,uqe-firstname,17760,wkq-city,1962/07/27,embn-street,183,PW,PALAU
-47,wahfx-name,zwd-firstname,13240,vic-city,1974/03/27,axpw-street,131,UT,UTAH
-48,denwt-name,bta-firstname,17300,hhj-city,1986/12/20,orwy-street,11,WV,WEST
VIRGINIA
-49,akdmy-name,ybz-firstname,14560,wtx-city,1962/11/08,nwba-street,123,MP,NORTHERN
MARIANA ISLANDS
-50,hqafg-name,nht-firstname,16080,gfu-city,1951/01/12,spsq-street,45,LA,LOUISIANA
-51,zhmbl-name,lnw-firstname,17460,hse-city,1986/12/21,scis-street,97,GA,GEORGIA
-52,snwnj-name,jyy-firstname,16400,hsz-city,1966/02/15,imhl-street,42,NC,NORTH
CAROLINA
-53,fuyla-name,mmp-firstname,11840,hgu-city,1986/08/16,ixiz-street,145,NC,NORTH
CAROLINA
-54,yvfqz-name,prz-firstname,11260,wjl-city,1982/05/06,fbzd-street,97,MO,MISSOURI
-55,usbgq-name,vhd-firstname,14080,dsb-city,1958/04/01,ggoc-street,54,KS,KANSAS
-56,yaeni-name,zpy-firstname,19100,sen-city,1954/12/10,sbsw-street,158,HI,HAWAII
-57,fgxvr-name,vzi-firstname,17520,lcf-city,1958/11/01,nbdv-street,10,GU,GUAM
-58,tqpbq-name,rwr-firstname,19140,zpd-city,1978/08/23,npvb-street,190,DC,DISTRICT
OF COLUMBIA
-59,ieigg-name,ayq-firstname,12960,ljc-city,1962/07/05,dnjz-street,163,FL,FLORIDA
-60,rfvzu-name,edm-firstname,13340,kvz-city,1954/12/08,eijd-street,4,RI,RHODE
ISLAND
-61,pduwm-name,gqb-firstname,14240,cyr-city,1954/07/03,ndux-street,13,SD,SOUTH
DAKOTA
-62,yyixf-name,yzt-firstname,18020,lwx-city,1974/01/29,iede-street,120,NV,NEVADA
-63,dkszq-name,ytd-firstname,14700,zwh-city,1979/01/11,nbjz-street,65,AS,AMERICAN
SAMOA
-64,slkzv-name,zbg-firstname,19880,oee-city,1978/11/01,sphg-street,119,OK,OKLAHOMA
-65,nvxim-name,phc-firstname,19220,vgg-city,1991/01/24,juok-street,106,FM,FEDERATED
STATES OF MICRONESIA
-66,piyfg-name,xtn-firstname,13760,nde-city,1954/07/22,vfrv-street,11,NY,NEW
YORK
-67,jnusz-name,mjw-firstname,12640,nwb-city,1986/08/23,kcsa-street,138,VA,VIRGINIA
-68,jnypj-name,ioq-firstname,17000,zqy-city,1986/01/09,croe-street,119,PW,PALAU
-69,uohts-name,btx-firstname,13480,dal-city,1990/10/22,llyw-street,150,WA,WASHINGTON
-70,aavpj-name,pvw-firstname,13780,lai-city,1954/09/23,nygu-street,171,FL,FLORIDA
-71,nbjcj-name,rsf-firstname,12000,kjl-city,1986/06/30,ijsb-street,123,ID,IDAHO
-72,syjxh-name,gkq-firstname,19960,rmd-city,1978/10/26,qmyp-street,161,MN,MINNESOTA
-73,vkojz-name,ryo-firstname,14300,bmz-city,1954/09/11,gcpj-street,71,ND,NORTH
DAKOTA
-74,pqzfw-name,kld-firstname,16400,qvq-city,1962/09/09,dhbv-street,92,ND,NORTH
DAKOTA
-75,owvjk-name,fez-firstname,19740,ldb-city,1978/06/14,kabf-street,87,VA,VIRGINIA
-76,qsfih-name,ixe-firstname,16860,qvr-city,1987/01/07,qean-street,159,CO,COLORADO
-77,slixq-name,gmb-firstname,19980,ftt-city,1982/06/22,xinx-street,111,VT,VERMONT
-78,eegsa-name,xlc-firstname,12680,byk-city,1954/04/23,beul-street,56,MD,MARYLAND
-79,phevp-name,ihs-firstname,16120,adc-city,1978/04/25,voig-street,98,NM,NEW
MEXICO
-80,njfoe-name,tag-firstname,16580,tnr-city,1966/12/04,dhky-street,108,LA,LOUISIANA
-81,bdncx-name,hcd-firstname,11260,xcl-city,1970/07/02,jvlp-street,49,GA,GEORGIA
-82,ikedo-name,tks-firstname,17460,odl-city,1958/08/25,iaaq-street,8,GU,GUAM
-83,iafxy-name,vur-firstname,11480,hgt-city,1962/08/03,hmec-street,164,TX,TEXAS
-84,lafhf-name,ssz-firstname,19560,wwp-city,1951/01/25,mxmq-street,96,IN,INDIANA
-85,okyny-name,hbu-firstname,16800,yok-city,1978/03/28,ipjz-street,135,NV,NEVADA
-86,hznby-name,fwy-firstname,13680,wbi-city,1970/07/25,mxui-street,170,CT,CONNECTICUT
-87,ztpoa-name,rzk-firstname,18500,qum-city,1970/07/26,blqr-street,152,ME,MAINE
-88,gitxz-name,axt-firstname,11800,fck-city,1974/01/12,tmjw-street,189,SD,SOUTH
DAKOTA
-89,ziomm-name,mcv-firstname,12940,iwq-city,1950/10/22,hqgj-street,140,DC,DISTRICT
OF COLUMBIA
-90,otncg-name,tuy-firstname,16540,ulk-city,1971/01/24,yuia-street,166,TX,TEXAS
-91,cnabb-name,hoq-firstname,16300,tuw-city,1962/06/17,ujvv-street,61,ME,MAINE
-92,ucogf-name,ggc-firstname,14500,fsj-city,1978/02/08,asfi-street,53,WV,WEST
VIRGINIA
-93,lbpmf-name,sdt-firstname,10780,ewj-city,1978/03/08,hxsp-street,102,NV,NEVADA
-94,tieqq-name,uyu-firstname,17740,wea-city,1966/10/31,abpl-street,187,MO,MISSOURI
-95,fsgwf-name,vjd-firstname,12460,ads-city,1970/11/29,yeou-street,10,MA,MASSACHUSETTS
-96,reeba-name,kzs-firstname,13100,zhc-city,1966/07/08,abmv-street,88,FL,FLORIDA
-97,shybc-name,gcp-firstname,10660,ahg-city,1950/12/15,hrqy-street,174,KS,KANSAS
-98,phszr-name,sst-firstname,13080,ydd-city,1954/09/23,quqn-street,2,RI,RHODE
ISLAND
-99,jteco-name,fxc-firstname,19760,agr-city,1986/05/06,dzxc-street,108,MD,MARYLAND
-100,qvaar-name,icx-firstname,16120,boc-city,1978/08/04,bfzf-street,12,NM,NEW
MEXICO
+id,name,firstname,zip,city,birthdate,street,housenr,stateCode,state,isActive,registrationTimestamp,metadataJson
+1,jwcdf-name,fsj-firstname,13520,oem-city,1954/02/07,amrb-street,145,AK,ALASKA,true,2025-12-16
09:47:47.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+2,flhxu-name,tum-firstname,17520,buo-city,1966/04/24,wfyz-street,96,GA,GEORGIA,false,2024-07-02
15:06:40.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+3,xthfg-name,gfe-firstname,12560,vtz-city,1990/01/11,doxx-street,46,NJ,NEW
JERSEY,true,2024-01-01
00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+4,ulzrz-name,bnl-firstname,11620,prz-city,1966/08/02,bxqn-street,104,NY,NEW
YORK,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+5,oxhyr-name,onx-firstname,15180,bpn-city,1970/11/14,pksn-street,133,IN,INDIANA,true,2022-09-01
05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+6,fiqjz-name,sce-firstname,16020,fnn-city,1954/09/24,wbhg-street,35,MD,MARYLAND,false,2022-01-01
00:00:00.000,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+7,tkiat-name,xti-firstname,12720,stt-city,1966/08/11,tvnf-street,21,PA,PENNSYLVANIA,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+8,kljcz-name,uqd-firstname,13340,ntt-city,1987/01/15,jyje-street,10,PW,PALAU,false,2020-01-01
00:00:00.456,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+9,pgunz-name,hcm-firstname,16680,gxh-city,1970/11/08,shbe-street,184,NC,NORTH
CAROLINA,true,2019-01-01
00:00:00.789,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+10,oyjha-name,uhj-firstname,18880,uyg-city,1966/04/10,bjgw-street,176,AR,ARKANSAS,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+11,igxbd-name,uph-firstname,13480,ndh-city,1962/12/03,jdcd-street,151,NH,NEW
HAMPSHIRE,true,2022-09-01
05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+12,vnaov-name,wha-firstname,13120,egm-city,1954/03/28,hpep-street,20,CA,CALIFORNIA,false,2022-01-01
00:00:00.000,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+13,dauuz-name,hwg-firstname,13740,khn-city,1958/05/15,etqx-street,5,OK,OKLAHOMA,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+14,gkuuo-name,kkb-firstname,13560,xdt-city,1962/04/07,sdoj-street,35,MT,MONTANA,false,2020-01-01
00:00:00.456,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+15,wdhze-name,jjk-firstname,16900,due-city,1970/07/17,pmmu-street,174,AS,AMERICAN
SAMOA,true,2024-01-01
00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+16,ncayz-name,ynb-firstname,15720,lxj-city,1974/04/27,mdtb-street,109,MA,MASSACHUSETTS,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+17,rdjin-name,hhu-firstname,14480,lpc-city,1958/11/16,wxik-street,145,KY,KENTUCKY,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+18,nxzij-name,bdl-firstname,10740,avx-city,1958/02/20,nybz-street,138,WI,WISCONSIN,false,2022-01-01
00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+19,xgrzc-name,dxw-firstname,18900,vpq-city,1990/11/16,wzjh-street,58,ME,MAINE,true,2021-01-01
00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+20,ehgrn-name,vbe-firstname,17500,cik-city,1978/05/21,ucnw-street,135,MD,MARYLAND,false,2020-01-01
00:00:00.456,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+21,gctjx-name,upx-firstname,11960,yqr-city,1958/03/03,rlko-street,141,TN,TENNESSEE,true,2019-01-01
00:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+22,ptzmg-name,hva-firstname,15740,gux-city,1978/05/04,pugy-street,122,VI,VIRGIN
ISLANDS,false,2023-03-31
12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+23,eyeti-name,gnw-firstname,17420,eko-city,1962/10/26,ylph-street,61,NC,NORTH
CAROLINA,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+24,wccwo-name,zpj-firstname,16600,uim-city,1962/09/29,ygih-street,26,WA,WASHINGTON,false,2022-01-01
00:00:00.000,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+25,bwkoe-name,ayl-firstname,18660,rtw-city,1978/07/16,mzww-street,179,CA,CALIFORNIA,true,2021-01-01
00:00:00.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+26,rezku-name,zio-firstname,19080,nvt-city,1982/07/14,wwkd-street,91,CA,CALIFORNIA,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+27,mjlsk-name,ecx-firstname,10800,yxu-city,1950/12/11,vttb-street,195,MO,MISSOURI,true,2022-09-01
05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+28,wdjsi-name,aoq-firstname,13660,smo-city,1954/02/01,kako-street,7,NV,NEVADA,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+29,mwfnd-name,nyb-firstname,19760,bbu-city,1986/09/23,apdi-street,91,MS,MISSISSIPPI,true,2021-01-01
00:00:00.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+30,vtuoz-name,jhh-firstname,17620,vad-city,1982/05/05,kzup-street,79,GA,GEORGIA,false,2020-01-01
00:00:00.456,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+31,rhhxk-name,ndr-firstname,16760,fub-city,1978/11/12,regd-street,55,OK,OKLAHOMA,true,2019-01-01
00:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+32,lpstk-name,mqz-firstname,18940,tnr-city,1982/09/16,cdhf-street,4,SD,SOUTH
DAKOTA,false,2023-03-31
12:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+33,ldhyr-name,yts-firstname,12000,auk-city,1986/11/14,abph-street,147,IN,INDIANA,true,2022-09-01
05:59:59.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+34,cjdml-name,iti-firstname,16900,wkq-city,1970/06/05,npow-street,96,NH,NEW
HAMPSHIRE,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+35,cpenz-name,sbi-firstname,16380,ssl-city,1962/08/19,kilz-street,44,MS,MISSISSIPPI,true,2021-01-01
00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+36,rxtbg-name,anr-firstname,14720,bqc-city,1958/08/10,pudg-street,140,NV,NEVADA,false,2020-01-01
00:00:00.456,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+37,udblf-name,raa-firstname,11500,wli-city,1978/12/13,xomd-street,41,PW,PALAU,true,2019-01-01
00:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+38,vvyce-name,gep-firstname,13740,gtd-city,1982/05/23,kwbv-street,123,undefined,undefined,false,2023-03-31
12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+39,kwfnz-name,ucu-firstname,10580,sns-city,1978/08/18,nnun-street,20,OK,OKLAHOMA,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+40,zxydx-name,tml-firstname,14680,jda-city,1974/05/29,wfjn-street,157,DC,DISTRICT
OF COLUMBIA,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+41,bfscx-name,jnl-firstname,16920,yyg-city,1970/11/30,cgfh-street,178,CO,COLORADO,true,2021-01-01
00:00:00.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+42,qitur-name,yra-firstname,15560,ijp-city,1978/01/30,fonc-street,155,AK,ALASKA,false,2020-01-01
00:00:00.456,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+43,msixi-name,ynb-firstname,12720,ksl-city,1958/07/17,zpjw-street,46,VI,VIRGIN
ISLANDS,true,2024-01-01
00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+44,wzkjq-name,rgh-firstname,19000,hkm-city,1974/08/12,yixf-street,134,CA,CALIFORNIA,false,2023-03-31
12:00:00.789,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+45,dqfmf-name,yxr-firstname,13840,vie-city,1962/10/23,stvx-street,39,TX,TEXAS,true,2022-09-01
05:59:59.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+46,biluz-name,uqe-firstname,17760,wkq-city,1962/07/27,embn-street,183,PW,PALAU,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+47,wahfx-name,zwd-firstname,13240,vic-city,1974/03/27,axpw-street,131,UT,UTAH,true,2021-01-01
00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+48,denwt-name,bta-firstname,17300,hhj-city,1986/12/20,orwy-street,11,WV,WEST
VIRGINIA,false,2020-01-01
00:00:00.456,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+49,akdmy-name,ybz-firstname,14560,wtx-city,1962/11/08,nwba-street,123,MP,NORTHERN
MARIANA ISLANDS,true,2019-01-01
00:00:00.789,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+50,hqafg-name,nht-firstname,16080,gfu-city,1951/01/12,spsq-street,45,LA,LOUISIANA,false,2023-03-31
12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+51,zhmbl-name,lnw-firstname,17460,hse-city,1986/12/21,scis-street,97,GA,GEORGIA,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+52,snwnj-name,jyy-firstname,16400,hsz-city,1966/02/15,imhl-street,42,NC,NORTH
CAROLINA,false,2022-01-01
00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+53,fuyla-name,mmp-firstname,11840,hgu-city,1986/08/16,ixiz-street,145,NC,NORTH
CAROLINA,true,2021-01-01
00:00:00.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+54,yvfqz-name,prz-firstname,11260,wjl-city,1982/05/06,fbzd-street,97,MO,MISSOURI,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+55,usbgq-name,vhd-firstname,14080,dsb-city,1958/04/01,ggoc-street,54,KS,KANSAS,true,2022-09-01
05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+56,yaeni-name,zpy-firstname,19100,sen-city,1954/12/10,sbsw-street,158,HI,HAWAII,false,2023-03-31
12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+57,fgxvr-name,vzi-firstname,17520,lcf-city,1958/11/01,nbdv-street,10,GU,GUAM,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+58,tqpbq-name,rwr-firstname,19140,zpd-city,1978/08/23,npvb-street,190,DC,DISTRICT
OF COLUMBIA,false,2022-01-01
00:00:00.000,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+59,ieigg-name,ayq-firstname,12960,ljc-city,1962/07/05,dnjz-street,163,FL,FLORIDA,true,2021-01-01
00:00:00.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+60,rfvzu-name,edm-firstname,13340,kvz-city,1954/12/08,eijd-street,4,RI,RHODE
ISLAND,false,2020-01-01
00:00:00.456,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+61,pduwm-name,gqb-firstname,14240,cyr-city,1954/07/03,ndux-street,13,SD,SOUTH
DAKOTA,true,2019-01-01
00:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+62,yyixf-name,yzt-firstname,18020,lwx-city,1974/01/29,iede-street,120,NV,NEVADA,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+63,dkszq-name,ytd-firstname,14700,zwh-city,1979/01/11,nbjz-street,65,AS,AMERICAN
SAMOA,true,2022-09-01
05:59:59.123,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+64,slkzv-name,zbg-firstname,19880,oee-city,1978/11/01,sphg-street,119,OK,OKLAHOMA,false,2022-01-01
00:00:00.000,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+65,nvxim-name,phc-firstname,19220,vgg-city,1991/01/24,juok-street,106,FM,FEDERATED
STATES OF MICRONESIA,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+66,piyfg-name,xtn-firstname,13760,nde-city,1954/07/22,vfrv-street,11,NY,NEW
YORK,false,2020-01-01
00:00:00.456,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+67,jnusz-name,mjw-firstname,12640,nwb-city,1986/08/23,kcsa-street,138,VA,VIRGINIA,true,2024-01-01
00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+68,jnypj-name,ioq-firstname,17000,zqy-city,1986/01/09,croe-street,119,PW,PALAU,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+69,uohts-name,btx-firstname,13480,dal-city,1990/10/22,llyw-street,150,WA,WASHINGTON,true,2022-09-01
05:59:59.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+70,aavpj-name,pvw-firstname,13780,lai-city,1954/09/23,nygu-street,171,FL,FLORIDA,false,2022-01-01
00:00:00.000,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+71,nbjcj-name,rsf-firstname,12000,kjl-city,1986/06/30,ijsb-street,123,ID,IDAHO,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+72,syjxh-name,gkq-firstname,19960,rmd-city,1978/10/26,qmyp-street,161,MN,MINNESOTA,false,2020-01-01
00:00:00.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+73,vkojz-name,ryo-firstname,14300,bmz-city,1954/09/11,gcpj-street,71,ND,NORTH
DAKOTA,true,2019-01-01
00:00:00.789,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+74,pqzfw-name,kld-firstname,16400,qvq-city,1962/09/09,dhbv-street,92,ND,NORTH
DAKOTA,false,2023-03-31
12:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+75,owvjk-name,fez-firstname,19740,ldb-city,1978/06/14,kabf-street,87,VA,VIRGINIA,true,2022-09-01
05:59:59.123,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+76,qsfih-name,ixe-firstname,16860,qvr-city,1987/01/07,qean-street,159,CO,COLORADO,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+77,slixq-name,gmb-firstname,19980,ftt-city,1982/06/22,xinx-street,111,VT,VERMONT,true,2021-01-01
00:00:00.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+78,eegsa-name,xlc-firstname,12680,byk-city,1954/04/23,beul-street,56,MD,MARYLAND,false,2023-03-31
12:00:00.789,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+79,phevp-name,ihs-firstname,16120,adc-city,1978/04/25,voig-street,98,NM,NEW
MEXICO,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+80,njfoe-name,tag-firstname,16580,tnr-city,1966/12/04,dhky-street,108,LA,LOUISIANA,false,2023-03-31
12:00:00.789,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+81,bdncx-name,hcd-firstname,11260,xcl-city,1970/07/02,jvlp-street,49,GA,GEORGIA,true,2022-09-01
05:59:59.123,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+82,ikedo-name,tks-firstname,17460,odl-city,1958/08/25,iaaq-street,8,GU,GUAM,false,2022-01-01
00:00:00.000,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+83,iafxy-name,vur-firstname,11480,hgt-city,1962/08/03,hmec-street,164,TX,TEXAS,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+84,lafhf-name,ssz-firstname,19560,wwp-city,1951/01/25,mxmq-street,96,IN,INDIANA,false,2020-01-01
00:00:00.456,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+85,okyny-name,hbu-firstname,16800,yok-city,1978/03/28,ipjz-street,135,NV,NEVADA,true,2019-01-01
00:00:00.789,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+86,hznby-name,fwy-firstname,13680,wbi-city,1970/07/25,mxui-street,170,CT,CONNECTICUT,false,2023-03-31
12:00:00.789,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+87,ztpoa-name,rzk-firstname,18500,qum-city,1970/07/26,blqr-street,152,ME,MAINE,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+88,gitxz-name,axt-firstname,11800,fck-city,1974/01/12,tmjw-street,189,SD,SOUTH
DAKOTA,false,2022-01-01
00:00:00.000,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+89,ziomm-name,mcv-firstname,12940,iwq-city,1950/10/22,hqgj-street,140,DC,DISTRICT
OF COLUMBIA,true,2021-01-01
00:00:00.123,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+90,otncg-name,tuy-firstname,16540,ulk-city,1971/01/24,yuia-street,166,TX,TEXAS,false,2020-01-01
00:00:00.456,"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+91,cnabb-name,hoq-firstname,16300,tuw-city,1962/06/17,ujvv-street,61,ME,MAINE,true,2024-01-01
00:00:00.123,"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+92,ucogf-name,ggc-firstname,14500,fsj-city,1978/02/08,asfi-street,53,WV,WEST
VIRGINIA,false,2023-03-31
12:00:00.789,"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+93,lbpmf-name,sdt-firstname,10780,ewj-city,1978/03/08,hxsp-street,102,NV,NEVADA,true,2022-09-01
05:59:59.123,"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+94,tieqq-name,uyu-firstname,17740,wea-city,1966/10/31,abpl-street,187,MO,MISSOURI,false,2022-01-01
00:00:00.000,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+95,fsgwf-name,vjd-firstname,12460,ads-city,1970/11/29,yeou-street,10,MA,MASSACHUSETTS,true,2021-01-01
00:00:00.123,"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+96,reeba-name,kzs-firstname,13100,zhc-city,1966/07/08,abmv-street,88,FL,FLORIDA,false,2020-01-01
00:00:00.456,"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+97,shybc-name,gcp-firstname,10660,ahg-city,1950/12/15,hrqy-street,174,KS,KANSAS,true,2019-01-01
00:00:00.789,"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+98,phszr-name,sst-firstname,13080,ydd-city,1954/09/23,quqn-street,2,RI,RHODE
ISLAND,false,2023-03-31
12:00:00.789,"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+99,jteco-name,fxc-firstname,19760,agr-city,1986/05/06,dzxc-street,108,MD,MARYLAND,true,2022-09-01
05:59:59.123,"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+100,qvaar-name,icx-firstname,16120,boc-city,1978/08/04,bfzf-street,12,NM,NEW
MEXICO,false,2022-01-01
00:00:00.000,"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
diff --git a/integration-tests/transforms/files/customers-100.txt
b/integration-tests/transforms/files/customers-100.txt
index 8b8bd04955..fca1fbb8d6 100644
--- a/integration-tests/transforms/files/customers-100.txt
+++ b/integration-tests/transforms/files/customers-100.txt
@@ -1,101 +1,101 @@
-id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state
- 1;jwcdf-name;fsj-firstname; 13520;oem-city;1954/02/07;amrb-street;
145;AK;ALASKA
- 2;flhxu-name;tum-firstname; 17520;buo-city;1966/04/24;wfyz-street;
96;GA;GEORGIA
- 3;xthfg-name;gfe-firstname; 12560;vtz-city;1990/01/11;doxx-street; 46;NJ;NEW
JERSEY
- 4;ulzrz-name;bnl-firstname; 11620;prz-city;1966/08/02;bxqn-street; 104;NY;NEW
YORK
- 5;oxhyr-name;onx-firstname; 15180;bpn-city;1970/11/14;pksn-street;
133;IN;INDIANA
- 6;fiqjz-name;sce-firstname; 16020;fnn-city;1954/09/24;wbhg-street;
35;MD;MARYLAND
- 7;tkiat-name;xti-firstname; 12720;stt-city;1966/08/11;tvnf-street;
21;PA;PENNSYLVANIA
- 8;kljcz-name;uqd-firstname; 13340;ntt-city;1987/01/15;jyje-street; 10;PW;PALAU
- 9;pgunz-name;hcm-firstname; 16680;gxh-city;1970/11/08;shbe-street;
184;NC;NORTH CAROLINA
- 10;oyjha-name;uhj-firstname; 18880;uyg-city;1966/04/10;bjgw-street;
176;AR;ARKANSAS
- 11;igxbd-name;uph-firstname; 13480;ndh-city;1962/12/03;jdcd-street;
151;NH;NEW HAMPSHIRE
- 12;vnaov-name;wha-firstname; 13120;egm-city;1954/03/28;hpep-street;
20;CA;CALIFORNIA
- 13;dauuz-name;hwg-firstname; 13740;khn-city;1958/05/15;etqx-street;
5;OK;OKLAHOMA
- 14;gkuuo-name;kkb-firstname; 13560;xdt-city;1962/04/07;sdoj-street;
35;MT;MONTANA
- 15;wdhze-name;jjk-firstname; 16900;due-city;1970/07/17;pmmu-street;
174;AS;AMERICAN SAMOA
- 16;ncayz-name;ynb-firstname; 15720;lxj-city;1974/04/27;mdtb-street;
109;MA;MASSACHUSETTS
- 17;rdjin-name;hhu-firstname; 14480;lpc-city;1958/11/16;wxik-street;
145;KY;KENTUCKY
- 18;nxzij-name;bdl-firstname; 10740;avx-city;1958/02/20;nybz-street;
138;WI;WISCONSIN
- 19;xgrzc-name;dxw-firstname; 18900;vpq-city;1990/11/16;wzjh-street;
58;ME;MAINE
- 20;ehgrn-name;vbe-firstname; 17500;cik-city;1978/05/21;ucnw-street;
135;MD;MARYLAND
- 21;gctjx-name;upx-firstname; 11960;yqr-city;1958/03/03;rlko-street;
141;TN;TENNESSEE
- 22;ptzmg-name;hva-firstname; 15740;gux-city;1978/05/04;pugy-street;
122;VI;VIRGIN ISLANDS
- 23;eyeti-name;gnw-firstname; 17420;eko-city;1962/10/26;ylph-street;
61;NC;NORTH CAROLINA
- 24;wccwo-name;zpj-firstname; 16600;uim-city;1962/09/29;ygih-street;
26;WA;WASHINGTON
- 25;bwkoe-name;ayl-firstname; 18660;rtw-city;1978/07/16;mzww-street;
179;CA;CALIFORNIA
- 26;rezku-name;zio-firstname; 19080;nvt-city;1982/07/14;wwkd-street;
91;CA;CALIFORNIA
- 27;mjlsk-name;ecx-firstname; 10800;yxu-city;1950/12/11;vttb-street;
195;MO;MISSOURI
- 28;wdjsi-name;aoq-firstname; 13660;smo-city;1954/02/01;kako-street;
7;NV;NEVADA
- 29;mwfnd-name;nyb-firstname; 19760;bbu-city;1986/09/23;apdi-street;
91;MS;MISSISSIPPI
- 30;vtuoz-name;jhh-firstname; 17620;vad-city;1982/05/05;kzup-street;
79;GA;GEORGIA
- 31;rhhxk-name;ndr-firstname; 16760;fub-city;1978/11/12;regd-street;
55;OK;OKLAHOMA
- 32;lpstk-name;mqz-firstname; 18940;tnr-city;1982/09/16;cdhf-street;
4;SD;SOUTH DAKOTA
- 33;ldhyr-name;yts-firstname; 12000;auk-city;1986/11/14;abph-street;
147;IN;INDIANA
- 34;cjdml-name;iti-firstname; 16900;wkq-city;1970/06/05;npow-street; 96;NH;NEW
HAMPSHIRE
- 35;cpenz-name;sbi-firstname; 16380;ssl-city;1962/08/19;kilz-street;
44;MS;MISSISSIPPI
- 36;rxtbg-name;anr-firstname; 14720;bqc-city;1958/08/10;pudg-street;
140;NV;NEVADA
- 37;udblf-name;raa-firstname; 11500;wli-city;1978/12/13;xomd-street;
41;PW;PALAU
- 38;vvyce-name;gep-firstname; 13740;gtd-city;1982/05/23;kwbv-street;
123;undefined;undefined
- 39;kwfnz-name;ucu-firstname; 10580;sns-city;1978/08/18;nnun-street;
20;OK;OKLAHOMA
- 40;zxydx-name;tml-firstname; 14680;jda-city;1974/05/29;wfjn-street;
157;DC;DISTRICT OF COLUMBIA
- 41;bfscx-name;jnl-firstname; 16920;yyg-city;1970/11/30;cgfh-street;
178;CO;COLORADO
- 42;qitur-name;yra-firstname; 15560;ijp-city;1978/01/30;fonc-street;
155;AK;ALASKA
- 43;msixi-name;ynb-firstname; 12720;ksl-city;1958/07/17;zpjw-street;
46;VI;VIRGIN ISLANDS
- 44;wzkjq-name;rgh-firstname; 19000;hkm-city;1974/08/12;yixf-street;
134;CA;CALIFORNIA
- 45;dqfmf-name;yxr-firstname; 13840;vie-city;1962/10/23;stvx-street;
39;TX;TEXAS
- 46;biluz-name;uqe-firstname; 17760;wkq-city;1962/07/27;embn-street;
183;PW;PALAU
- 47;wahfx-name;zwd-firstname; 13240;vic-city;1974/03/27;axpw-street;
131;UT;UTAH
- 48;denwt-name;bta-firstname; 17300;hhj-city;1986/12/20;orwy-street;
11;WV;WEST VIRGINIA
- 49;akdmy-name;ybz-firstname; 14560;wtx-city;1962/11/08;nwba-street;
123;MP;NORTHERN MARIANA ISLANDS
- 50;hqafg-name;nht-firstname; 16080;gfu-city;1951/01/12;spsq-street;
45;LA;LOUISIANA
- 51;zhmbl-name;lnw-firstname; 17460;hse-city;1986/12/21;scis-street;
97;GA;GEORGIA
- 52;snwnj-name;jyy-firstname; 16400;hsz-city;1966/02/15;imhl-street;
42;NC;NORTH CAROLINA
- 53;fuyla-name;mmp-firstname; 11840;hgu-city;1986/08/16;ixiz-street;
145;NC;NORTH CAROLINA
- 54;yvfqz-name;prz-firstname; 11260;wjl-city;1982/05/06;fbzd-street;
97;MO;MISSOURI
- 55;usbgq-name;vhd-firstname; 14080;dsb-city;1958/04/01;ggoc-street;
54;KS;KANSAS
- 56;yaeni-name;zpy-firstname; 19100;sen-city;1954/12/10;sbsw-street;
158;HI;HAWAII
- 57;fgxvr-name;vzi-firstname; 17520;lcf-city;1958/11/01;nbdv-street; 10;GU;GUAM
- 58;tqpbq-name;rwr-firstname; 19140;zpd-city;1978/08/23;npvb-street;
190;DC;DISTRICT OF COLUMBIA
- 59;ieigg-name;ayq-firstname; 12960;ljc-city;1962/07/05;dnjz-street;
163;FL;FLORIDA
- 60;rfvzu-name;edm-firstname; 13340;kvz-city;1954/12/08;eijd-street;
4;RI;RHODE ISLAND
- 61;pduwm-name;gqb-firstname; 14240;cyr-city;1954/07/03;ndux-street;
13;SD;SOUTH DAKOTA
- 62;yyixf-name;yzt-firstname; 18020;lwx-city;1974/01/29;iede-street;
120;NV;NEVADA
- 63;dkszq-name;ytd-firstname; 14700;zwh-city;1979/01/11;nbjz-street;
65;AS;AMERICAN SAMOA
- 64;slkzv-name;zbg-firstname; 19880;oee-city;1978/11/01;sphg-street;
119;OK;OKLAHOMA
- 65;nvxim-name;phc-firstname; 19220;vgg-city;1991/01/24;juok-street;
106;FM;FEDERATED STATES OF MICRONESIA
- 66;piyfg-name;xtn-firstname; 13760;nde-city;1954/07/22;vfrv-street; 11;NY;NEW
YORK
- 67;jnusz-name;mjw-firstname; 12640;nwb-city;1986/08/23;kcsa-street;
138;VA;VIRGINIA
- 68;jnypj-name;ioq-firstname; 17000;zqy-city;1986/01/09;croe-street;
119;PW;PALAU
- 69;uohts-name;btx-firstname; 13480;dal-city;1990/10/22;llyw-street;
150;WA;WASHINGTON
- 70;aavpj-name;pvw-firstname; 13780;lai-city;1954/09/23;nygu-street;
171;FL;FLORIDA
- 71;nbjcj-name;rsf-firstname; 12000;kjl-city;1986/06/30;ijsb-street;
123;ID;IDAHO
- 72;syjxh-name;gkq-firstname; 19960;rmd-city;1978/10/26;qmyp-street;
161;MN;MINNESOTA
- 73;vkojz-name;ryo-firstname; 14300;bmz-city;1954/09/11;gcpj-street;
71;ND;NORTH DAKOTA
- 74;pqzfw-name;kld-firstname; 16400;qvq-city;1962/09/09;dhbv-street;
92;ND;NORTH DAKOTA
- 75;owvjk-name;fez-firstname; 19740;ldb-city;1978/06/14;kabf-street;
87;VA;VIRGINIA
- 76;qsfih-name;ixe-firstname; 16860;qvr-city;1987/01/07;qean-street;
159;CO;COLORADO
- 77;slixq-name;gmb-firstname; 19980;ftt-city;1982/06/22;xinx-street;
111;VT;VERMONT
- 78;eegsa-name;xlc-firstname; 12680;byk-city;1954/04/23;beul-street;
56;MD;MARYLAND
- 79;phevp-name;ihs-firstname; 16120;adc-city;1978/04/25;voig-street; 98;NM;NEW
MEXICO
- 80;njfoe-name;tag-firstname; 16580;tnr-city;1966/12/04;dhky-street;
108;LA;LOUISIANA
- 81;bdncx-name;hcd-firstname; 11260;xcl-city;1970/07/02;jvlp-street;
49;GA;GEORGIA
- 82;ikedo-name;tks-firstname; 17460;odl-city;1958/08/25;iaaq-street; 8;GU;GUAM
- 83;iafxy-name;vur-firstname; 11480;hgt-city;1962/08/03;hmec-street;
164;TX;TEXAS
- 84;lafhf-name;ssz-firstname; 19560;wwp-city;1951/01/25;mxmq-street;
96;IN;INDIANA
- 85;okyny-name;hbu-firstname; 16800;yok-city;1978/03/28;ipjz-street;
135;NV;NEVADA
- 86;hznby-name;fwy-firstname; 13680;wbi-city;1970/07/25;mxui-street;
170;CT;CONNECTICUT
- 87;ztpoa-name;rzk-firstname; 18500;qum-city;1970/07/26;blqr-street;
152;ME;MAINE
- 88;gitxz-name;axt-firstname; 11800;fck-city;1974/01/12;tmjw-street;
189;SD;SOUTH DAKOTA
- 89;ziomm-name;mcv-firstname; 12940;iwq-city;1950/10/22;hqgj-street;
140;DC;DISTRICT OF COLUMBIA
- 90;otncg-name;tuy-firstname; 16540;ulk-city;1971/01/24;yuia-street;
166;TX;TEXAS
- 91;cnabb-name;hoq-firstname; 16300;tuw-city;1962/06/17;ujvv-street;
61;ME;MAINE
- 92;ucogf-name;ggc-firstname; 14500;fsj-city;1978/02/08;asfi-street;
53;WV;WEST VIRGINIA
- 93;lbpmf-name;sdt-firstname; 10780;ewj-city;1978/03/08;hxsp-street;
102;NV;NEVADA
- 94;tieqq-name;uyu-firstname; 17740;wea-city;1966/10/31;abpl-street;
187;MO;MISSOURI
- 95;fsgwf-name;vjd-firstname; 12460;ads-city;1970/11/29;yeou-street;
10;MA;MASSACHUSETTS
- 96;reeba-name;kzs-firstname; 13100;zhc-city;1966/07/08;abmv-street;
88;FL;FLORIDA
- 97;shybc-name;gcp-firstname; 10660;ahg-city;1950/12/15;hrqy-street;
174;KS;KANSAS
- 98;phszr-name;sst-firstname; 13080;ydd-city;1954/09/23;quqn-street;
2;RI;RHODE ISLAND
- 99;jteco-name;fxc-firstname; 19760;agr-city;1986/05/06;dzxc-street;
108;MD;MARYLAND
- 100;qvaar-name;icx-firstname; 16120;boc-city;1978/08/04;bfzf-street;
12;NM;NEW MEXICO
+id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state;isActive;registrationTimestamp;metadataJson
+ 1;jwcdf-name;fsj-firstname; 13520;oem-city;1954/02/07;amrb-street;
145;AK;ALASKA;true;2025-12-16
09:47:47.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 2;flhxu-name;tum-firstname; 17520;buo-city;1966/04/24;wfyz-street;
96;GA;GEORGIA;false;2024-07-02
15:06:40.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 3;xthfg-name;gfe-firstname; 12560;vtz-city;1990/01/11;doxx-street; 46;NJ;NEW
JERSEY;true;2024-01-01
00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 4;ulzrz-name;bnl-firstname; 11620;prz-city;1966/08/02;bxqn-street; 104;NY;NEW
YORK;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 5;oxhyr-name;onx-firstname; 15180;bpn-city;1970/11/14;pksn-street;
133;IN;INDIANA;true;2022-09-01
05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 6;fiqjz-name;sce-firstname; 16020;fnn-city;1954/09/24;wbhg-street;
35;MD;MARYLAND;false;2022-01-01
00:00:00.000;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 7;tkiat-name;xti-firstname; 12720;stt-city;1966/08/11;tvnf-street;
21;PA;PENNSYLVANIA;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 8;kljcz-name;uqd-firstname; 13340;ntt-city;1987/01/15;jyje-street;
10;PW;PALAU;false;2020-01-01
00:00:00.456;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 9;pgunz-name;hcm-firstname; 16680;gxh-city;1970/11/08;shbe-street;
184;NC;NORTH CAROLINA;true;2019-01-01
00:00:00.789;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 10;oyjha-name;uhj-firstname; 18880;uyg-city;1966/04/10;bjgw-street;
176;AR;ARKANSAS;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 11;igxbd-name;uph-firstname; 13480;ndh-city;1962/12/03;jdcd-street;
151;NH;NEW HAMPSHIRE;true;2022-09-01
05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 12;vnaov-name;wha-firstname; 13120;egm-city;1954/03/28;hpep-street;
20;CA;CALIFORNIA;false;2022-01-01
00:00:00.000;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 13;dauuz-name;hwg-firstname; 13740;khn-city;1958/05/15;etqx-street;
5;OK;OKLAHOMA;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 14;gkuuo-name;kkb-firstname; 13560;xdt-city;1962/04/07;sdoj-street;
35;MT;MONTANA;false;2020-01-01
00:00:00.456;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 15;wdhze-name;jjk-firstname; 16900;due-city;1970/07/17;pmmu-street;
174;AS;AMERICAN SAMOA;true;2024-01-01
00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 16;ncayz-name;ynb-firstname; 15720;lxj-city;1974/04/27;mdtb-street;
109;MA;MASSACHUSETTS;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 17;rdjin-name;hhu-firstname; 14480;lpc-city;1958/11/16;wxik-street;
145;KY;KENTUCKY;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 18;nxzij-name;bdl-firstname; 10740;avx-city;1958/02/20;nybz-street;
138;WI;WISCONSIN;false;2022-01-01
00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 19;xgrzc-name;dxw-firstname; 18900;vpq-city;1990/11/16;wzjh-street;
58;ME;MAINE;true;2021-01-01
00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 20;ehgrn-name;vbe-firstname; 17500;cik-city;1978/05/21;ucnw-street;
135;MD;MARYLAND;false;2020-01-01
00:00:00.456;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 21;gctjx-name;upx-firstname; 11960;yqr-city;1958/03/03;rlko-street;
141;TN;TENNESSEE;true;2019-01-01
00:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 22;ptzmg-name;hva-firstname; 15740;gux-city;1978/05/04;pugy-street;
122;VI;VIRGIN ISLANDS;false;2023-03-31
12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 23;eyeti-name;gnw-firstname; 17420;eko-city;1962/10/26;ylph-street;
61;NC;NORTH CAROLINA;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 24;wccwo-name;zpj-firstname; 16600;uim-city;1962/09/29;ygih-street;
26;WA;WASHINGTON;false;2022-01-01
00:00:00.000;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 25;bwkoe-name;ayl-firstname; 18660;rtw-city;1978/07/16;mzww-street;
179;CA;CALIFORNIA;true;2021-01-01
00:00:00.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 26;rezku-name;zio-firstname; 19080;nvt-city;1982/07/14;wwkd-street;
91;CA;CALIFORNIA;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 27;mjlsk-name;ecx-firstname; 10800;yxu-city;1950/12/11;vttb-street;
195;MO;MISSOURI;true;2022-09-01
05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 28;wdjsi-name;aoq-firstname; 13660;smo-city;1954/02/01;kako-street;
7;NV;NEVADA;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 29;mwfnd-name;nyb-firstname; 19760;bbu-city;1986/09/23;apdi-street;
91;MS;MISSISSIPPI;true;2021-01-01
00:00:00.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 30;vtuoz-name;jhh-firstname; 17620;vad-city;1982/05/05;kzup-street;
79;GA;GEORGIA;false;2020-01-01
00:00:00.456;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 31;rhhxk-name;ndr-firstname; 16760;fub-city;1978/11/12;regd-street;
55;OK;OKLAHOMA;true;2019-01-01
00:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 32;lpstk-name;mqz-firstname; 18940;tnr-city;1982/09/16;cdhf-street;
4;SD;SOUTH DAKOTA;false;2023-03-31
12:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 33;ldhyr-name;yts-firstname; 12000;auk-city;1986/11/14;abph-street;
147;IN;INDIANA;true;2022-09-01
05:59:59.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 34;cjdml-name;iti-firstname; 16900;wkq-city;1970/06/05;npow-street; 96;NH;NEW
HAMPSHIRE;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 35;cpenz-name;sbi-firstname; 16380;ssl-city;1962/08/19;kilz-street;
44;MS;MISSISSIPPI;true;2021-01-01
00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 36;rxtbg-name;anr-firstname; 14720;bqc-city;1958/08/10;pudg-street;
140;NV;NEVADA;false;2020-01-01
00:00:00.456;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 37;udblf-name;raa-firstname; 11500;wli-city;1978/12/13;xomd-street;
41;PW;PALAU;true;2019-01-01
00:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 38;vvyce-name;gep-firstname; 13740;gtd-city;1982/05/23;kwbv-street;
123;undefined;undefined;false;2023-03-31
12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 39;kwfnz-name;ucu-firstname; 10580;sns-city;1978/08/18;nnun-street;
20;OK;OKLAHOMA;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 40;zxydx-name;tml-firstname; 14680;jda-city;1974/05/29;wfjn-street;
157;DC;DISTRICT OF COLUMBIA;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 41;bfscx-name;jnl-firstname; 16920;yyg-city;1970/11/30;cgfh-street;
178;CO;COLORADO;true;2021-01-01
00:00:00.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 42;qitur-name;yra-firstname; 15560;ijp-city;1978/01/30;fonc-street;
155;AK;ALASKA;false;2020-01-01
00:00:00.456;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 43;msixi-name;ynb-firstname; 12720;ksl-city;1958/07/17;zpjw-street;
46;VI;VIRGIN ISLANDS;true;2024-01-01
00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 44;wzkjq-name;rgh-firstname; 19000;hkm-city;1974/08/12;yixf-street;
134;CA;CALIFORNIA;false;2023-03-31
12:00:00.789;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 45;dqfmf-name;yxr-firstname; 13840;vie-city;1962/10/23;stvx-street;
39;TX;TEXAS;true;2022-09-01
05:59:59.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 46;biluz-name;uqe-firstname; 17760;wkq-city;1962/07/27;embn-street;
183;PW;PALAU;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 47;wahfx-name;zwd-firstname; 13240;vic-city;1974/03/27;axpw-street;
131;UT;UTAH;true;2021-01-01
00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 48;denwt-name;bta-firstname; 17300;hhj-city;1986/12/20;orwy-street;
11;WV;WEST VIRGINIA;false;2020-01-01
00:00:00.456;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 49;akdmy-name;ybz-firstname; 14560;wtx-city;1962/11/08;nwba-street;
123;MP;NORTHERN MARIANA ISLANDS;true;2019-01-01
00:00:00.789;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 50;hqafg-name;nht-firstname; 16080;gfu-city;1951/01/12;spsq-street;
45;LA;LOUISIANA;false;2023-03-31
12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 51;zhmbl-name;lnw-firstname; 17460;hse-city;1986/12/21;scis-street;
97;GA;GEORGIA;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 52;snwnj-name;jyy-firstname; 16400;hsz-city;1966/02/15;imhl-street;
42;NC;NORTH CAROLINA;false;2022-01-01
00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 53;fuyla-name;mmp-firstname; 11840;hgu-city;1986/08/16;ixiz-street;
145;NC;NORTH CAROLINA;true;2021-01-01
00:00:00.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 54;yvfqz-name;prz-firstname; 11260;wjl-city;1982/05/06;fbzd-street;
97;MO;MISSOURI;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 55;usbgq-name;vhd-firstname; 14080;dsb-city;1958/04/01;ggoc-street;
54;KS;KANSAS;true;2022-09-01
05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 56;yaeni-name;zpy-firstname; 19100;sen-city;1954/12/10;sbsw-street;
158;HI;HAWAII;false;2023-03-31
12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 57;fgxvr-name;vzi-firstname; 17520;lcf-city;1958/11/01;nbdv-street;
10;GU;GUAM;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 58;tqpbq-name;rwr-firstname; 19140;zpd-city;1978/08/23;npvb-street;
190;DC;DISTRICT OF COLUMBIA;false;2022-01-01
00:00:00.000;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 59;ieigg-name;ayq-firstname; 12960;ljc-city;1962/07/05;dnjz-street;
163;FL;FLORIDA;true;2021-01-01
00:00:00.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 60;rfvzu-name;edm-firstname; 13340;kvz-city;1954/12/08;eijd-street;
4;RI;RHODE ISLAND;false;2020-01-01
00:00:00.456;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 61;pduwm-name;gqb-firstname; 14240;cyr-city;1954/07/03;ndux-street;
13;SD;SOUTH DAKOTA;true;2019-01-01
00:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 62;yyixf-name;yzt-firstname; 18020;lwx-city;1974/01/29;iede-street;
120;NV;NEVADA;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 63;dkszq-name;ytd-firstname; 14700;zwh-city;1979/01/11;nbjz-street;
65;AS;AMERICAN SAMOA;true;2022-09-01
05:59:59.123;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 64;slkzv-name;zbg-firstname; 19880;oee-city;1978/11/01;sphg-street;
119;OK;OKLAHOMA;false;2022-01-01
00:00:00.000;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 65;nvxim-name;phc-firstname; 19220;vgg-city;1991/01/24;juok-street;
106;FM;FEDERATED STATES OF MICRONESIA;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 66;piyfg-name;xtn-firstname; 13760;nde-city;1954/07/22;vfrv-street; 11;NY;NEW
YORK;false;2020-01-01
00:00:00.456;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 67;jnusz-name;mjw-firstname; 12640;nwb-city;1986/08/23;kcsa-street;
138;VA;VIRGINIA;true;2024-01-01
00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 68;jnypj-name;ioq-firstname; 17000;zqy-city;1986/01/09;croe-street;
119;PW;PALAU;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 69;uohts-name;btx-firstname; 13480;dal-city;1990/10/22;llyw-street;
150;WA;WASHINGTON;true;2022-09-01
05:59:59.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 70;aavpj-name;pvw-firstname; 13780;lai-city;1954/09/23;nygu-street;
171;FL;FLORIDA;false;2022-01-01
00:00:00.000;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 71;nbjcj-name;rsf-firstname; 12000;kjl-city;1986/06/30;ijsb-street;
123;ID;IDAHO;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 72;syjxh-name;gkq-firstname; 19960;rmd-city;1978/10/26;qmyp-street;
161;MN;MINNESOTA;false;2020-01-01
00:00:00.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 73;vkojz-name;ryo-firstname; 14300;bmz-city;1954/09/11;gcpj-street;
71;ND;NORTH DAKOTA;true;2019-01-01
00:00:00.789;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 74;pqzfw-name;kld-firstname; 16400;qvq-city;1962/09/09;dhbv-street;
92;ND;NORTH DAKOTA;false;2023-03-31
12:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 75;owvjk-name;fez-firstname; 19740;ldb-city;1978/06/14;kabf-street;
87;VA;VIRGINIA;true;2022-09-01
05:59:59.123;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 76;qsfih-name;ixe-firstname; 16860;qvr-city;1987/01/07;qean-street;
159;CO;COLORADO;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 77;slixq-name;gmb-firstname; 19980;ftt-city;1982/06/22;xinx-street;
111;VT;VERMONT;true;2021-01-01
00:00:00.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 78;eegsa-name;xlc-firstname; 12680;byk-city;1954/04/23;beul-street;
56;MD;MARYLAND;false;2023-03-31
12:00:00.789;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 79;phevp-name;ihs-firstname; 16120;adc-city;1978/04/25;voig-street; 98;NM;NEW
MEXICO;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 80;njfoe-name;tag-firstname; 16580;tnr-city;1966/12/04;dhky-street;
108;LA;LOUISIANA;false;2023-03-31
12:00:00.789;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 81;bdncx-name;hcd-firstname; 11260;xcl-city;1970/07/02;jvlp-street;
49;GA;GEORGIA;true;2022-09-01
05:59:59.123;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 82;ikedo-name;tks-firstname; 17460;odl-city;1958/08/25;iaaq-street;
8;GU;GUAM;false;2022-01-01
00:00:00.000;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 83;iafxy-name;vur-firstname; 11480;hgt-city;1962/08/03;hmec-street;
164;TX;TEXAS;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 84;lafhf-name;ssz-firstname; 19560;wwp-city;1951/01/25;mxmq-street;
96;IN;INDIANA;false;2020-01-01
00:00:00.456;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 85;okyny-name;hbu-firstname; 16800;yok-city;1978/03/28;ipjz-street;
135;NV;NEVADA;true;2019-01-01
00:00:00.789;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 86;hznby-name;fwy-firstname; 13680;wbi-city;1970/07/25;mxui-street;
170;CT;CONNECTICUT;false;2023-03-31
12:00:00.789;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 87;ztpoa-name;rzk-firstname; 18500;qum-city;1970/07/26;blqr-street;
152;ME;MAINE;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 88;gitxz-name;axt-firstname; 11800;fck-city;1974/01/12;tmjw-street;
189;SD;SOUTH DAKOTA;false;2022-01-01
00:00:00.000;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 89;ziomm-name;mcv-firstname; 12940;iwq-city;1950/10/22;hqgj-street;
140;DC;DISTRICT OF COLUMBIA;true;2021-01-01
00:00:00.123;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
+ 90;otncg-name;tuy-firstname; 16540;ulk-city;1971/01/24;yuia-street;
166;TX;TEXAS;false;2020-01-01
00:00:00.456;"{""ip"":""192.168.0.12"",""session_id"":""z5a6b7c8"",""duration_sec"":600}"
+ 91;cnabb-name;hoq-firstname; 16300;tuw-city;1962/06/17;ujvv-street;
61;ME;MAINE;true;2024-01-01
00:00:00.123;"{""ip"":""10.0.3.4"",""session_id"":""d9e0f1g2"",""duration_sec"":180}"
+ 92;ucogf-name;ggc-firstname; 14500;fsj-city;1978/02/08;asfi-street;
53;WV;WEST VIRGINIA;false;2023-03-31
12:00:00.789;"{""ip"":""172.16.8.9"",""session_id"":""h3i4j5k6"",""duration_sec"":75}"
+ 93;lbpmf-name;sdt-firstname; 10780;ewj-city;1978/03/08;hxsp-street;
102;NV;NEVADA;true;2022-09-01
05:59:59.123;"{""ip"":""192.168.2.34"",""session_id"":""l7m8n9o0"",""duration_sec"":330}"
+ 94;tieqq-name;uyu-firstname; 17740;wea-city;1966/10/31;abpl-street;
187;MO;MISSOURI;false;2022-01-01
00:00:00.000;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 95;fsgwf-name;vjd-firstname; 12460;ads-city;1970/11/29;yeou-street;
10;MA;MASSACHUSETTS;true;2021-01-01
00:00:00.123;"{""ip"":""192.168.1.10"",""session_id"":""a1b2c3d4"",""duration_sec"":120}"
+ 96;reeba-name;kzs-firstname; 13100;zhc-city;1966/07/08;abmv-street;
88;FL;FLORIDA;false;2020-01-01
00:00:00.456;"{""ip"":""10.0.0.5"",""session_id"":""f5g6h7i8"",""duration_sec"":300}"
+ 97;shybc-name;gcp-firstname; 10660;ahg-city;1950/12/15;hrqy-street;
174;KS;KANSAS;true;2019-01-01
00:00:00.789;"{""ip"":""172.16.0.23"",""session_id"":""j9k0l1m2"",""duration_sec"":45}"
+ 98;phszr-name;sst-firstname; 13080;ydd-city;1954/09/23;quqn-street;
2;RI;RHODE ISLAND;false;2023-03-31
12:00:00.789;"{""ip"":""192.168.1.45"",""session_id"":""n3o4p5q6"",""duration_sec"":420}"
+ 99;jteco-name;fxc-firstname; 19760;agr-city;1986/05/06;dzxc-street;
108;MD;MARYLAND;true;2022-09-01
05:59:59.123;"{""ip"":""10.1.2.3"",""session_id"":""r7s8t9u0"",""duration_sec"":215}"
+ 100;qvaar-name;icx-firstname; 16120;boc-city;1978/08/04;bfzf-street;
12;NM;NEW MEXICO;false;2022-01-01
00:00:00.000;"{""ip"":""172.16.5.6"",""session_id"":""v1w2x3y4"",""duration_sec"":90}"
diff --git
a/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
b/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
index c7b7ea52dd..753123b113 100644
--- a/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
+++ b/integration-tests/transforms/metadata/dataset/golden-parquet-input.json
@@ -8,80 +8,104 @@
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "id"
+ "field_name": "id",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "name"
+ "field_name": "name",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "firstname"
+ "field_name": "firstname",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "zip"
+ "field_name": "zip",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "city"
+ "field_name": "city",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 3,
"field_precision": -1,
- "field_format": "yyyy/MM/dd",
- "field_name": "birthdate"
+ "field_name": "birthdate",
+ "field_format": "yyyy/MM/dd"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "street"
+ "field_name": "street",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 5,
"field_precision": 0,
- "field_format": "####0;-####0",
- "field_name": "housenr"
+ "field_name": "housenr",
+ "field_format": "####0;-####0"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "stateCode"
+ "field_name": "stateCode",
+ "field_format": ""
},
{
"field_comment": "",
"field_length": -1,
"field_type": 2,
"field_precision": -1,
- "field_format": "",
- "field_name": "state"
+ "field_name": "state",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "isActive",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 9,
+ "field_precision": -1,
+ "field_name": "registrationTimestamp",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 11,
+ "field_precision": -1,
+ "field_name": "metadataJson",
+ "field_format": ""
}
],
"folder_name": ""
diff --git a/integration-tests/transforms/metadata/unit-test/0029-parquet-input
UNIT.json b/integration-tests/transforms/metadata/unit-test/0029-parquet-input
UNIT.json
index f90e98b5bd..570cefbbae 100644
--- a/integration-tests/transforms/metadata/unit-test/0029-parquet-input
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0029-parquet-input
UNIT.json
@@ -1,7 +1,10 @@
{
- "variableValues": [],
"database_replacements": [],
"autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
"basePath": "",
"golden_data_sets": [
{
@@ -45,20 +48,29 @@
{
"transform_field": "zip",
"data_set_field": "zip"
+ },
+ {
+ "transform_field": "isActive",
+ "data_set_field": "isActive"
+ },
+ {
+ "transform_field": "registrationTimestamp",
+ "data_set_field": "registrationTimestamp"
+ },
+ {
+ "transform_field": "metadataJson",
+ "data_set_field": "metadataJson"
}
],
"field_order": [
"id"
],
- "transform_name": "Verify",
- "data_set_name": "golden-parquet-input"
+ "data_set_name": "golden-parquet-input",
+ "transform_name": "Verify"
}
],
"input_data_sets": [],
"name": "0029-parquet-input UNIT",
- "description": "",
"trans_test_tweaks": [],
- "persist_filename": "",
- "pipeline_filename": "./0029-parquet-input.hpl",
- "test_type": "UNIT_TEST"
+ "pipeline_filename": "./0029-parquet-input.hpl"
}
\ No newline at end of file
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
index 820b0ba356..274fbbdbfd 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInputMeta.java
@@ -42,6 +42,13 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.JsonLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
@@ -148,21 +155,38 @@ public class ParquetInputMeta extends
BaseTransformMeta<ParquetInput, ParquetInp
}
PrimitiveType primitiveType = column.getPrimitiveType();
int hopType = IValueMeta.TYPE_STRING;
- switch (primitiveType.getPrimitiveTypeName()) {
- case INT32, INT64:
- hopType = IValueMeta.TYPE_INTEGER;
- break;
- case INT96:
+ LogicalTypeAnnotation logicalType =
primitiveType.getLogicalTypeAnnotation();
+ if (logicalType != null) {
+ if ((logicalType instanceof TimestampLogicalTypeAnnotation)
+ || (logicalType instanceof TimeLogicalTypeAnnotation)) {
hopType = IValueMeta.TYPE_TIMESTAMP;
- break;
- case FLOAT, DOUBLE:
- hopType = IValueMeta.TYPE_NUMBER;
- break;
- case BOOLEAN:
- hopType = IValueMeta.TYPE_BOOLEAN;
- break;
- default:
- break;
+ } else if (logicalType instanceof DateLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_DATE;
+ } else if (logicalType instanceof JsonLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_JSON;
+ } else if (logicalType instanceof DecimalLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_BIGNUMBER;
+ } else if (logicalType instanceof IntLogicalTypeAnnotation) {
+ hopType = IValueMeta.TYPE_INTEGER;
+ }
+ } else {
+ switch (primitiveType.getPrimitiveTypeName()) {
+ case INT32, INT64:
+ hopType = IValueMeta.TYPE_INTEGER;
+ break;
+ case INT96:
+ hopType = IValueMeta.TYPE_BINARY;
+ break;
+ case FLOAT, DOUBLE:
+ hopType = IValueMeta.TYPE_NUMBER;
+ break;
+ case BOOLEAN:
+ hopType = IValueMeta.TYPE_BOOLEAN;
+ break;
+ case BINARY:
+ hopType = IValueMeta.TYPE_BINARY;
+ break;
+ }
}
IValueMeta valueMeta = ValueMetaFactory.createValueMeta(sourceField,
hopType, -1, -1);
rowMeta.addValueMeta(valueMeta);
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
index 95bda47002..5ff78fb73c 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetRowConverter.java
@@ -55,7 +55,10 @@ public class ParquetRowConverter extends GroupConverter {
}
}
- return new ParquetValueConverter(group, rowIndex);
+ return new ParquetValueConverter(
+ group,
+ rowIndex,
+
messageType.getColumns().get(schemaIndex).getPrimitiveType().getLogicalTypeAnnotation());
}
@Override
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
index 6328809ac0..21c4c12380 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetValueConverter.java
@@ -17,27 +17,38 @@
package org.apache.hop.parquet.transforms.input;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.ZoneId;
import java.util.Date;
+import java.util.TimeZone;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.row.IValueMeta;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
public class ParquetValueConverter extends PrimitiveConverter {
private final RowMetaAndData group;
private final IValueMeta valueMeta;
private final int rowIndex;
+ private final LogicalTypeAnnotation logicalTypeAnnotation;
- public ParquetValueConverter(RowMetaAndData group, int rowIndex) {
+ public ParquetValueConverter(
+ RowMetaAndData group, int rowIndex, LogicalTypeAnnotation
logicalTypeAnnotation) {
this.group = group;
this.valueMeta = group.getValueMeta(rowIndex);
this.rowIndex = rowIndex;
+ this.logicalTypeAnnotation = logicalTypeAnnotation;
}
@Override
@@ -57,9 +68,27 @@ public class ParquetValueConverter extends
PrimitiveConverter {
try {
object = new BigDecimal(value.toStringUsingUTF8());
} catch (NumberFormatException e) {
- object = binaryToDecimal(value, valueMeta.getLength(),
valueMeta.getPrecision());
+ if ((this.logicalTypeAnnotation instanceof
DecimalLogicalTypeAnnotation)) {
+ object =
+ binaryToDecimal(
+ value,
+ ((DecimalLogicalTypeAnnotation)
this.logicalTypeAnnotation).getPrecision(),
+ ((DecimalLogicalTypeAnnotation)
this.logicalTypeAnnotation).getScale());
+ } else {
+ object = binaryToDecimal(value, valueMeta.getLength(),
valueMeta.getPrecision());
+ }
}
break;
+ case IValueMeta.TYPE_JSON:
+ JsonNode node = null;
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ node = mapper.readTree(value.toStringUsingUTF8());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to parse an json value : " +
e.getMessage());
+ }
+ object = node;
+ break;
case IValueMeta.TYPE_TIMESTAMP:
if (value.length() == 12) {
// This is a binary form of an int96 (12-byte) Timestamp with
nanosecond precision.
@@ -107,10 +136,23 @@ public class ParquetValueConverter extends
PrimitiveConverter {
object = Long.toString(value);
break;
case IValueMeta.TYPE_DATE:
- object = new Date(value);
+ if (this.logicalTypeAnnotation instanceof DateLogicalTypeAnnotation) {
+ LocalDate date = LocalDate.ofEpochDay(value);
+ Date utilDate =
Date.from(date.atStartOfDay(ZoneId.systemDefault()).toInstant());
+ object = utilDate;
+ } else {
+ object = convertToTimestamp(value, this.logicalTypeAnnotation);
+ }
break;
case IValueMeta.TYPE_BIGNUMBER:
object = new BigDecimal(value);
+ if ((this.logicalTypeAnnotation instanceof
DecimalLogicalTypeAnnotation)) {
+ int scale = ((DecimalLogicalTypeAnnotation)
this.logicalTypeAnnotation).getScale();
+ object = (new BigDecimal(((BigDecimal)
object).doubleValue())).movePointLeft(scale);
+ }
+ break;
+ case IValueMeta.TYPE_TIMESTAMP:
+ object = convertToTimestamp(value, this.logicalTypeAnnotation);
break;
default:
throw new RuntimeException(
@@ -175,6 +217,60 @@ public class ParquetValueConverter extends
PrimitiveConverter {
addLong(value);
}
+ /**
+ * Converts a numeric epoch timestamp (in millis, micros, or nanos) into a
java.sql.Timestamp
+ * according to the logical type's time unit.
+ *
+ * @param value
+ * @param logicalTypeAnnotation
+ */
+ private Timestamp convertToTimestamp(long value, LogicalTypeAnnotation
logicalTypeAnnotation) {
+ LogicalTypeAnnotation.TimeUnit unit = null;
+ boolean isUTC = true;
+ if (logicalTypeAnnotation instanceof
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
+ unit =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation)
logicalTypeAnnotation).getUnit();
+ isUTC =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation)
this.logicalTypeAnnotation)
+ .isAdjustedToUTC();
+ } else if (logicalTypeAnnotation instanceof
LogicalTypeAnnotation.TimeLogicalTypeAnnotation) {
+ unit = ((LogicalTypeAnnotation.TimeLogicalTypeAnnotation)
logicalTypeAnnotation).getUnit();
+ isUTC = false;
+ }
+ if (unit == null) {
+ throw new RuntimeException(
+ "Unknown timestamp unit for the logical type: " +
logicalTypeAnnotation);
+ }
+ long epochMillis;
+ // Convert the timestamp to milliseconds since the Epoch based on the
original unit
+ switch (unit) {
+ case MILLIS:
+ epochMillis = value;
+ break;
+ case MICROS:
+ epochMillis = value / 1_000L;
+ break;
+ case NANOS:
+ epochMillis = value / 1_000_000L;
+ break;
+ default:
+ throw new RuntimeException("Unknown timestamp unit: " + unit);
+ }
+ Timestamp ts = new Timestamp(epochMillis);
+ // If the timestamp is local time, adjust it to UTC
+ if (!isUTC) {
+ int offset = TimeZone.getDefault().getOffset(epochMillis);
+ ts.setTime(ts.getTime() - offset);
+ }
+ // Adjust nanosecond precision for microsecond or nanosecond timestamps
+ if (unit == LogicalTypeAnnotation.TimeUnit.MICROS) {
+ ts.setNanos((int) ((value % 1_000_000L) * 1_000L));
+ } else if (unit == LogicalTypeAnnotation.TimeUnit.NANOS) {
+ ts.setNanos((int) (value % 1_000_000_000L));
+ }
+ return ts;
+ }
+
/**
* Source code from:
*
diff --git
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
index eb0232e378..c42178c7bd 100644
---
a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
+++
b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
@@ -17,6 +17,7 @@
package org.apache.hop.parquet.transforms.output;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -29,7 +30,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hop.core.Const;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
@@ -114,7 +117,40 @@ public class ParquetOutput extends
BaseTransform<ParquetOutputMeta, ParquetOutpu
// Write the row, handled by class ParquetWriteSupport
//
try {
- data.writer.write(new RowMetaAndData(getInputRowMeta(), row));
+ IRowMeta parquetRowMeta = getInputRowMeta().clone();
+
+ // convert date/timestamp => long
+ for (int i = 0; i < data.sourceFieldIndexes.size(); i++) {
+ int idx = data.sourceFieldIndexes.get(i);
+ IValueMeta valueMeta = parquetRowMeta.getValueMeta(idx);
+ if (valueMeta.getType() == IValueMeta.TYPE_TIMESTAMP) {
+ // Update of type meta
+ IValueMeta longMeta = new ValueMetaInteger(valueMeta.getName());
+ longMeta.setConversionMask(valueMeta.getConversionMask());
+ longMeta.setLength(valueMeta.getLength(), valueMeta.getPrecision());
+ parquetRowMeta.setValueMeta(idx, longMeta);
+ }
+ }
+
+ // Clone Rows and convert Date & Timetims to Long
+ Object[] parquetRow = row.clone();
+ for (int i = 0; i < data.sourceFieldIndexes.size(); i++) {
+ int idx = data.sourceFieldIndexes.get(i);
+ Object value = parquetRow[idx];
+ if (getInputRowMeta().getValueMeta(idx).getType() ==
IValueMeta.TYPE_TIMESTAMP) {
+ if (value instanceof java.util.Date) {
+ parquetRow[idx] = ((java.util.Date) value).getTime();
+ } else if (value instanceof byte[]) {
+ String dateStr = new String((byte[]) value,
StandardCharsets.UTF_8);
+ SimpleDateFormat sdf =
+ new
SimpleDateFormat(parquetRowMeta.getValueMeta(idx).getFormatMask());
+ Date date = sdf.parse(dateStr);
+ parquetRow[idx] = date.getTime();
+ }
+ }
+ }
+
+ data.writer.write(new RowMetaAndData(parquetRowMeta, parquetRow));
incrementLinesOutput();
data.splitRowCount++;
} catch (Exception e) {
@@ -162,9 +198,10 @@ public class ParquetOutput extends
BaseTransform<ParquetOutputMeta, ParquetOutpu
// Match these data types with class ParquetWriteSupport
//
+ Schema timestampMilliType;
switch (valueMeta.getType()) {
- case IValueMeta.TYPE_DATE:
- Schema timestampMilliType =
+ case IValueMeta.TYPE_TIMESTAMP, IValueMeta.TYPE_DATE:
+ timestampMilliType =
LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
fieldAssembler =
fieldAssembler
@@ -194,6 +231,12 @@ public class ParquetOutput extends
BaseTransform<ParquetOutputMeta, ParquetOutpu
case IValueMeta.TYPE_BINARY:
fieldAssembler = fieldBuilder.bytesType().noDefault();
break;
+ case IValueMeta.TYPE_JSON:
+ fieldAssembler = fieldBuilder.stringType().noDefault();
+ break;
+ case IValueMeta.TYPE_UUID:
+ fieldAssembler = fieldBuilder.stringType().noDefault();
+ break;
default:
throw new HopException(
"Writing Hop data type '"
