See
<https://builds.apache.org/job/beam_PostCommit_Python35/61/display/redirect?page=changes>
Changes:
[kenn] [BEAM-7755] adding support for nested rows and arrays in BigQuery to
------------------------------------------
[...truncated 113.02 KB...]
},
"output_name": "out",
"user_name": "To Entity.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s2"
},
"serialized_fn":
"eNq1Vflz20QUXjlHW7UpPYC2lMNAAaXUVhzHOaCEw72CiRvUQJcjiJW09srR9aRV0szEMzAdZ/ojfzJv16EhQDPMMIzPffve996+79unXyYsn2XMF9z1OIvrMmdJ0UvzuKj7ac7NNosi5kX8cc6yjOd30nuJCWT2VzCGULHoKUJIlqc+LwqY8IMwiuqu+jZdP+dMcrdXJr4MUwyatI7tRykLXLmXcROm6GmEaacB38Q1TI/glAOnrY7RIfipdC63Z54Rsk/IU4P0DfIIznRHYM5SA6OewNkRnKMx/rVFGnN7wJPtMCn++K0VEdvh9m6abxd4TG6rU7obaSHbaRyH0t3YkyJNmi27yH27CLYLO9MW+09dsY+6Yquu1LM9mNFF345Y7AVsFc6vX5hqE3iJVtDaS+DCCC7OSrjkwOVjx+5z6TIpcxNe1gBeGUYS64RXdC9xW+3CqwdwxYGrx0LDOEtz6cZpUEbYtWv0OgacwB28NoLrDryu87gI4kvXhTcO4E0H3qJn0Rizbe7yRIZyD6rdf6LP57iAt8UUNdF/nWNnAk3SOyN414EbYtISyFQFmZrsTHXabVdqpgYGkRUymCD7BhlMksBQ3OF7MEWe4ikrJKhot2k0GsMKkafIfkU5DDFkggxOk2CCyDMqHI2h/jZw4xG8Z9GqUouurzouvtrL07jaD3d4Ui1kHib9OrxPp9CtlL3aMnxAp5VOWY7uMBatnyZSrZCjmxadQEuH78GHOshF4QRwS/sJVogo9KBGJ3FVCNaAukbjiY+CBZueUV78SRD2eSFhTjfWHedytxGzod3vjrs8T88rGBQB3hq8TzLkBTR1fiwcFiS0rHEmHvVgUczqLQWzdJhVwyxLWKG//b+aD1O772d2wCQrJMrJ3mkkfPdo7e7mISoE4bIw41GY6Gvxkah21tomqdw0zuHrmgEfz2KPbzvwibgqrtHVv2g2TOuYpf4cta6z1E/IAqsj+NSBzwQq+XMHvhDVbncIbQl3HLjr/zdsc0zT4bAz4R4OuvtDeGCJxr9BVuOsMFFIJqxh5JdD6DwfkQPuS/iKXn5+/ppCqkkUDeoV1oXV1VJKWMw1a9Dt0hktWylcHvEYuS/gIb2CNhWlRDq3NL84P9dcaK0szC0uzcMGvYG7K41mq9dsLNYaHue1hZa/Ulvx5pq1XstvLi/Mzbc4X4GvD8ApPXFLbJTeATxyYHME3zjw7QgeD4FaYxmrA7kiVIm/O/6YwA1trwccZw3DJhTm2kM1Gh4oswnfYwN+QGJ+HEOFSVZKjVfAVpeeQ1NayiPbT93yAFxvnJbl/SLjvotz9GexWUpgDnj6JiiTL7y/VRcIlY9jvp4ltrpCw/W90tsCMYRwCwYnPuUeYy/TXeTBhG3EiYYQW/SSanSIbEgWZ66fxh4qJIekY9CLikXfL+MyYurppoYyhxR3dGFh4Qa8x8pIQvZMs4pDqd/nOVYPLyrk0MW8M47cPFxCjgUVeobv6ioRQ74IY+xh3o9Sj0XjQyEVJSLslJ6E3frv0EiOsA==",
"user_name": "To Entity"
}
},
{
"kind": "ParallelDo",
"name": "s4",
"properties": {
"display_data": [
{
"key": "project",
"namespace":
"apache_beam.io.gcp.datastore.v1new.datastoreio._DatastoreWriteFn",
"type": "STRING",
"value": "apache-beam-testing"
},
{
"key": "mutation",
"namespace":
"apache_beam.io.gcp.datastore.v1new.datastoreio._DatastoreWriteFn",
"type": "STRING",
"value": "Write (upsert)"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "_DatastoreWriteFn",
"type": "STRING",
"value":
"apache_beam.io.gcp.datastore.v1new.datastoreio._DatastoreWriteFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "Write to Datastore/Write Batch to Datastore.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s3"
},
"serialized_fn":
"eNqtmXlcTe32wHdlK6dCGSKiZCgSGZM5JRYVFUI5TqdTHZ3O6exzTgqRKWxTennNveKNMs/zPJTpDZF5zlC4mSJkuOs5Wd7u/Xx+935+n8/1h/U9az977WcNz3qe3U4xc5FHKlUqdyn7XyKNVuilMr1ekGi5UAuO4yIMSpVeqdZpTULN8SdeZle1pqlasyBtNbksXiaPUUgjFLI4d6XGPVoe7x4p08t0eo2gcE/wUCvG//1bqZGMEJR6RYjGh1QSLR9qg2alvzTGEf3V2uqpWvMgrYXrVG2NZK3EJdSSjdInxSukMUq1Xqe1/JdHswtGvXukQq4RZGhJJxkYGILqAUwt0VqhJeuAZG3NSlNKdbxBb7Sn09YKCLVClcag/1tXO8CQqrWJCJUYHxsjaPR6lULQ2v43h91lkbJ4vTJB8fdNkr4/VSG/NNo6OJu6ydp6LqHW7AEylUoqKLQGhQ49q/9fn2HQK1USf02CUh0dbIiTaO3QWoNkbUOXyulGGOSxmMU4ndbev9gstCbTyQ2CoFDrpUp1pCJR22hQZXJ/DtVpG4drHVzCtU1cBnGDOEW4timBI4ETQTMCZ4LmBC0IWhK0InAhcCVoTdCGwI2gLYE7QTuC9gQeBB0IOhJ0IuhM0IWgK4EnQTcCL4LuBD0IehL0IuhN0IegL4E3QT8CHwJfgv4EfgQDCAYSAMEggsEE/gQBBIEEQwiGEgQRBBOEEAwjGE4wgiCUYCTBKILRBGEE4QRjCKQEYwlkBBEEcoJIAgVBFEE0QQyBkmAcQSyBiiCOQE2gIYgn0BIIBDoCPYGBIIFgPEEiQRLBBIKJBJMIkgkmE0whSCGYSjCNYDrBDIKZBKkEswhmE8whEAnmEswjmE+wgGAhQRrBIoJ0gt8IFhMsqQSFsX1K1Ya4X/3k90GJoXZV+06cTqpTquUKqSJeI4/RLh3EGSJC67IROoNcrtDpogxVWuCyGNbSlidrV7jEsB4Wgw0rBnvUSuxRq+jpqwkyCP4gWEOQSbCWYB3BnwRZBOsJNhBkE+QQbCTYRLCZYAvBVoJtBNsJdhDsJNhFsJtgD8Fegn0E+wkOEBwkOERwmOAIwVGCYwTHCU4QnCQ4RXCa4AxBLkEewVmCcwTnCS4QXCT4iyCf4BLBZYIrBAUEVwmuERQSXCe4QXCT4BbBbYI7BHcJ7hHcJ3hA8JDgEcFjgiKCJwRPCZ4RPCcoJigheEHwkuAVwT8ISgleE7wheEvwjuA9QRnBB4KPBOUEnwg+E3whqCD4SvCN4DvBj58gcCYcR2xShU2rsFkVrlaF+SpcvQqbV2GLKlyjCkuqsGUVtqrC1lW4ZhWuVYVrV2GbKmxbhev8YkUMtrOYypbFzsBSQaaO1MQJdXGEvJIlQZVCqIc61yChPopBZi4ij8NTOLgesG2wyHtcV5Zx8Hl1eAjYaUq0Ip+3ZvxyDhKa9YuF6NVTwkQenhZc4EDedcgoKO/Vp73ILwvLyeREPjIsfzoHspDxUlANOuEBPcz4KXDY1Nxe5N++SdzKQfLJeHN4+OR8MLx6ftwBbli2mCzy01XxHzkYaRYwCAov21mKfMXxmsVoLX9o3jkOuh555gpC2cK2IF81LR4uvz3fEYblZbUW+dvPBszgwHCovynozoYYwGPSejfQ+IS6wvbSPjqRL40sP8/Bfr9pevhe9tUVMveudYMDP8JHibxK2WcOB94ZISNgzFd5I5GfMsD+CAfT9o9yEPmX7fb9xsHroCtOIt/5e/tCNpVhCTkoSupe+YMDh7oPh4n89ZzDKzmY4zCmusiLBx6kc+BcsKCpyK/orC3HoZvC3bZxkOFW2woNxm+ezcGIff6JMPfAjrEiL429fwXHvM1rk8GMq+cu4cDC8xsPESsUpjDvnm4kDH340hTCMpvwIt94wYuPOGyPfXVMy6p3n7uI/K2Th15zcHXY+2SRr7vmwAMOHIs6yEHVKi5A5NcfvLoex1uV37rEwdQdMX1g7ofPZnB53HtPaD4jArM9ZarkGwezM9YMQj86TWajb55L+h2DfdC+JTg6HXPDEK3awvxo/7A3u2zIWr4CxRI/jzso5nj8+IaiT7Npjzjo+WZxnMgv91K/56BV4iQz6Hb0jhmc2JNiLvI/9jrc4WDyqRM4rX15/co5+GPl7FrwZMLWRAj4mh0DU7wvdBT53DNNmfUPPsevojj0/ccBFEVi1BoOoke3DhV570KLD6jqMjpkE4qUrJQyFHEHvR6gkGn+uI3ijEuzRyjSZyWwmPIfFpdyYP88RAbbgx0iRX51bLNS1NedZLIVhfXDBfs5ODJ4S32Rn7Erdz0Htt772mAU3pWxiRStmIfh1cYWmkAtYY0drBma3VLkt3Z/hNk3uLXwF3m70rXX2YROb8eqXnWvdSAG7F7WJg5uL6kTLPKjl+/E0l2w+q4jLJ33IUrkT97zXMVBQ/54MmTlFg0X+e2FUTs4yH5cmAizdfYdoatbH1+4UroXA+XcKg8TXVFvAWbhzpWeuPSG1LBNEvkhDj3yORCsYLzIv+i//RoHucErvND20r9YvFLXzTvDQVMPayzC0yseiRzsCC6uLfJj/b5jZgMvdreG2MVOtcA213ECHJ1YHdPTOH3aZg4uWC11gjo9cvxE3iRQk8VB5GZLU/irLDgJvjwoMIBm2gwrqPFjcih0Tu/VES6nNBiAk0yp9YGDkhsZA2BP87W+MHdZhR+Upt3HOJXtlt/m4PFD0yEiX5iczBrDP9JiWMGcmrUcwyL/1t5U5B+37rcIVUlzVqej+NTYdQEK8f1XbCBp85uOhtoOoyYAF9fXFlpqTtlAD/2BDmCtemkHNuWzMRhvgive4Q12kRMXcnC4gQJDmqtPxbjNifOsDz0s9NHglL3VIPLR9dYt5mCY5oMrbOlaWB+sQiM6QPPNJS2g4PnhQHBcuCQMsvaUSHAOxxuxOH7v0ycNRcjvn16zUtcffoYtreWwVnCzNHsEdB7dsR7sL5mNIbDWTTnOQc7ecKyHhpanWGU+StvDarFpyUIsC/Ov+WawtOhLPZF/3urYH6gvHjV3HzM9LxMva6vX8hb5RutesspOSGo3FYW+8Q22vg6Xp7H+kDc+ENtK8pIdTUR+8SW/3RycCnQNhLG2n5QQ8bVeosjvzl6FZZWqXNwJLJaWjRb5nvkt53Jgd/daHejezcRa5Fv2ylyLRXt8h17kX/22fC8HjXfmY/wsNng84cDLqwiLLH27NVs6Jy+UorGiLo9koOtkWUvkv0mvnUL9FL8AXCAZIzc1REfaL37Dgb7pnHAIrbiOFcP3P48PHGLnaAYmq6yrwVW9Oab927UBuO5D63rguqhfY9NOtHL97MpcFGsrsq6heFUtZAuuh7lvW8PtZVGxcOnq7XYYt07xLPT97Bed5GDbBCtcSSduZhahyr5TPjZs9RYOF+noXacPoSorK4q15suqlfNQlBmWF3PQt/hGD9izziMW4qXJLpCp+ojbydS8Zaz4aq6pi24UZLyvDae9KnrCcNs2fnAl0cEUZrWt1gIiak8eCV53bF3BZmHJGJEvdxsyC5/4+FwLaHLxiYfIf15ss5qD3b2Wo39zA+etRpNhgblp2IaLHYdCDYkp7iv7PFvjTc71+nbHvlSxDTO2US3I4XPridjl9n9qgD3pcCmvEPmcqJIFHIx92MUCnJNau2NcWubf5ODph6ccnJ18DHekwb2LZuMjlp9v/wVXb3rXCEg/ubkrLmdZA6w6eTdfTOIuj4xDHJhWnxOD2XO2RPdqTtSPhsejWmBjb3WUz+ageLO/RuTNd9lgQ7st3AI4WPjCDIPXout9tC4dGYYNIHbNzC4gv7mnEziM3dJM5J0meH3GtqBp2kvkdfuDPuHAVdXbst3AUJqwmd220IY12bDbD1l9DGvZ6hzLx6EIjI+tJLaDyO881+EEqubfWoeLJ2z4blfIbbO8p8i/7rbnCWvS24b/zu7ovAR7yMyKHSNE/l1qX1zI2hh+HBTkdWgGH7Pn24i8omDBAdReu9AYKiyGm4Jn0pwJcP9VWydomKbBI8DNXr4HOXi4MbsjKO7U5qGxMiZe5F3L1+BmPuJyYSzugmEZrAHNtinBht/hqBZVlw9dn8qBb0VbzIPl0LRLeHnasUYVKHIUbypwsWz65A2/1fowARLyPZMgursftqvPbxeygM2/n4S7fjftk0SwjyrsBxYDY5yh7+TTmN7aQ+68xRHfnDuwgd6tjZu21LoAl3pwdP/emKEazbFEVzqWj4VUnzi0mbL3d0z3yrzOJthMrnlja2wiCbCE7evKMFSnfIsxNavrN8F2fnd74C6snB9BgNV8I51tZOPk8Z9ZCjqW5+CMY5viEatYorqIW8rcT0Ng0mI12ixIuPgGx3wsb78Gha1XyHIUx/d3e8zBg8eZOGBoonQpqpznFWEZD03+hCWVnTrmNO4cR3R4gCppIrmLqzi0Z1fY6LRzosjHv/hyhvWl7A1sE41dt/odBz4tLesgD4tNZa1iadpdFE5Pg/FIctHJBU+TL3Y+xpbuczooCm5rGmrgYK9YNVzYOzECxv/ZBI8yC493fMuBU+xa3OaFe9tYc8jsmoOV6+P3lxsUGPLwfNVqegk+6Uz6KzNIWg04yznL2uGIRt5tvaHb3cXRIj/r4IyzHOz90A4tvh9ddIWDoxYH8ZR2TDYVz5YXLjn0hR319xigqH8bL9iamtpc5Dv8Zc4y9PH5/csofut3EcO9wXJ3CDxVjQyDZdtrdYTIpW0xb7OqPb/OwRsbf/Tzr50X0DOzFadcYVETXx42pASEwb1nUpxTUKeLB9HQvmvdMrAkJ4XbYaAsj6awsvL3/oQx7ejcAN71+D4Qm++8/eyAGyzxx/i+9uwyCmbeX9cXTyfyJzdQP3zrVizDCcPf1oc2Pi00sGrZF0vwPTIHt4vi6IffcYSnRTB29VX7zklBMycjDDLvz4yA5u4t8Sh+48/EbBxx8JgzW7fpeQIeQ8tSHtoCpEnxsGDVI19EfYP5jX6gmHq6nM3Zqv4S3AwiFwqRsPL2wcEQXvv6AAg65OMHY2s5VYNxo47WEPmjO76wNRL+7tEKDo6n7caT8i2PLqfwRHJ/Vw+R3/jkO9tElo0+iwHqdb8MO//uQcuWoaphwdWlWMnx5tUgqfMi7IzKmpr7HGTl1MROMT2Ff4lbZc5nNNH5YwA7ZWeYTF+HzWrMmgm4bm9ZYbdT7A2IgT1dcwfDHdUnnMq+tMVY9YWOr63h4uNWvtCs2Gkw1GueJ+DLS+a4bWhDXX08pnP4l1lWUH8wYMceljGRbe6P7JtgI/R61Qob+vwLdVahaqG53ywUD6bM+pOdQZd9+Yri2dYu7DwC3DlcoXkm9bG3e3VoxPZw14HzWQaO2qtYCyv6tvI0CvMxA7AtKLZdw1125dma91B1usbF8yhyn5vjVPMbHkkCmfOm/iL/9MvNPRzcXCEmgl1go4GwYdy+YXDW4iWWf+OsHNwg7vSX+cPcsUPxaL8xn70Z5KcvQsc0V0ffQ49eXMKpRBbAZQ6OveprCq5tTFCxq78Ck7LpRKQnuqV4ziZoMuAKq2vPfdv2sy21ZzJbsnGlu9ix9VnfQ6yHPK5pMReFo58cy/Bkj1kGmN/zrVzkYy7uzeVgc3wN3G4+29pgimeY3Zgk8j5jn2G3veORYAsbN/UZAF+evPQHWb3P2FOdB01mzS/+6N7jKJIzjU3XbXq1mSgqvJzYSS/Mdz07BxyevusVB/cc3Qb6x5vqBTt8Aw2YJTRAERFai721ahIUgkoji8TXV71SIzTEK369yzjjP3yzNX4mEOLl9Ac5hU6wxyEx//qNJ06hF5Ry3U8p8a/8KREa4VDjlxwfhUoRjQ9QR/fTGNR6hSA0xkupggP+HyQ0Ya/HU4WmKJIFR3YP+xhTaUyqlsUpBCf2Uv1/PzFCplP8fGoADpcIzSotOhstNmfv7KHV0KTRVgv2gHr469fXk6B4efAv91qyyzV+DtbhIxVCK6bqg6r/33ct91+fr/wNehn7fiW4mLCo6n7Gnj4GRUp1CrlOcMWL437GorVx5m2Yhvk+7qdDbkZ1W+bQOKMfjdCO3BBnUMmqfEtiX4AUco06Uie4s5v/d/NvR/M3fieKFzTjFHK90J7Zr/PLfltmv61eoWP5Fjwon8YywsxrBJ3QoYqrHY0+dfp3Vzsb1V3+dtX231LmW2mq6//WQ0/y0Pg3HblKqVDrhW5s2RgiwgUv46S64//hQo9/r0m9IFProjRCnM5dzr5ijlCqIzXjMQYSoWelT72Mt/c21iNzR6/EEtPL4uKlck1chFKN0enD/kZkYlw0MjllVqOWxmkiFULfyovGaCp10khFlMyg0gveqJ5trFlcAdHRCiFKLfT7D7P7OUriU3l/yM+fgk/lLH1ZMFmCxxsdQGP9/4OxykESP5UmQqaqdBlXvl+lqQHGaOqFgSjd/wkaDWno",
"user_name": "Write to Datastore/Write Batch to Datastore"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-07-26T20:35:25.221377Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-07-26_13_35_23-15535733344004831356'
location: 'us-central1'
name: 'beamapp-jenkins-0726203459-240887'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-07-26T20:35:25.221377Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-07-26_13_35_23-15535733344004831356]
root: INFO: To access the Dataflow monitoring console, please navigate to
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_35_23-15535733344004831356?project=apache-beam-testing
root: INFO: Job 2019-07-26_13_35_23-15535733344004831356 is in state
JOB_STATE_RUNNING
root: INFO: 2019-07-26T20:35:23.870Z: JOB_MESSAGE_DETAILED: Autoscaling is
enabled for job 2019-07-26_13_35_23-15535733344004831356. The number of workers
will be between 1 and 1000.
root: INFO: 2019-07-26T20:35:23.920Z: JOB_MESSAGE_DETAILED: Autoscaling was
automatically enabled for job 2019-07-26_13_35_23-15535733344004831356.
root: INFO: 2019-07-26T20:35:27.185Z: JOB_MESSAGE_DETAILED: Checking
permissions granted to controller Service Account.
root: INFO: 2019-07-26T20:35:27.873Z: JOB_MESSAGE_BASIC: Worker configuration:
n1-standard-1 in us-central1-a.
root: INFO: 2019-07-26T20:35:28.490Z: JOB_MESSAGE_DETAILED: Expanding
CoGroupByKey operations into optimizable parts.
root: INFO: 2019-07-26T20:35:28.553Z: JOB_MESSAGE_DETAILED: Expanding
GroupByKey operations into optimizable parts.
root: INFO: 2019-07-26T20:35:28.600Z: JOB_MESSAGE_DETAILED: Lifting
ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-07-26T20:35:28.635Z: JOB_MESSAGE_DEBUG: Annotating graph with
Autotuner information.
root: INFO: 2019-07-26T20:35:28.683Z: JOB_MESSAGE_DETAILED: Fusing adjacent
ParDo, Read, Write, and Flatten operations
root: INFO: 2019-07-26T20:35:28.719Z: JOB_MESSAGE_DETAILED: Fusing consumer To
String into Input/Read
root: INFO: 2019-07-26T20:35:28.754Z: JOB_MESSAGE_DETAILED: Fusing consumer
Write to Datastore/Write Batch to Datastore into To Entity
root: INFO: 2019-07-26T20:35:28.797Z: JOB_MESSAGE_DETAILED: Fusing consumer To
Entity into To String
root: INFO: 2019-07-26T20:35:28.843Z: JOB_MESSAGE_DEBUG: Workflow config is
missing a default resource spec.
root: INFO: 2019-07-26T20:35:28.892Z: JOB_MESSAGE_DEBUG: Adding StepResource
setup and teardown to workflow graph.
root: INFO: 2019-07-26T20:35:28.940Z: JOB_MESSAGE_DEBUG: Adding workflow start
and stop steps.
root: INFO: 2019-07-26T20:35:28.988Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-07-26T20:35:29.164Z: JOB_MESSAGE_DEBUG: Executing wait step
start3
root: INFO: 2019-07-26T20:35:29.261Z: JOB_MESSAGE_BASIC: Executing operation
Input/Read+To String+To Entity+Write to Datastore/Write Batch to Datastore
root: INFO: 2019-07-26T20:35:29.322Z: JOB_MESSAGE_DEBUG: Starting worker pool
setup.
root: INFO: 2019-07-26T20:35:29.357Z: JOB_MESSAGE_BASIC: Starting 1 workers in
us-central1-a...
root: INFO: 2019-07-26T20:35:29.752Z: JOB_MESSAGE_BASIC: Worker configuration:
n1-standard-1 in us-central1-a.
root: INFO: 2019-07-26T20:36:23.029Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised
the number of workers to 1 based on the rate of progress in the currently
running step(s).
root: INFO: 2019-07-26T20:37:29.776Z: JOB_MESSAGE_DETAILED: Workers have
started successfully.
root: INFO: 2019-07-26T20:37:29.819Z: JOB_MESSAGE_DETAILED: Workers have
started successfully.
root: INFO: 2019-07-26T20:38:29.731Z: JOB_MESSAGE_BASIC: Worker configuration:
n1-standard-1 in us-central1-a.
--------------------- >> end captured logging << ---------------------
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1140:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_57-1926837498095467616?project=apache-beam-testing.
method_to_use = self._compute_method(p, p.options)
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_42_04-14956909601112182114?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_52_04-2135056096059476016?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_14_00_40-14713533234812040646?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_57-14328198923509947698?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1140:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_39_58-1700086350657230281?project=apache-beam-testing.
method_to_use = self._compute_method(p, p.options)
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_51_12-13718772550676714960?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_53-5818768364053791682?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py>:686:
BeamDeprecationWarning: BigQuerySink is deprecated since 2.11.0. Use
WriteToBigQuery instead.
kms_key=transform.kms_key))
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_51_07-10048582251710616315?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/fileio_test.py>:232:
FutureWarning: MatchAll is experimental.
| 'GetPath' >> beam.Map(lambda metadata: metadata.path))
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/fileio_test.py>:243:
FutureWarning: MatchAll is experimental.
| 'Checksums' >> beam.Map(compute_hash))
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/fileio_test.py>:243:
FutureWarning: ReadMatches is experimental.
| 'Checksums' >> beam.Map(compute_hash))
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_54-9396332865777808166?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_48_24-11729059615216017658?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_57_08-5413874140010873858?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_55-15852845377039223554?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py>:686:
BeamDeprecationWarning: BigQuerySink is deprecated since 2.11.0. Use
WriteToBigQuery instead.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_36_02-6448446236531137454?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_44_52-2799500963002087277?project=apache-beam-testing.
kms_key=transform.kms_key))
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_55_49-13401548833253491813?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/big_query_query_to_table_pipeline.py>:73:
BeamDeprecationWarning: BigQuerySink is deprecated since 2.11.0. Use
WriteToBigQuery instead.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_14_04_47-5350573980647695928?project=apache-beam-testing.
kms_key=kms_key))
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_53-16228945354764082008?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_34_31-11396233653111762919?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_45_11-2760696442263714019?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py>:686:
BeamDeprecationWarning: BigQuerySink is deprecated since 2.11.0. Use
WriteToBigQuery instead.
kms_key=transform.kms_key))
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_55_46-11551509583042163821?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_14_04_35-2225467735482276212?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_59-8573225736831019203?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1140:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
method_to_use = self._compute_method(p, p.options)
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_36_07-2621308163653222166?project=apache-beam-testing.
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:557:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_46_37-9711059535301931466?project=apache-beam-testing.
temp_location = p.options.view_as(GoogleCloudOptions).temp_location
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_57_05-12375618122742026015?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_14_05_51-14276522622963509411?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_24_55-2126996741952549921?project=apache-beam-testing.
Exception in thread Thread-2:
Traceback (most recent call last):
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_35_23-15535733344004831356?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_45_04-187858721200578051?project=apache-beam-testing.
Found:
https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-07-26_13_55_52-12048379172191256024?project=apache-beam-testing.
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py";,>
line 190, in poll_for_job_completion
job_id, page_token=page_token, start_time=last_message_time)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/utils/retry.py";,>
line 197, in wrapper
return fun(*args, **kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py";,>
line 755, in list_messages
response = self._client.projects_locations_jobs_messages.List(request)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py";,>
line 553, in List
config, request, global_params=global_params)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/build/gradleenv/-1734967054/lib/python3.5/site-packages/apitools/base/py/base_api.py";,>
line 731, in _RunMethod
return self.ProcessHttpResponse(method_config, http_response, request)
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/build/gradleenv/-1734967054/lib/python3.5/site-packages/apitools/base/py/base_api.py";,>
line 737, in ProcessHttpResponse
self.__ProcessHttpResponse(method_config, http_response, request))
File
"<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/build/gradleenv/-1734967054/lib/python3.5/site-packages/apitools/base/py/base_api.py";,>
line 604, in __ProcessHttpResponse
http_response, method_config=method_config, request=request)
apitools.base.py.exceptions.HttpNotFoundError: HttpError accessing
<https://dataflow.googleapis.com/v1b3/projects/apache-beam-testing/locations/us-central1/jobs/2019-07-26_13_35_23-15535733344004831356/messages?alt=json&startTime=2019-07-26T20%3A38%3A29.731Z>:
response: <{'x-xss-protection': '0', 'x-content-type-options': 'nosniff',
'transfer-encoding': 'chunked', 'x-frame-options': 'SAMEORIGIN', 'server':
'ESF', 'content-length': '279', '-content-encoding': 'gzip', 'status': '404',
'cache-control': 'private', 'content-type': 'application/json; charset=UTF-8',
'date': 'Fri, 26 Jul 2019 20:40:34 GMT', 'vary': 'Origin, X-Origin, Referer'}>,
content <{
"error": {
"code": 404,
"message": "(75c66d719086766): Information about job
2019-07-26_13_35_23-15535733344004831356 could not be found in our system.
Please double check the id is correct. If it is please contact customer
support.",
"status": "NOT_FOUND"
}
}
>
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery_test.py>:565:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
streaming = self.test_pipeline.options.view_as(StandardOptions).streaming
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1140:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
method_to_use = self._compute_method(p, p.options)
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:557:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = p.options.view_as(GoogleCloudOptions).temp_location
----------------------------------------------------------------------
XML: nosetests-postCommitIT-df-py35.xml
----------------------------------------------------------------------
XML:
<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 41 tests in 3064.131s
FAILED (SKIP=4, failures=1)
> Task :sdks:python:test-suites:dataflow:py35:postCommitIT FAILED
FAILURE: Build failed with an exception.
* Where:
Build file
'<https://builds.apache.org/job/beam_PostCommit_Python35/ws/src/sdks/python/test-suites/dataflow/py35/build.gradle'>
line: 49
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py35:postCommitIT'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 52m 26s
63 actionable tasks: 47 executed, 16 from cache
Publishing build scan...
https://gradle.com/s/rzejg2vdmzcgu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
