Ayub Khan created ATLAS-1572:
--------------------------------
Summary: sqoop metadata ingest fails with
"org.apache.atlas.exception.AtlasBaseException: expected type List or Set;
found org.apache.atlas.typesystem.Referenceable"
Key: ATLAS-1572
URL: https://issues.apache.org/jira/browse/ATLAS-1572
Project: Atlas
Issue Type: Bug
Components: atlas-intg
Affects Versions: trunk, 0.8-incubating
Reporter: Ayub Khan
Priority: Blocker
Fix For: trunk, 0.8-incubating
Sqoop import succeeds but the metadata ingest fails with below error
{noformat}
2017-02-20 11:20:43,179 WARN - [NotificationHookConsumer thread-0:] ~ Max
retries exceeded for message [{Id='(type: sqoop_dbdatastore, id:
<unassigned>)', traits=[], values={owner=hrt_qa,
storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=,
source=createSqoopTablejmnwwf8rji, storeUse=TABLE}}, {Id='(type: hive_db, id:
<unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}, {Id='(type: hive_table, id: <unassigned>)', traits=[],
values={name=createsqooptablejmnwwf8rji,
qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db,
id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}}}, {Id='(type: sqoop_process, id: <unassigned>)', traits=[],
values={outputs={Id='(type: hive_table, id: <unassigned>)', traits=[],
values={name=createsqooptablejmnwwf8rji,
qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db,
id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}}}, commandlineOpts={db.clear.staging.table=false,
hive.import=true, db.require.password=true,
codegen.output.delimiters.enclose=0, codegen.input.delimiters.field=0,
customtool.options.jsonmap={}, hive.compute.stats.table=false,
db.connect.string=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
incremental.mode=None, db.table=createSqoopTablejmnwwf8rji, verbose=false,
codegen.output.delimiters.enclose.required=false, mapreduce.num.mappers=4,
hdfs.append.dir=false, direct.import=false, hive.drop.delims=false,
hive.overwrite.table=false, hbase.bulk.load.enabled=false,
hive.fail.table.exists=false, relaxed.isolation=false,
hdfs.delete-target.dir=false, split.limit=null, db.username=sqoop,
codegen.input.delimiters.enclose.required=false, codegen.output.dir=.,
import.direct.split.size=0, reset.onemapper=false,
codegen.output.delimiters.record=10, temporary.dirRoot=_sqoop,
hcatalog.create.table=false, db.batch=false, import.fetch.size=-2147483648,
accumulo.max.latency=5000, hdfs.file.format=TextFile,
codegen.output.delimiters.field=1, mainframe.input.dataset.type=p,
codegen.output.delimiters.escape=0, hcatalog.drop.and.create.table=false,
import.max.inline.lob.size=16777216, hbase.create.table=false,
codegen.auto.compile.dir=true,
codegen.compile.dir=/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef,
codegen.input.delimiters.enclose=0, export.new.update=UpdateOnly,
enable.compression=false, skip.dist.cache=false, accumulo.batch.size=10240000,
hdfs.warehouse.dir=/user/hrt_qa, codegen.input.delimiters.record=0,
codegen.input.delimiters.escape=0, accumulo.create.table=false},
inputs={Id='(type: sqoop_dbdatastore, id: <unassigned>)', traits=[],
values={owner=hrt_qa,
storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=,
source=createSqoopTablejmnwwf8rji, storeUse=TABLE}}, qualifiedName=sqoop import
--connect
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default
--hive-table createsqooptablejmnwwf8rji --hive-cluster cl1, name=sqoop import
--connect
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default
--hive-table createsqooptablejmnwwf8rji --hive-cluster cl1,
startTime=2017-02-20T11:20:04.901Z, endTime=2017-02-20T11:20:40.055Z,
userName=hrt_qa, operation=import}}] (NotificationHookConsumer:324)
org.apache.atlas.exception.AtlasBaseException: expected type List or Set; found
org.apache.atlas.typesystem.Referenceable
at
org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:50)
at
org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:34)
at
org.apache.atlas.repository.converters.AtlasStructFormatConverter.fromV1ToV2(AtlasStructFormatConverter.java:169)
at
org.apache.atlas.repository.converters.AtlasEntityFormatConverter.fromV1ToV2(AtlasEntityFormatConverter.java:74)
at
org.apache.atlas.repository.converters.AtlasInstanceConverter.fromV1toV2Entity(AtlasInstanceConverter.java:222)
at
org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasEntities(AtlasInstanceConverter.java:204)
at
org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.handleMessage(NotificationHookConsumer.java:256)
at
org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.run(NotificationHookConsumer.java:230)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2017-02-20 11:21:06,094 ERROR - [NotificationHookConsumer thread-0:] ~
[DROPPED_NOTIFICATION]
{"version":{"version":"1.0.0"},"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table
createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table
createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888059","version":0,"typeName":"sqoop_process","state":"ACTIVE"},"typeName":"sqoop_process","values":{"name":"sqoop
import --connect
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default
--hive-table createsqooptablejmnwwf8rji --hive-cluster
cl1","startTime":"2017-02-20T11:20:04.901Z","outputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},"commandlineOpts":{"db.clear.staging.table":"false","hive.import":"true","db.require.password":"true","codegen.output.delimiters.enclose":"0","codegen.input.delimiters.field":"0","customtool.options.jsonmap":"{}","hive.compute.stats.table":"false","db.connect.string":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","incremental.mode":"None","db.table":"createSqoopTablejmnwwf8rji","verbose":"false","codegen.output.delimiters.enclose.required":"false","mapreduce.num.mappers":"4","hdfs.append.dir":"false","direct.import":"false","hive.drop.delims":"false","hive.overwrite.table":"false","hbase.bulk.load.enabled":"false","hive.fail.table.exists":"false","relaxed.isolation":"false","hdfs.delete-target.dir":"false","split.limit":"null","db.username":"sqoop","codegen.input.delimiters.enclose.required":"false","codegen.output.dir":".","import.direct.split.size":"0","reset.onemapper":"false","codegen.output.delimiters.record":"10","temporary.dirRoot":"_sqoop","hcatalog.create.table":"false","db.batch":"false","import.fetch.size":"-2147483648","accumulo.max.latency":"5000","hdfs.file.format":"TextFile","codegen.output.delimiters.field":"1","mainframe.input.dataset.type":"p","codegen.output.delimiters.escape":"0","hcatalog.drop.and.create.table":"false","import.max.inline.lob.size":"16777216","hbase.create.table":"false","codegen.auto.compile.dir":"true","codegen.compile.dir":"/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef","codegen.input.delimiters.enclose":"0","export.new.update":"UpdateOnly","enable.compression":"false","skip.dist.cache":"false","accumulo.batch.size":"10240000","hdfs.warehouse.dir":"/user/hrt_qa","codegen.input.delimiters.record":"0","codegen.input.delimiters.escape":"0","accumulo.create.table":"false"},"endTime":"2017-02-20T11:20:40.055Z","inputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table
createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table
createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},"operation":"import","qualifiedName":"sqoop
import --connect
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default
--hive-table createsqooptablejmnwwf8rji --hive-cluster
cl1","userName":"hrt_qa"},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_CREATE","user":"hrt_qa"}}
(FAILED:339)
{noformat}
Steps to repro:
Run the below sqoop import command
{noformat}
/usr/hdp/current/sqoop-client/bin/sqoop import --connect
jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests
--username sqoop --password sqoop --hive-import --warehouse-dir /user/hrt_qa
--table createSqoopTablejmnwwf8rji
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
