Refactor packages and scripts to Atlas (cherry picked from commit 414beba)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/30711973 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/30711973 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/30711973 Branch: refs/heads/dal Commit: 307119733894a5bb9b121b32f75f469c4c2ddf16 Parents: 2604810 Author: Venkatesh Seetharam <[email protected]> Authored: Mon Jun 8 17:41:01 2015 -0700 Committer: Venkatesh Seetharam <[email protected]> Committed: Mon Jun 8 20:43:10 2015 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/falcon/FalconImporter.java | 144 ++ .../apache/atlas/falcon/FalconTypeSystem.java | 169 +++ .../apache/metadata/falcon/FalconImporter.java | 144 -- .../metadata/falcon/FalconTypeSystem.java | 176 --- .../apache/atlas/falcon/FalconImporterTest.java | 108 ++ .../atlas/falcon/FalconTypeSystemTest.java | 35 + .../metadata/falcon/FalconImporterTest.java | 109 -- .../metadata/falcon/FalconTypeSystemTest.java | 35 - .../atlas/hive/bridge/HiveMetaStoreBridge.java | 512 +++++++ .../org/apache/atlas/hive/hook/HiveHook.java | 339 +++++ .../hive/model/HiveDataModelGenerator.java | 482 +++++++ .../apache/atlas/hive/model/HiveDataTypes.java | 52 + .../hive/bridge/HiveMetaStoreBridge.java | 512 ------- .../hadoop/metadata/hive/hook/HiveHook.java | 339 ----- .../hive/model/HiveDataModelGenerator.java | 482 ------- .../metadata/hive/model/HiveDataTypes.java | 52 - .../atlas/hive/hook/BaseSSLAndKerberosTest.java | 142 ++ .../org/apache/atlas/hive/hook/HiveHookIT.java | 379 +++++ .../hook/NegativeSSLAndKerberosHiveHookIT.java | 162 +++ .../hive/hook/SSLAndKerberosHiveHookIT.java | 229 ++++ .../apache/atlas/hive/hook/SSLHiveHookIT.java | 224 +++ .../hive/hook/BaseSSLAndKerberosTest.java | 141 -- .../hadoop/metadata/hive/hook/HiveHookIT.java | 379 ----- .../hook/NegativeSSLAndKerberosHiveHookIT.java | 159 --- .../hive/hook/SSLAndKerberosHiveHookIT.java | 223 --- .../metadata/hive/hook/SSLHiveHookIT.java | 219 --- .../org/apache/atlas/MetadataServiceClient.java | 368 +++++ .../apache/atlas/MetadataServiceException.java | 45 + .../java/org/apache/atlas/PropertiesUtil.java | 58 + .../atlas/security/SecureClientUtils.java | 195 +++ .../atlas/security/SecurityProperties.java | 36 + .../hadoop/metadata/MetadataServiceClient.java | 368 ----- .../metadata/MetadataServiceException.java | 45 - .../apache/hadoop/metadata/PropertiesUtil.java | 58 - .../metadata/security/SecureClientUtils.java | 195 --- .../metadata/security/SecurityProperties.java | 36 - .../apache/atlas/security/BaseSecurityTest.java | 128 ++ .../metadata/security/BaseSecurityTest.java | 128 -- .../java/org/apache/atlas/GraphTransaction.java | 26 + .../atlas/GraphTransactionInterceptor.java | 51 + .../apache/atlas/RepositoryMetadataModule.java | 77 ++ .../atlas/discovery/DiscoveryException.java | 74 + .../atlas/discovery/DiscoveryService.java | 52 + .../atlas/discovery/HiveLineageService.java | 229 ++++ .../apache/atlas/discovery/LineageService.java | 67 + .../apache/atlas/discovery/SearchIndexer.java | 29 + .../graph/DefaultGraphPersistenceStrategy.java | 210 +++ .../graph/GraphBackedDiscoveryService.java | 219 +++ .../atlas/listener/EntityChangeListener.java | 54 + .../atlas/listener/TypesChangeListener.java | 48 + .../org/apache/atlas/repository/Constants.java | 84 ++ .../atlas/repository/DiscoverInstances.java | 76 ++ .../repository/EntityNotFoundException.java | 44 + .../apache/atlas/repository/IRepository.java | 50 + .../atlas/repository/MetadataRepository.java | 162 +++ .../atlas/repository/RepositoryException.java | 47 + .../graph/GraphBackedMetadataRepository.java | 1292 ++++++++++++++++++ .../graph/GraphBackedSearchIndexer.java | 348 +++++ .../atlas/repository/graph/GraphHelper.java | 139 ++ .../atlas/repository/graph/GraphProvider.java | 28 + .../repository/graph/TitanGraphProvider.java | 78 ++ .../repository/memory/AttributeStores.java | 655 +++++++++ .../atlas/repository/memory/ClassStore.java | 94 ++ .../memory/HierarchicalTypeStore.java | 209 +++ .../repository/memory/IAttributeStore.java | 49 + .../atlas/repository/memory/MemRepository.java | 329 +++++ .../memory/ReplaceIdWithInstance.java | 143 ++ .../atlas/repository/memory/StructStore.java | 87 ++ .../atlas/repository/memory/TraitStore.java | 57 + .../typestore/GraphBackedTypeStore.java | 329 +++++ .../atlas/repository/typestore/ITypeStore.java | 48 + .../repository/typestore/StorageException.java | 35 + .../atlas/services/DefaultMetadataService.java | 427 ++++++ .../apache/atlas/services/MetadataService.java | 125 ++ .../hadoop/metadata/GraphTransaction.java | 26 - .../metadata/GraphTransactionInterceptor.java | 51 - .../metadata/RepositoryMetadataModule.java | 78 -- .../metadata/discovery/DiscoveryException.java | 74 - .../metadata/discovery/DiscoveryService.java | 54 - .../metadata/discovery/HiveLineageService.java | 229 ---- .../metadata/discovery/LineageService.java | 67 - .../metadata/discovery/SearchIndexer.java | 29 - .../graph/DefaultGraphPersistenceStrategy.java | 203 --- .../graph/GraphBackedDiscoveryService.java | 219 --- .../metadata/listener/EntityChangeListener.java | 54 - .../metadata/listener/TypesChangeListener.java | 48 - .../hadoop/metadata/repository/Constants.java | 84 -- .../metadata/repository/DiscoverInstances.java | 76 -- .../repository/EntityNotFoundException.java | 46 - .../hadoop/metadata/repository/IRepository.java | 50 - .../metadata/repository/MetadataRepository.java | 162 --- .../repository/RepositoryException.java | 47 - .../graph/GraphBackedMetadataRepository.java | 1292 ------------------ .../graph/GraphBackedSearchIndexer.java | 351 ----- .../metadata/repository/graph/GraphHelper.java | 139 -- .../repository/graph/GraphProvider.java | 28 - .../repository/graph/TitanGraphProvider.java | 79 -- .../repository/memory/AttributeStores.java | 655 --------- .../metadata/repository/memory/ClassStore.java | 94 -- .../memory/HierarchicalTypeStore.java | 209 --- .../repository/memory/IAttributeStore.java | 49 - .../repository/memory/MemRepository.java | 329 ----- .../memory/ReplaceIdWithInstance.java | 143 -- .../metadata/repository/memory/StructStore.java | 87 -- .../metadata/repository/memory/TraitStore.java | 57 - .../typestore/GraphBackedTypeStore.java | 330 ----- .../repository/typestore/ITypeStore.java | 48 - .../repository/typestore/StorageException.java | 35 - .../services/DefaultMetadataService.java | 427 ------ .../metadata/services/MetadataService.java | 125 -- .../org/apache/atlas/query/ClosureQuery.scala | 329 +++++ .../org/apache/atlas/query/Expressions.scala | 739 ++++++++++ .../query/GraphPersistenceStrategies.scala | 327 +++++ .../apache/atlas/query/GremlinEvaluator.scala | 139 ++ .../org/apache/atlas/query/GremlinQuery.scala | 348 +++++ .../org/apache/atlas/query/QueryParser.scala | 398 ++++++ .../org/apache/atlas/query/QueryProcessor.scala | 61 + .../scala/org/apache/atlas/query/Resolver.scala | 157 +++ .../org/apache/atlas/query/TypeUtils.scala | 263 ++++ .../hadoop/metadata/query/ClosureQuery.scala | 330 ----- .../hadoop/metadata/query/Expressions.scala | 739 ---------- .../query/GraphPersistenceStrategies.scala | 327 ----- .../metadata/query/GremlinEvaluator.scala | 139 -- .../hadoop/metadata/query/GremlinQuery.scala | 349 ----- .../hadoop/metadata/query/QueryParser.scala | 398 ------ .../hadoop/metadata/query/QueryProcessor.scala | 61 - .../apache/hadoop/metadata/query/Resolver.scala | 157 --- .../hadoop/metadata/query/TypeUtils.scala | 263 ---- .../atlas/RepositoryServiceLoadingTest.java | 46 + .../test/java/org/apache/atlas/TestUtils.java | 172 +++ .../GraphBackedDiscoveryServiceTest.java | 320 +++++ .../atlas/discovery/HiveLineageServiceTest.java | 593 ++++++++ .../org/apache/atlas/repository/BaseTest.java | 206 +++ .../GraphBackedMetadataRepositoryTest.java | 676 +++++++++ .../graph/GraphRepoMapperScaleTest.java | 272 ++++ .../atlas/repository/memory/ClassTest.java | 76 ++ .../atlas/repository/memory/EnumTest.java | 315 +++++ .../repository/memory/InstanceE2ETest.java | 167 +++ .../atlas/repository/memory/StorageTest.java | 127 ++ .../atlas/repository/memory/StructTest.java | 112 ++ .../atlas/repository/memory/TraitTest.java | 207 +++ .../typestore/GraphBackedTypeStoreTest.java | 128 ++ .../metadata/RepositoryServiceLoadingTest.java | 46 - .../org/apache/hadoop/metadata/TestUtils.java | 172 --- .../GraphBackedDiscoveryServiceTest.java | 320 ----- .../discovery/HiveLineageServiceTest.java | 590 -------- .../hadoop/metadata/repository/BaseTest.java | 206 --- .../GraphBackedMetadataRepositoryTest.java | 676 --------- .../graph/GraphRepoMapperScaleTest.java | 272 ---- .../metadata/repository/memory/ClassTest.java | 76 -- .../metadata/repository/memory/EnumTest.java | 315 ----- .../repository/memory/InstanceE2ETest.java | 164 --- .../metadata/repository/memory/StorageTest.java | 127 -- .../metadata/repository/memory/StructTest.java | 112 -- .../metadata/repository/memory/TraitTest.java | 207 --- .../typestore/GraphBackedTypeStoreTest.java | 128 -- .../org/apache/atlas/query/ExpressionTest.scala | 171 +++ .../org/apache/atlas/query/GremlinTest.scala | 153 +++ .../org/apache/atlas/query/GremlinTest2.scala | 135 ++ .../apache/atlas/query/HiveTitanSample.scala | 345 +++++ .../org/apache/atlas/query/LexerTest.scala | 48 + .../apache/atlas/query/LineageQueryTest.scala | 515 +++++++ .../org/apache/atlas/query/ParserTest.scala | 101 ++ .../apache/atlas/query/QueryTestsUtils.scala | 169 +++ .../hadoop/metadata/query/ExpressionTest.scala | 171 --- .../hadoop/metadata/query/GremlinTest.scala | 153 --- .../hadoop/metadata/query/GremlinTest2.scala | 136 -- .../hadoop/metadata/query/HiveTitanSample.scala | 345 ----- .../hadoop/metadata/query/LexerTest.scala | 48 - .../metadata/query/LineageQueryTest.scala | 515 ------- .../hadoop/metadata/query/ParserTest.scala | 101 -- .../hadoop/metadata/query/QueryTestsUtils.scala | 169 --- src/bin/atlas_config.py | 285 ++++ src/bin/atlas_start.py | 81 ++ src/bin/atlas_stop.py | 59 + src/bin/metadata_config.py | 285 ---- src/bin/metadata_start.py | 81 -- src/bin/metadata_stop.py | 59 - .../atlas/tools/dsl/DynamicTypedStruct.scala | 38 + .../org/apache/atlas/tools/dsl/package.scala | 99 ++ .../apache/atlas/tools/simpleserver/Main.scala | 59 + .../tools/simpleserver/MetadataActor.scala | 121 ++ .../tools/simpleserver/RestInterface.scala | 126 ++ .../atlas/tools/thrift/ThriftParser.scala | 664 +++++++++ .../atlas/tools/thrift/ThriftTypesGen.scala | 316 +++++ .../metadata/tools/dsl/DynamicTypedStruct.scala | 38 - .../hadoop/metadata/tools/dsl/package.scala | 102 -- .../metadata/tools/simpleserver/Main.scala | 59 - .../tools/simpleserver/MetadataActor.scala | 121 -- .../tools/simpleserver/RestInterface.scala | 124 -- .../metadata/tools/thrift/ThriftParser.scala | 665 --------- .../metadata/tools/thrift/ThriftTypesGen.scala | 316 ----- .../org/apache/atlas/tools/dsl/DSLTest.scala | 241 ++++ .../tools/hive/HiveMockMetadataService.scala | 84 ++ .../atlas/tools/thrift/ThriftLexerTest.scala | 525 +++++++ .../atlas/tools/thrift/ThriftParserTest.scala | 772 +++++++++++ .../hadoop/metadata/tools/dsl/DSLTest.scala | 242 ---- .../tools/hive/HiveMockMetadataService.scala | 84 -- .../metadata/tools/thrift/ThriftLexerTest.scala | 525 ------- .../tools/thrift/ThriftParserTest.scala | 772 ----------- .../org/apache/atlas/MetadataException.java | 45 + .../java/org/apache/atlas/ParamChecker.java | 148 ++ .../atlas/classification/InterfaceAudience.java | 48 + .../org/apache/atlas/typesystem/IInstance.java | 38 + .../typesystem/IReferenceableInstance.java | 35 + .../org/apache/atlas/typesystem/IStruct.java | 26 + .../typesystem/ITypedReferenceableInstance.java | 22 + .../apache/atlas/typesystem/ITypedStruct.java | 22 + .../apache/atlas/typesystem/Referenceable.java | 93 ++ .../org/apache/atlas/typesystem/Struct.java | 63 + .../persistence/DownCastStructInstance.java | 71 + .../apache/atlas/typesystem/persistence/Id.java | 242 ++++ .../atlas/typesystem/persistence/MapIds.java | 71 + .../persistence/ReferenceableInstance.java | 103 ++ .../typesystem/persistence/StructInstance.java | 757 ++++++++++ .../typesystem/types/AbstractDataType.java | 37 + .../typesystem/types/AttributeDefinition.java | 93 ++ .../atlas/typesystem/types/AttributeInfo.java | 94 ++ .../atlas/typesystem/types/ClassType.java | 231 ++++ .../atlas/typesystem/types/DataTypes.java | 660 +++++++++ .../typesystem/types/DownCastFieldMapping.java | 53 + .../apache/atlas/typesystem/types/EnumType.java | 90 ++ .../typesystem/types/EnumTypeDefinition.java | 54 + .../atlas/typesystem/types/EnumValue.java | 57 + .../atlas/typesystem/types/FieldMapping.java | 135 ++ .../typesystem/types/HierarchicalType.java | 499 +++++++ .../types/HierarchicalTypeDefinition.java | 76 ++ .../typesystem/types/IConstructableType.java | 34 + .../atlas/typesystem/types/IDataType.java | 31 + .../atlas/typesystem/types/Multiplicity.java | 90 ++ .../typesystem/types/ObjectGraphTraversal.java | 198 +++ .../typesystem/types/ObjectGraphWalker.java | 221 +++ .../atlas/typesystem/types/StructType.java | 204 +++ .../typesystem/types/StructTypeDefinition.java | 62 + .../atlas/typesystem/types/TraitType.java | 77 ++ .../atlas/typesystem/types/TypeSystem.java | 673 +++++++++ .../atlas/typesystem/types/TypeUtils.java | 92 ++ .../typesystem/types/TypedStructHandler.java | 117 ++ .../types/ValueConversionException.java | 55 + .../atlas/typesystem/types/package-info.java | 125 ++ .../atlas/typesystem/types/utils/TypesUtil.java | 85 ++ .../hadoop/metadata/MetadataException.java | 45 - .../apache/hadoop/metadata/ParamChecker.java | 148 -- .../classification/InterfaceAudience.java | 48 - .../hadoop/metadata/typesystem/IInstance.java | 38 - .../typesystem/IReferenceableInstance.java | 35 - .../hadoop/metadata/typesystem/IStruct.java | 26 - .../typesystem/ITypedReferenceableInstance.java | 22 - .../metadata/typesystem/ITypedStruct.java | 22 - .../metadata/typesystem/Referenceable.java | 93 -- .../hadoop/metadata/typesystem/Struct.java | 63 - .../persistence/DownCastStructInstance.java | 71 - .../metadata/typesystem/persistence/Id.java | 242 ---- .../metadata/typesystem/persistence/MapIds.java | 71 - .../persistence/ReferenceableInstance.java | 103 -- .../typesystem/persistence/StructInstance.java | 757 ---------- .../typesystem/types/AbstractDataType.java | 37 - .../typesystem/types/AttributeDefinition.java | 93 -- .../typesystem/types/AttributeInfo.java | 94 -- .../metadata/typesystem/types/ClassType.java | 231 ---- .../metadata/typesystem/types/DataTypes.java | 660 --------- .../typesystem/types/DownCastFieldMapping.java | 53 - .../metadata/typesystem/types/EnumType.java | 90 -- .../typesystem/types/EnumTypeDefinition.java | 54 - .../metadata/typesystem/types/EnumValue.java | 57 - .../metadata/typesystem/types/FieldMapping.java | 135 -- .../typesystem/types/HierarchicalType.java | 500 ------- .../types/HierarchicalTypeDefinition.java | 76 -- .../typesystem/types/IConstructableType.java | 34 - .../metadata/typesystem/types/IDataType.java | 31 - .../metadata/typesystem/types/Multiplicity.java | 90 -- .../typesystem/types/ObjectGraphTraversal.java | 198 --- .../typesystem/types/ObjectGraphWalker.java | 221 --- .../metadata/typesystem/types/StructType.java | 204 --- .../typesystem/types/StructTypeDefinition.java | 62 - .../metadata/typesystem/types/TraitType.java | 77 -- .../metadata/typesystem/types/TypeSystem.java | 674 --------- .../metadata/typesystem/types/TypeUtils.java | 92 -- .../typesystem/types/TypedStructHandler.java | 117 -- .../types/ValueConversionException.java | 55 - .../metadata/typesystem/types/package-info.java | 125 -- .../typesystem/types/utils/TypesUtil.java | 85 -- .../org/apache/atlas/typesystem/TypesDef.scala | 62 + .../typesystem/builders/InstanceBuilder.scala | 217 +++ .../typesystem/builders/TypesBuilder.scala | 188 +++ .../typesystem/json/InstanceSerialization.scala | 320 +++++ .../atlas/typesystem/json/Serialization.scala | 303 ++++ .../typesystem/json/TypesSerialization.scala | 244 ++++ .../hadoop/metadata/typesystem/TypesDef.scala | 65 - .../typesystem/builders/InstanceBuilder.scala | 218 --- .../typesystem/builders/TypesBuilder.scala | 188 --- .../typesystem/json/InstanceSerialization.scala | 320 ----- .../typesystem/json/Serialization.scala | 303 ---- .../typesystem/json/TypesSerialization.scala | 244 ---- .../typesystem/json/SerializationJavaTest.java | 177 +++ .../apache/atlas/typesystem/types/BaseTest.java | 193 +++ .../atlas/typesystem/types/ClassTest.java | 72 + .../apache/atlas/typesystem/types/EnumTest.java | 221 +++ .../atlas/typesystem/types/StructTest.java | 82 ++ .../atlas/typesystem/types/TraitTest.java | 224 +++ .../typesystem/types/TypeInheritanceTest.java | 256 ++++ .../atlas/typesystem/types/TypeSystemTest.java | 139 ++ .../atlas/typesystem/types/ValidationTest.java | 102 ++ .../typesystem/json/SerializationJavaTest.java | 177 --- .../metadata/typesystem/types/BaseTest.java | 194 --- .../metadata/typesystem/types/ClassTest.java | 72 - .../metadata/typesystem/types/EnumTest.java | 221 --- .../metadata/typesystem/types/StructTest.java | 82 -- .../metadata/typesystem/types/TraitTest.java | 224 --- .../typesystem/types/TypeInheritanceTest.java | 256 ---- .../typesystem/types/TypeSystemTest.java | 139 -- .../typesystem/types/ValidationTest.java | 102 -- .../atlas/typesystem/builders/BuilderTest.scala | 80 ++ .../builders/InstanceBuilderTest.scala | 105 ++ .../typesystem/builders/MultiplicityTest.scala | 125 ++ .../typesystem/builders/TypesBuilderTest.scala | 35 + .../typesystem/json/SerializationTest.scala | 242 ++++ .../json/TypesSerializationTest.scala | 300 ++++ .../typesystem/builders/BuilderTest.scala | 80 -- .../builders/InstanceBuilderTest.scala | 105 -- .../typesystem/builders/MultiplicityTest.scala | 125 -- .../typesystem/builders/TypesBuilderTest.scala | 36 - .../typesystem/json/SerializationTest.scala | 242 ---- .../json/TypesSerializationTest.scala | 300 ---- webapp/src/main/java/org/apache/atlas/Main.java | 144 ++ .../org/apache/atlas/examples/QuickStart.java | 478 +++++++ .../atlas/util/CredentialProviderUtility.java | 149 ++ .../web/errors/LoggingExceptionMapper.java | 63 + .../apache/atlas/web/filters/AuditFilter.java | 108 ++ .../filters/MetadataAuthenticationFilter.java | 104 ++ .../atlas/web/listeners/GuiceServletConfig.java | 112 ++ .../atlas/web/listeners/LoginProcessor.java | 160 +++ .../apache/atlas/web/params/AbstractParam.java | 138 ++ .../apache/atlas/web/params/BooleanParam.java | 48 + .../apache/atlas/web/params/DateTimeParam.java | 38 + .../atlas/web/resources/AdminResource.java | 104 ++ .../atlas/web/resources/EntityResource.java | 355 +++++ .../web/resources/HiveLineageResource.java | 178 +++ .../resources/MetadataDiscoveryResource.java | 322 +++++ .../web/resources/RexsterGraphResource.java | 408 ++++++ .../atlas/web/resources/TypesResource.java | 181 +++ .../atlas/web/service/EmbeddedServer.java | 87 ++ .../atlas/web/service/SecureEmbeddedServer.java | 116 ++ .../apache/atlas/web/util/DateTimeHelper.java | 123 ++ .../org/apache/atlas/web/util/Servlets.java | 152 +++ .../java/org/apache/hadoop/metadata/Main.java | 144 -- .../hadoop/metadata/examples/QuickStart.java | 478 ------- .../util/CredentialProviderUtility.java | 146 -- .../web/errors/LoggingExceptionMapper.java | 63 - .../metadata/web/filters/AuditFilter.java | 108 -- .../filters/MetadataAuthenticationFilter.java | 104 -- .../web/listeners/GuiceServletConfig.java | 121 -- .../metadata/web/listeners/LoginProcessor.java | 161 --- .../metadata/web/params/AbstractParam.java | 138 -- .../metadata/web/params/BooleanParam.java | 48 - .../metadata/web/params/DateTimeParam.java | 38 - .../metadata/web/resources/AdminResource.java | 104 -- .../metadata/web/resources/EntityResource.java | 355 ----- .../web/resources/HiveLineageResource.java | 173 --- .../resources/MetadataDiscoveryResource.java | 322 ----- .../web/resources/RexsterGraphResource.java | 408 ------ .../metadata/web/resources/TypesResource.java | 185 --- .../metadata/web/service/EmbeddedServer.java | 87 -- .../web/service/SecureEmbeddedServer.java | 109 -- .../metadata/web/util/DateTimeHelper.java | 123 -- .../hadoop/metadata/web/util/Servlets.java | 152 --- .../atlas/CredentialProviderUtilityIT.java | 272 ++++ .../MetadataAuthenticationKerberosFilterIT.java | 179 +++ .../MetadataAuthenticationSimpleFilterIT.java | 94 ++ .../atlas/web/listeners/LoginProcessorIT.java | 104 ++ .../web/resources/AdminJerseyResourceIT.java | 69 + .../atlas/web/resources/BaseResourceIT.java | 102 ++ .../web/resources/EntityJerseyResourceIT.java | 679 +++++++++ .../resources/HiveLineageJerseyResourceIT.java | 399 ++++++ .../MetadataDiscoveryJerseyResourceIT.java | 223 +++ .../resources/RexsterGraphJerseyResourceIT.java | 86 ++ .../web/resources/TypesJerseyResourceIT.java | 235 ++++ .../web/service/SecureEmbeddedServerIT.java | 61 + .../web/service/SecureEmbeddedServerITBase.java | 224 +++ .../metadata/CredentialProviderUtilityIT.java | 272 ---- .../MetadataAuthenticationKerberosFilterIT.java | 175 --- .../MetadataAuthenticationSimpleFilterIT.java | 94 -- .../web/listeners/LoginProcessorIT.java | 104 -- .../web/resources/AdminJerseyResourceIT.java | 69 - .../metadata/web/resources/BaseResourceIT.java | 102 -- .../web/resources/EntityJerseyResourceIT.java | 679 --------- .../resources/HiveLineageJerseyResourceIT.java | 399 ------ .../MetadataDiscoveryJerseyResourceIT.java | 223 --- .../resources/RexsterGraphJerseyResourceIT.java | 86 -- .../web/resources/TypesJerseyResourceIT.java | 235 ---- .../web/service/SecureEmbeddedServerIT.java | 61 - .../web/service/SecureEmbeddedServerITBase.java | 221 --- 392 files changed, 37072 insertions(+), 37068 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java new file mode 100755 index 0000000..4ad1e3d --- /dev/null +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.falcon; + +import com.google.inject.Inject; +import org.apache.atlas.MetadataException; +import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.typesystem.ITypedInstance; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.types.EnumType; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructType; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.commons.lang.StringUtils; +import org.apache.falcon.client.FalconCLIException; +import org.apache.falcon.client.FalconClient; +import org.apache.falcon.entity.v0.Entity; +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.cluster.Interface; +import org.apache.falcon.entity.v0.cluster.Location; +import org.apache.falcon.entity.v0.cluster.Properties; +import org.apache.falcon.entity.v0.cluster.Property; +import org.apache.falcon.resource.EntityList; + +import javax.xml.bind.JAXBException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class FalconImporter { + + private static final TypeSystem typeSystem = TypeSystem.getInstance(); + + private final FalconClient client; + private final MetadataRepository repository; + + @Inject + public FalconImporter(FalconClient client, MetadataRepository repo) { + this.client = client; + this.repository = repo; + } + + private Entity getEntity(FalconClient client, EntityType type, String name) throws FalconCLIException, JAXBException { + String entityStr = client.getDefinition(type.name(), name); + return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr)); + } + + public void importClusters() throws MetadataException { + try { + EntityList clusters = client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null); + for (EntityList.EntityElement element : clusters.getElements()) { + Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name); + + Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name()); + clusterRef.set("name", cluster.getName()); + + if (cluster.getACL() != null) { + Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name()); + acl.set("owner", cluster.getACL().getOwner()); + acl.set("group", cluster.getACL().getGroup()); + acl.set("permission", cluster.getACL().getPermission()); + StructType aclType = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name()); + clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED)); + } + + if (StringUtils.isNotEmpty(cluster.getTags())) { + String[] parts = cluster.getTags().split(","); + List<ITypedInstance> tags = new ArrayList<>(); + for (String part : parts) { + TraitType tagType = typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()); + String[] kv = part.trim().split("="); + Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name()); + tag.set("name", kv[0]); + tag.set("value", kv[0]); + tags.add(tagType.convert(tag, Multiplicity.REQUIRED)); + } + clusterRef.set("tags", tags); + } + + if (cluster.getProperties() != null) { + clusterRef.set("properties", getMap(cluster.getProperties())); + } + + if (cluster.getLocations() != null) { + List<ITypedInstance> locations = new ArrayList<>(); + for (Location loc : cluster.getLocations().getLocations()) { + Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); + EnumType locationType = typeSystem.getDataType(EnumType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name()); + location.set("type", locationType.fromValue(loc.getName().toUpperCase())); + location.set("path", loc.getPath()); + StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); + locations.add(type.convert(location, Multiplicity.REQUIRED)); + } + clusterRef.set("locations", locations); + } + + if (cluster.getInterfaces() != null) { + List<ITypedInstance> interfaces = new ArrayList<>(); + for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) { + Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); + interfaceStruct.set("type", interfaceFld.getType().name()); + interfaceStruct.set("endpoint", interfaceFld.getEndpoint()); + interfaceStruct.set("version", interfaceFld.getVersion()); + StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); + interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED)); + } + clusterRef.set("interfaces", interfaces); + } + repository.createEntity(clusterRef); + } + } catch (Exception e) { + throw new MetadataException(e); + } + } + + private Map<String, String> getMap(Properties properties) { + Map<String, String> map = new HashMap(); + for (Property property : properties.getProperties()) { + map.put(property.getName().trim(), property.getValue().trim()); + } + return map; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java new file mode 100755 index 0000000..6cbd970 --- /dev/null +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.falcon; + +import com.google.common.collect.ImmutableList; +import org.apache.atlas.MetadataException; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class FalconTypeSystem { + public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class); + public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance(); + private static FalconTypeSystem INSTANCE; + + private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>(); + private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>(); + + private FalconTypeSystem() throws MetadataException { + HierarchicalTypeDefinition<ClassType> cluster = defineCluster(); + //TODO define feed and process + + TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions), + ImmutableList.of(cluster)); + } + + public static FalconTypeSystem getInstance() throws MetadataException { + if (INSTANCE == null) { + synchronized(LOG) { + if (INSTANCE == null) { + INSTANCE = new FalconTypeSystem(); + } + } + } + return INSTANCE; + } + + private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException { + defineACL(); + defineClusterInterface(); + defineClusterLocation(); + defineTags(); + + AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ + new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null), + new AttributeDefinition("locations", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.COLLECTION, false, null), + new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION, false, null), + new AttributeDefinition("properties", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null), + }; + HierarchicalTypeDefinition<ClassType> cluster = + new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(), ImmutableList.<String>of(), attributeDefinitions); + LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name()); + return cluster; + } + + private HierarchicalTypeDefinition<TraitType> defineTags() { + AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ + new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null) + }; + + HierarchicalTypeDefinition<TraitType> traitType = new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(), attributeDefinitions); + LOG.debug("Created definition for " + DefinedTypes.TAG.name()); + traitTypeDefinitions.add(traitType); + return traitType; + } + + private StructTypeDefinition defineClusterLocation() throws MetadataException { + EnumValue values[] = { + new EnumValue("WORKING", 1), + new EnumValue("STAGING", 2), + new EnumValue("TEMP", 3), + }; + + LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name()); + EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values); + TYPE_SYSTEM.defineEnumType(locationType); + + AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ + new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + }; + LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name()); + StructTypeDefinition location = new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions); + structTypeDefinitions.add(location); + return location; + } + + private StructTypeDefinition defineClusterInterface() throws MetadataException { + EnumValue values[] = { + new EnumValue("READONLY", 1), + new EnumValue("WRITE", 2), + new EnumValue("EXECUTE", 3), + new EnumValue("WORKFLOW", 4), + new EnumValue("MESSAGING", 5), + new EnumValue("REGISTRY", 6), + }; + + LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name()); + EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values); + TYPE_SYSTEM.defineEnumType(interfaceType); + + AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ + new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), + }; + LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name()); + StructTypeDefinition interfaceEntity = new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions); + structTypeDefinitions.add(interfaceEntity); + return interfaceEntity; + } + + public static enum DefinedTypes { + ACL, + TAG, + + CLUSTER, + CLUSTER_INTERFACE, + CLUSTER_INTERFACE_TYPE, + CLUSTER_LOCATION, + CLUSTER_LOCATION_TYPE + } + + private StructTypeDefinition defineACL() { + AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ + new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), + Multiplicity.REQUIRED, false, null), + new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(), + Multiplicity.REQUIRED, false, null), + new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(), + Multiplicity.OPTIONAL, false, null), + }; + LOG.debug("Created definition for " + DefinedTypes.ACL.name()); + StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions); + structTypeDefinitions.add(acl); + return acl; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java b/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java deleted file mode 100755 index 1d01d75..0000000 --- a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metadata.falcon; - -import com.google.inject.Inject; -import org.apache.falcon.client.FalconCLIException; -import org.apache.falcon.client.FalconClient; -import org.apache.falcon.entity.v0.Entity; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.entity.v0.cluster.Interface; -import org.apache.falcon.entity.v0.cluster.Location; -import org.apache.falcon.entity.v0.cluster.Properties; -import org.apache.falcon.entity.v0.cluster.Property; -import org.apache.falcon.resource.EntityList; -import org.apache.hadoop.metadata.ITypedInstance; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.Referenceable; -import org.apache.hadoop.metadata.Struct; -import org.apache.hadoop.metadata.repository.MetadataRepository; -import org.apache.hadoop.metadata.typesystem.types.EnumType; -import org.apache.hadoop.metadata.typesystem.types.Multiplicity; -import org.apache.hadoop.metadata.typesystem.types.StructType; -import org.apache.hadoop.metadata.typesystem.types.TraitType; -import org.apache.hadoop.metadata.typesystem.types.TypeSystem; -import org.parboiled.common.StringUtils; - -import javax.xml.bind.JAXBException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class FalconImporter { - - private static final TypeSystem typeSystem = TypeSystem.getInstance(); - - private final FalconClient client; - private final MetadataRepository repository; - - @Inject - public FalconImporter(FalconClient client, MetadataRepository repo) { - this.client = client; - this.repository = repo; - } - - private Entity getEntity(FalconClient client, EntityType type, String name) throws FalconCLIException, JAXBException { - String entityStr = client.getDefinition(type.name(), name); - return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr)); - } - - public void importClusters() throws MetadataException { - try { - EntityList clusters = client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null); - for (EntityList.EntityElement element : clusters.getElements()) { - Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name); - - Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name()); - clusterRef.set("name", cluster.getName()); - - if (cluster.getACL() != null) { - Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name()); - acl.set("owner", cluster.getACL().getOwner()); - acl.set("group", cluster.getACL().getGroup()); - acl.set("permission", cluster.getACL().getPermission()); - StructType aclType = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name()); - clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED)); - } - - if (StringUtils.isNotEmpty(cluster.getTags())) { - String[] parts = cluster.getTags().split(","); - List<ITypedInstance> tags = new ArrayList<>(); - for (String part : parts) { - TraitType tagType = typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()); - String[] kv = part.trim().split("="); - Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name()); - tag.set("name", kv[0]); - tag.set("value", kv[0]); - tags.add(tagType.convert(tag, Multiplicity.REQUIRED)); - } - clusterRef.set("tags", tags); - } - - if (cluster.getProperties() != null) { - clusterRef.set("properties", getMap(cluster.getProperties())); - } - - if (cluster.getLocations() != null) { - List<ITypedInstance> locations = new ArrayList<>(); - for (Location loc : cluster.getLocations().getLocations()) { - Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); - EnumType locationType = typeSystem.getDataType(EnumType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name()); - location.set("type", locationType.fromValue(loc.getName().toUpperCase())); - location.set("path", loc.getPath()); - StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); - locations.add(type.convert(location, Multiplicity.REQUIRED)); - } - clusterRef.set("locations", locations); - } - - if (cluster.getInterfaces() != null) { - List<ITypedInstance> interfaces = new ArrayList<>(); - for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) { - Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); - interfaceStruct.set("type", interfaceFld.getType().name()); - interfaceStruct.set("endpoint", interfaceFld.getEndpoint()); - interfaceStruct.set("version", interfaceFld.getVersion()); - StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); - interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED)); - } - clusterRef.set("interfaces", interfaces); - } - repository.createEntity(clusterRef, clusterRef.getTypeName()); - } - } catch (Exception e) { - throw new MetadataException(e); - } - } - - private Map<String, String> getMap(Properties properties) { - Map<String, String> map = new HashMap(); - for (Property property : properties.getProperties()) { - map.put(property.getName().trim(), property.getValue().trim()); - } - return map; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java b/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java deleted file mode 100755 index 0388b84..0000000 --- a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metadata.falcon; - -import com.google.common.collect.ImmutableList; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.types.AttributeDefinition; -import org.apache.hadoop.metadata.types.ClassType; -import org.apache.hadoop.metadata.types.DataTypes; -import org.apache.hadoop.metadata.types.EnumTypeDefinition; -import org.apache.hadoop.metadata.types.EnumValue; -import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition; -import org.apache.hadoop.metadata.types.Multiplicity; -import org.apache.hadoop.metadata.types.StructTypeDefinition; -import org.apache.hadoop.metadata.types.TraitType; -import org.apache.hadoop.metadata.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -public class FalconTypeSystem { - public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class); - public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance(); - - private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>(); - private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>(); - - private FalconTypeSystem() throws MetadataException { - HierarchicalTypeDefinition<ClassType> cluster = defineCluster(); - //TODO define feed and process - - TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions), - ImmutableList.of(cluster)); - } - - public static FalconTypeSystem getInstance() throws MetadataException { - if (INSTANCE == null) { - synchronized(LOG) { - if (INSTANCE == null) { - INSTANCE = new FalconTypeSystem(); - } - } - } - return INSTANCE; - } - - private FalconTypeSystem() throws MetadataException { - HierarchicalTypeDefinition<ClassType> cluster = defineCluster(); - //TODO define feed and process - - TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions), - ImmutableList.of(cluster)); - } - - private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException { - defineACL(); - defineClusterInterface(); - defineClusterLocation(); - defineTags(); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null), - new AttributeDefinition("locations", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.COLLECTION, false, null), - new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION, false, null), - new AttributeDefinition("properties", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null), - }; - HierarchicalTypeDefinition<ClassType> cluster = - new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(), ImmutableList.<String>of(), attributeDefinitions); - LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name()); - return cluster; - } - - private HierarchicalTypeDefinition<TraitType> defineTags() { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null) - }; - - HierarchicalTypeDefinition<TraitType> traitType = new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(), attributeDefinitions); - LOG.debug("Created definition for " + DefinedTypes.TAG.name()); - traitTypeDefinitions.add(traitType); - return traitType; - } - - private StructTypeDefinition defineClusterLocation() throws MetadataException { - EnumValue values[] = { - new EnumValue("WORKING", 1), - new EnumValue("STAGING", 2), - new EnumValue("TEMP", 3), - }; - - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name()); - EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values); - TYPE_SYSTEM.defineEnumType(locationType); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - }; - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name()); - StructTypeDefinition location = new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions); - structTypeDefinitions.add(location); - return location; - } - - private StructTypeDefinition defineClusterInterface() throws MetadataException { - EnumValue values[] = { - new EnumValue("READONLY", 1), - new EnumValue("WRITE", 2), - new EnumValue("EXECUTE", 3), - new EnumValue("WORKFLOW", 4), - new EnumValue("MESSAGING", 5), - new EnumValue("REGISTRY", 6), - }; - - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name()); - EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values); - TYPE_SYSTEM.defineEnumType(interfaceType); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - }; - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name()); - StructTypeDefinition interfaceEntity = new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions); - structTypeDefinitions.add(interfaceEntity); - return interfaceEntity; - } - - public static enum DefinedTypes { - ACL, - TAG, - - CLUSTER, - CLUSTER_INTERFACE, - CLUSTER_INTERFACE_TYPE, - CLUSTER_LOCATION, - CLUSTER_LOCATION_TYPE - } - - private StructTypeDefinition defineACL() { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(), - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(), - Multiplicity.OPTIONAL, false, null), - }; - LOG.debug("Created definition for " + DefinedTypes.ACL.name()); - StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions); - structTypeDefinitions.add(acl); - return acl; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java new file mode 100755 index 0000000..7562b89 --- /dev/null +++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.falcon; + +import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.falcon.client.FalconClient; +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.cluster.Interface; +import org.apache.falcon.entity.v0.cluster.Interfaces; +import org.apache.falcon.entity.v0.cluster.Interfacetype; +import org.apache.falcon.entity.v0.cluster.Location; +import org.apache.falcon.entity.v0.cluster.Locations; +import org.apache.falcon.resource.EntityList; +import org.testng.annotations.Test; + +import java.io.StringWriter; +import java.util.UUID; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FalconImporterTest { + @Test + public void testImport() throws Exception { + MetadataRepository repo = mock(MetadataRepository.class); + FalconClient client = mock(FalconClient.class); + FalconTypeSystem.getInstance(); + + FalconImporter importer = new FalconImporter(client, repo); + when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, + null)).thenReturn(getEntityList()); + //TODO Set other fields in cluster + when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster()); + when(repo.createEntity(any(IReferenceableInstance.class), anyString())).thenReturn(UUID.randomUUID().toString()); + + importer.importClusters(); + } + + public EntityList getEntityList() { + EntityList.EntityElement[] entities = new EntityList.EntityElement[2]; + entities[0] = new EntityList.EntityElement(); + entities[0].name = "c1"; + entities[1] = new EntityList.EntityElement(); + entities[1].name = "c2"; + return new EntityList(entities); + } + + private Interface getInterface(Interfacetype type, String endpoint) { + Interface clusterInterface = new Interface(); + clusterInterface.setEndpoint(endpoint); + clusterInterface.setType(type); + clusterInterface.setVersion("2.2"); + return clusterInterface; + } + + public String getCluster() throws Exception { + Cluster cluster = new Cluster(); + cluster.setName(RandomStringUtils.randomAlphabetic(10)); + cluster.setColo(RandomStringUtils.randomAlphabetic(5)); + + cluster.setTags("owner=xyz,team=abc"); + Interfaces interfaces = new Interfaces(); + Interface clusterInterface = new Interface(); + clusterInterface.setEndpoint("hdfs://localhost:8030"); + clusterInterface.setType(Interfacetype.WRITE); + clusterInterface.setVersion("2.2"); + interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030")); + interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030")); + interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040";)); + cluster.setInterfaces(interfaces); + + Locations locations = new Locations(); + locations.getLocations().add(getLocation()); + cluster.setLocations(locations); + + StringWriter writer = new StringWriter(); + EntityType.CLUSTER.getMarshaller().marshal(cluster, writer); + return writer.toString(); + } + + public Location getLocation() { + Location location = new Location(); + location.setName("staging"); + location.setPath("/staging"); + return location; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java new file mode 100755 index 0000000..cc2b65f --- /dev/null +++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.falcon; + +import org.apache.atlas.MetadataException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.junit.Assert; +import org.testng.annotations.Test; + +public class FalconTypeSystemTest { + @Test + public void testTypeSystem() throws MetadataException { + FalconTypeSystem.getInstance(); + Assert.assertNotNull(TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name())); + Assert.assertNotNull(TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name())); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java b/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java deleted file mode 100755 index 1104b70..0000000 --- a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metadata.falcon; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.falcon.client.FalconCLIException; -import org.apache.falcon.client.FalconClient; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.entity.v0.cluster.Interface; -import org.apache.falcon.entity.v0.cluster.Interfaces; -import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.cluster.Location; -import org.apache.falcon.entity.v0.cluster.Locations; -import org.apache.falcon.resource.EntityList; -import org.apache.hadoop.metadata.IReferenceableInstance; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.repository.MetadataRepository; -import org.testng.annotations.Test; - -import java.io.StringWriter; -import java.util.UUID; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FalconImporterTest { - @Test - public void testImport() throws Exception { - MetadataRepository repo = mock(MetadataRepository.class); - FalconClient client = mock(FalconClient.class); - FalconTypeSystem.getInstance(); - - FalconImporter importer = new FalconImporter(client, repo); - when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null)).thenReturn(getEntityList()); - //TODO Set other fields in cluster - when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster()); - when(repo.createEntity(any(IReferenceableInstance.class), anyString())).thenReturn(UUID.randomUUID().toString()); - - importer.importClusters(); - } - - public EntityList getEntityList() { - EntityList.EntityElement[] entities = new EntityList.EntityElement[2]; - entities[0] = new EntityList.EntityElement(); - entities[0].name = "c1"; - entities[1] = new EntityList.EntityElement(); - entities[1].name = "c2"; - return new EntityList(entities); - } - - private Interface getInterface(Interfacetype type, String endpoint) { - Interface clusterInterface = new Interface(); - clusterInterface.setEndpoint(endpoint); - clusterInterface.setType(type); - clusterInterface.setVersion("2.2"); - return clusterInterface; - } - - public String getCluster() throws Exception { - Cluster cluster = new Cluster(); - cluster.setName(RandomStringUtils.randomAlphabetic(10)); - cluster.setColo(RandomStringUtils.randomAlphabetic(5)); - - cluster.setTags("owner=xyz,team=abc"); - Interfaces interfaces = new Interfaces(); - Interface clusterInterface = new Interface(); - clusterInterface.setEndpoint("hdfs://localhost:8030"); - clusterInterface.setType(Interfacetype.WRITE); - clusterInterface.setVersion("2.2"); - interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030")); - interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030")); - interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040";)); - cluster.setInterfaces(interfaces); - - Locations locations = new Locations(); - locations.getLocations().add(getLocation()); - cluster.setLocations(locations); - - StringWriter writer = new StringWriter(); - EntityType.CLUSTER.getMarshaller().marshal(cluster, writer); - return writer.toString(); - } - - public Location getLocation() { - Location location = new Location(); - location.setName("staging"); - location.setPath("/staging"); - return location; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java b/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java deleted file mode 100755 index a067249..0000000 --- a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metadata.falcon; - -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.types.ClassType; -import org.apache.hadoop.metadata.types.TraitType; -import org.apache.hadoop.metadata.types.TypeSystem; -import org.junit.Assert; -import org.testng.annotations.Test; - -public class FalconTypeSystemTest { - @Test - public void testTypeSystem() throws MetadataException { - FalconTypeSystem.getInstance(); - Assert.assertNotNull(TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name())); - Assert.assertNotNull(TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name())); - } -}
