Repository: incubator-atlas Updated Branches: refs/heads/branch-0.5-incubating bb887fdbf -> e46c19937
ATLAS-17 Parameterize schema API query per typeName (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e46c1993 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e46c1993 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e46c1993 Branch: refs/heads/branch-0.5-incubating Commit: e46c19937ffd373eef9b9b1cd96635718235ab3b Parents: bb887fd Author: Shwetha GS <[email protected]> Authored: Wed Jun 17 14:10:21 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jun 17 14:10:21 2015 +0530 ---------------------------------------------------------------------- release-log.txt | 10 +----- .../atlas/discovery/HiveLineageService.java | 32 ++++++++++++-------- .../src/test/resources/application.properties | 2 +- src/conf/application.properties | 3 +- .../src/main/resources/application.properties | 2 +- 5 files changed, 24 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 56be318..0bfac32 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,18 +1,10 @@ Apache Atlas Release Notes ========================== ---trunk - unreleased - -INCOMPATIBLE CHANGES: - -ALL CHANGES: - -ATLAS-10 Update trunk version to 0.6-incubating-SNAPSHOT (shwethags) - --Release 0.5-incubating ALL CHANGES: - +ATLAS-17 Parameterize schema API query per typeName (shwethags) ATLAS-18 Build fails because of a missing artifact (Ajay Yadava via Venkatesh Seetharam) ATLAS-13 Add project website (Venkatesh Seetharam) ATLAS-12 Update the copyright in Notice and License files (Venkatesh Seetharam) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java index 9764f8f..76d94a3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java @@ -32,6 +32,7 @@ import org.apache.atlas.query.HiveWhereUsedQuery; import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.typesystem.persistence.ReferenceableInstance; import org.apache.commons.configuration.PropertiesConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,26 +54,27 @@ public class HiveLineageService implements LineageService { private static final Option<List<String>> SELECT_ATTRIBUTES = Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"})); + public static final String HIVE_TABLE_SCHEMA_QUERY_PREFIX = "atlas.lineage.hive.table.schema.query."; + private static final String HIVE_TABLE_TYPE_NAME; private static final String HIVE_PROCESS_TYPE_NAME; private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME; private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME; - private static final String HIVE_TABLE_SCHEMA_QUERY; private static final String HIVE_TABLE_EXISTS_QUERY; + private static final PropertiesConfiguration propertiesConf; + static { // todo - externalize this using type system - dog food try { - PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties(); - HIVE_TABLE_TYPE_NAME = conf.getString("atlas.lineage.hive.table.type.name", "DataSet"); - HIVE_PROCESS_TYPE_NAME = conf.getString("atlas.lineage.hive.process.type.name", "Process"); - HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.inputs.name", "inputs"); - HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.outputs.name", "outputs"); - - HIVE_TABLE_SCHEMA_QUERY = - conf.getString("atlas.lineage.hive.table.schema.query", "hive_table where name=\"%s\", columns"); - HIVE_TABLE_EXISTS_QUERY = conf.getString("atlas.lineage.hive.table.exists.query", + propertiesConf = PropertiesUtil.getApplicationProperties(); + HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name", "DataSet"); + HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name", "Process"); + HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name", "inputs"); + HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.outputs.name", "outputs"); + + HIVE_TABLE_EXISTS_QUERY = propertiesConf.getString("atlas.lineage.hive.table.exists.query", "from " + HIVE_TABLE_TYPE_NAME + " where name=\"%s\""); } catch (AtlasException e) { throw new RuntimeException(e); @@ -195,9 +197,10 @@ public class HiveLineageService implements LineageService { public String getSchema(String tableName) throws AtlasException { LOG.info("Fetching schema for tableName={}", tableName); ParamChecker.notEmpty(tableName, "table name cannot be null"); - validateTableExists(tableName); + String typeName = validateTableExists(tableName); - final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName); + final String schemaQuery = + String.format(propertiesConf.getString(HIVE_TABLE_SCHEMA_QUERY_PREFIX + typeName), tableName); return discoveryService.searchByDSL(schemaQuery); } @@ -206,11 +209,14 @@ public class HiveLineageService implements LineageService { * * @param tableName table name */ - private void validateTableExists(String tableName) throws AtlasException { + private String validateTableExists(String tableName) throws AtlasException { final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName); GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery); if (!(queryResult.rows().length() > 0)) { throw new EntityNotFoundException(tableName + " does not exist"); } + + ReferenceableInstance referenceable = (ReferenceableInstance)queryResult.rows().apply(0); + return referenceable.getTypeName(); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/repository/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/application.properties b/repository/src/test/resources/application.properties index 32d40b0..20c0c51 100755 --- a/repository/src/test/resources/application.properties +++ b/repository/src/test/resources/application.properties @@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.local-mode=true #atlas.lineage.hive.process.outputs.name=outputs ## Schema -#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns +atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns ######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/src/conf/application.properties ---------------------------------------------------------------------- diff --git a/src/conf/application.properties b/src/conf/application.properties index 5818701..6c4c7d2 100755 --- a/src/conf/application.properties +++ b/src/conf/application.properties @@ -36,7 +36,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000 #atlas.lineage.hive.process.outputs.name=outputs ## Schema -#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns +atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns +atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns ######### Security Properties ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/webapp/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/webapp/src/main/resources/application.properties b/webapp/src/main/resources/application.properties index 9ade2a7..c9b8408 100755 --- a/webapp/src/main/resources/application.properties +++ b/webapp/src/main/resources/application.properties @@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000 #atlas.lineage.hive.process.outputs.name=outputs ## Schema -#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns +atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns ######### Security Properties #########
