postgresql extractor fixes * covering more data types * adding source.conn.database which allows you to connect to a database that has a different name than the source schema (i.e. this is 3-part naming convention). Without this you cannot connect to a database "workgroup" to pull the table "public.http_requests"
Closes #2152 from DepthDeluxe/master Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/34de6bfd Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/34de6bfd Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/34de6bfd Branch: refs/heads/0.12.0 Commit: 34de6bfd3c06399a29937249390454fc5fa57f00 Parents: 3598d10 Author: Colin Heinzmann <[email protected]> Authored: Mon Feb 5 12:05:55 2018 -0800 Committer: Abhishek Tiwari <[email protected]> Committed: Mon Feb 5 12:05:55 2018 -0800 ---------------------------------------------------------------------- .../apache/gobblin/source/jdbc/PostgresqlExtractor.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/34de6bfd/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java index 33b4e00..6876739 100644 --- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java +++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java @@ -44,6 +44,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class PostgresqlExtractor extends JdbcExtractor { + private static final String CONNECTION_DATABASE = "source.conn.database"; private static final String POSTGRES_TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"; private static final String POSTGRES_DATE_FORMAT = "yyyy-MM-dd"; private static final String POSTGRES_HOUR_FORMAT = "HH"; @@ -168,7 +169,8 @@ public class PostgresqlExtractor extends JdbcExtractor { public String getConnectionUrl() { String host = this.workUnitState.getProp(ConfigurationKeys.SOURCE_CONN_HOST_NAME); String port = this.workUnitState.getProp(ConfigurationKeys.SOURCE_CONN_PORT); - String database = this.workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_SCHEMA); + String database = this.workUnitState.getProp(CONNECTION_DATABASE); + return "jdbc:postgresql://" + host.trim() + ":" + port + "/" + database.trim(); } @@ -182,13 +184,15 @@ public class PostgresqlExtractor extends JdbcExtractor { public Map<String, String> getDataTypeMap() { Map<String, String> dataTypeMap = ImmutableMap.<String, String>builder().put("tinyint", "int").put("smallint", "int").put("mediumint", "int") - .put("int", "int").put("bigint", "long").put("float", "float").put("double", "double") - .put("decimal", "double").put("numeric", "double").put("date", "date").put("timestamp", "timestamp") + .put("integer", "int").put("int", "int").put("bigint", "long").put("float", "float").put("double", "double") + .put("double precision", "double").put("decimal", "double").put("numeric", "double").put("date", "date").put("timestamp", "timestamp") + .put("timestamp without time zone", "timestamp").put("timestamp with time zone", "timestamp") .put("datetime", "timestamp").put("time", "time").put("char", "string").put("varchar", "string") .put("varbinary", "string").put("text", "string").put("tinytext", "string").put("mediumtext", "string") - .put("longtext", "string").put("blob", "string").put("tinyblob", "string").put("mediumblob", "string") + .put("character varying", "string").put("longtext", "string").put("blob", "string").put("tinyblob", "string").put("mediumblob", "string") .put("longblob", "string").put("enum", "string").build(); return dataTypeMap; + } @Override
