Repository: incubator-gobblin Updated Branches: refs/heads/master 8d297de3e -> 0975312c7
Add `CLUSTER` to the launcher_type enum in the JobHistory DB Modified DatabaseJobHistoryStoreV101 to return values of LauncherTypeEnum.CLUSTER as "YARN" and created DatabaseJobHistoryStoreV102 which keeps them as "CLUSTER". Have DatabaseJobHistoryStoreV102 implement VersionedDatabaseJobHistoryStore Add missing import Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/6e6883ad Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/6e6883ad Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/6e6883ad Branch: refs/heads/master Commit: 6e6883adf3240468f3feccb34e9f644d729d7548 Parents: 16dad89 Author: Joel Baranick <[email protected]> Authored: Mon May 1 16:07:17 2017 -0700 Committer: Joel Baranick <[email protected]> Committed: Thu Jun 22 14:32:49 2017 -0700 ---------------------------------------------------------------------- .../database/DatabaseJobHistoryStoreV101.java | 12 ++++- .../database/DatabaseJobHistoryStoreV102.java | 46 ++++++++++++++++++++ .../V1_0_2__LauncherType_enum_expansion.sql | 18 ++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java ---------------------------------------------------------------------- diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java index 6909afc..beeee3f 100644 --- a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java +++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java @@ -250,6 +250,16 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS return this.dataSource.getConnection(); } + protected String getLauncherType(JobExecutionInfo info) { + if (info.hasLauncherType()) { + if (info.getLauncherType() == LauncherTypeEnum.CLUSTER) { + return LauncherTypeEnum.YARN.name(); + } + return info.getLauncherType().name(); + } + return null; + } + private void upsertJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobName()); @@ -267,7 +277,7 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS upsertStatement.setString(++index, info.hasState() ? info.getState().name() : null); upsertStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1); upsertStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1); - upsertStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name() : null); + upsertStatement.setString(++index, getLauncherType(info)); upsertStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null); upsertStatement.executeUpdate(); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java ---------------------------------------------------------------------- diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java new file mode 100644 index 0000000..f59bf78 --- /dev/null +++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gobblin.metastore.database; + +import gobblin.metastore.JobHistoryStore; + +import java.io.IOException; + +import gobblin.rest.JobExecutionInfo; + + +/** + * An implementation of {@link JobHistoryStore} backed by MySQL. + * + * <p> + * The DDLs for the MySQL job history store can be found under metastore/src/main/resources. + * </p> + * + * @author Joel Baranick + */ +@SupportedDatabaseVersion(isDefault = false, version = "1.0.2") +public class DatabaseJobHistoryStoreV102 extends DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryStore { + + @Override + protected String getLauncherType(JobExecutionInfo info) { + if (info.hasLauncherType()) { + return info.getLauncherType().name(); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql ---------------------------------------------------------------------- diff --git a/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql new file mode 100644 index 0000000..af23b03 --- /dev/null +++ b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql @@ -0,0 +1,18 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER TABLE `gobblin_job_executions` CHANGE `launcher_type` `launcher_type` ENUM('LOCAL', 'MAPREDUCE', 'CLUSTER', 'YARN'); \ No newline at end of file
